From d6caf09d65cca85ad56fe6b2f5d938fa809453df Mon Sep 17 00:00:00 2001 From: zelig Date: Tue, 2 Jan 2018 13:19:42 +0100 Subject: [PATCH 001/210] swarm/network: network rewrite basic skeleton --- swarm/network/README.md | 150 ++++++ swarm/network/bitvector/bitvector.go | 34 ++ .../network/priorityqueues/priorityqueues.go | 95 ++++ swarm/network/protocol.go | 10 + swarm/network/requests.go | 166 +++++++ swarm/network/streamer.go | 427 ++++++++++++++++++ swarm/network/syncer.go | 266 +++++++++++ 7 files changed, 1148 insertions(+) create mode 100644 swarm/network/README.md create mode 100644 swarm/network/bitvector/bitvector.go create mode 100644 swarm/network/priorityqueues/priorityqueues.go create mode 100644 swarm/network/requests.go create mode 100644 swarm/network/streamer.go create mode 100644 swarm/network/syncer.go diff --git a/swarm/network/README.md b/swarm/network/README.md new file mode 100644 index 0000000000..335ff42e61 --- /dev/null +++ b/swarm/network/README.md @@ -0,0 +1,150 @@ +# Requirements + +- eventual consistency: so each chunk historical should be syncable +- since the same chunk can and will arrive from many peers, (network traffic should be +optimised (only one transfer of data per chunk) +- explicit request deliveries should be prioritised higher than recent chunks received +during the ongoing session which in turn should be higher than historical chunks. +- insured chunks should get receipted for finger pointing litigation, the receipts storage +should be organised efficiently, upstream peer should also be able to find these +receipts for a deleted chunk easily to refute their challenge. +- syncing should be resilient to cut connections, metadata should be persisted that +keep track of syncing state across sessions. +- extra data structures to support syncing should be kept at minimum +- syncing is organized separately for chunk types (resource update v content chunk) + + +When two peers connect, the bidirectional protocol is a result of two identical +syncing protocols mirrored. So take one direction and call the two parties +upstream and downstream peer. + +when a new chunk is stored, its hash is appended to a boundless stream maintained for +each kademlia bin. This state is always permanently recorded periodically possibly +using mutable resource update scheme. + +At any point in time (with n chunks in total ) the swarm hash of this hash stream state is +well defined. Upstream peer pushes so that the reader reads sequential hashes and +periodically calculates the swarm hash of their stream. + +peers syncronise by getting the chunks closer to the downstream peer than to the upstream one. +Consequently peers just sync all stored items for the kad bin the receiving peer falls into. +The special case of nearest neighbour sets is handled by the downstream peer +indicating they want to sync all kademlia bins with proximity equal to or higher +than their depth. + +When peers connect upstream peer sends the latest sync state (item index/cursor length) +for each relevant bin downstream peer is interested in. +This sync state represents the initial state of a sync connection session. +Conversely downstream peers maintain the last state (swarm hash with length) which +the ranges of covered offsets. + +Retrieval is dictated by downstream peers simply using the the chunker joiner to read certain offsets. + +Syncing chunks created during the session by the upstream peer is called session Syncing +while syncing of earlier chunks is historical syncing. + +Historical syncing is simply carried out by iteratively requesting ranges of hash offsets. +For simplicity we assume that the minimum unit requested is a chunk. If every 128 chunks +is considered syncable one can use data chunk index instead of offset in byte length. +Note that data chunk here is a sequence of hashes above one ground level of stream of content. + +Once the relevant chunk is retrieved, downstream peer looks up all hash segments in its localstore +and sends to the upstream peer a message with a 128-long bitvector (uint16) to indicate +missing chunks (e.g., for chunk `k`, hash with chunk internal index which case ) +new items. In turn upstream peer sends the relevant chunk data alongside their index. + +On sending chunks there is a priority queue system. If during looking up hashes in its localstore, +downstream peer hits on an open request then a retrieve request is sent immediately to the upstream peer indicating +that no extra round of checks is needed. If another peers syncer hits the same open request, it is slightly unsafe to not ask +that peer too: if the first one disconnects before delivering or fails to deliver and therefore gets +disconnected, we should still be able to continue with the other. The minimum redundant traffic coming from such simultaneous +eventualities should be sufficiently rare not to warrant more complex treatment. + +Session syncing involves downstream peer to request a new state on a bin from upstream. +using the new state, the range (of chunks) between the previous state and the new one are retrieved +and chunks are requested identical to the historical case. After receiving all the missing chunks +from the new hashes, downstream peer will request a new range. If this happens before upstream peer updates a new state, +we say that session syncing is live or the two peers are in sync. + +If there is no historical backlog, downstream peer is said to be fully synced with the upstream. +If a peer is fully synced with all its storer peers, it can advertise itself as globally fully synced. + +For healthy operation, however, it is expected that the session is regularly in sync. If this is +not the case, that indicates that traffic during the session is continuously more than the peer can cope with and +downstream peer is effectively accumulating a historical backlog. + +The downstream peer persists the record of the last synced offset. When the two peers disconnect and +reconnect syncing can start from there. +This situation however can also happen while historical syncing is not yet complete. +Effectively this means that the peer needs to persist a record of an arbitrary array of offset ranges covered. + +The swarm hash over the hash stream has many advantages. It implements a provable data transfer +and provide efficient storage for receipts in the form of inclusion proofs useable for finger pointing litigation. +When challenged on a missing chunk, upstream peer will provide an inclusion proof of a chunk hash against the state of the +sync stream. In order to be able to generate such an inclusion proof, upstream peer needs to store the hash index (counting consecutive hash-size segments) alongside the chunk data and preserve it even when the chunk data is deleted until the chunk is no longer insured. +if there is no valid insurance on the files the entry may be deleted. +As long as the chunk is preserved, no takeover proof will be needed since the node can respond to any challenge. +However, once the node needs to delete an insured chunk for capacity reasons, a receipt should be available to +refute the challenge by finger pointing to a downstream peer. +As part of the deletion protocol then, hashes of insured chunks to be removed are pushed to an infinite stream for every bin. + +Downstream peer on the other hand needs to make sure that they can only be finger pointed about a chunk they did receive and store. +For this the check of a state should be exhaustive. If historical syncing finishes on one state, all hashes before are covered, no +surprises. In other words historical syncing this process is self verifying. With session syncing however, it is not enough to check going back covering the range from old offset to new. Continuity (i.e., that the new state is extension of the old) needs to be verified: after downstream peer reads the range into a buffer, it appends the buffer the last known state at the last known offset and verifies the resulting hash matches +the latest state. The same goes with intervals of historical syncing where the state used to verify the preceding interval is different from the one used to cover the current range. In these cases too, verification by append is needed for complete security for downstream peer. + +Upstream peer signs the states, downstream peers can use as handover proofs. +Downstream peers sign off on a state together with an initial offset. +latter needed for reasonable sized not ever growing syncer +possible is eachn chunk needs to be reentered if they remain insured + +Once historical syncing is complete and the session does not lag, downstream peer only preserves the latest upstream state and store the signed version. + +Upstream peer needs to keep the latest takeover states: each deleted chunk's hash should be covered by takeover proof of at least one peer. If historical syncing is complete, upstream peer typically will store only the latest takeover proof from downstream peer. +Crucially, the structure is totally independent of the number of peers in the bin, so it scales extremely well. + +implementation + +The simplest protocol just involves upstream peer to prefix the key with the kademlia proximity order (say 0-15 or 0-31) +and simply iterate on index per bin when syncing with a peer. + +priority queues are used for sending chunks so that user triggered requests should be responded to first, session syncing second, and historical with lower priority. +The request on chunks remains implemented as a dataless entry in the memory store. +The lifecycle of this object should be more carefully thought through, ie., when it fails to retrieve it should be removed. + + + +Model 1 +The main appeal in this model is that downstream driven syncing falls back to the exact same retrieval mechanism as the one used when downloading a file. If the chunks of the hash stream (datachunks as well as intermediate chunks of the swarm tree above it) are themselves distributed by upstream peer in the normal way, then requesting them from swarm is viable. +This requires no extra implementation. However, it is unlikely this is feasible for live syncing since the chunks' delay to arrive at their destination has a lag exactly due to session syncronisation. +Note that if upstream peer handles chunks of the hash stream as normal chunks, there are issues. One is that some of these chunks will fall in the same bin as the one building leading to a situation where the hash stream grows even though there are no external chunks received. If upstream peer wishes to use finger pointing proofs, it has to either store these chunks themselves or insure them. + + +Model 2 +In this model, when retrieving the hash stream from the state, requests are targeted to the upstream peer. +The simplest way to generate the right requests from a sync state is to have a +peer specific dpa chunkstore for chunker join. This can be used by all chunk types and all bins. +All it does is when picking retrieve tasks off the chunk channel, it marks the chunk with a reference to the +upstream peer so that when netstore does not find it, the request is sent to the upstream peer (only). +(Normally the request would be routed based on its address). +We need to introduce a special field on the chunk to indicate that the data should be requested from a particular peer. +Upstream peer could still distribute these chunks used in the hash stream in swarm as usual, e.g., long non-synced early +history. +This model does not suffer from the availability lag of the first one, correctly puts the burden on upstream peer to preserve +chunks of the hash stream either in swarm or not. + +Model 3 +in another alternative upstream peer sends only the data level (the hash stream interval) not all the intermediate chunks. This saves on traffic and downstream peer can calculate the state by append to verify against the state (root hash). +This mode of operation is anyhow feasible in cases where the same peer having the top request will be expected to have all the children chunks +of an intermediate one. This is the case for syncing and hash stream or when light swarm clients channel all their requests to a proxy node (public database lookup or unencrypted content). + +This model would require no peer specific dpa and would involve the chunker split only to get the right hand side for the append for verification. + +Delivery requests + +once the appropriate ranges of the hashstream are retrieved and buffered, downstream peer just scans the hashes, looks them up in localstore, if not found, create a request entry with specific reference to the upstream peer as source. +The range is referenced by the chunk index. Alongside the name (indicating the stream, e.g., content chunks for bin 6) and the range +downstream peer sends a 128 long bitvector indicating which chunks are needed. +Newly created requests are satisfied bound together in a waitgroup which when done, will prompt sending the next one. +to be able to do check and storage concurrently, we keep a buffer of one, we start with two chunks. +For session syncing too, if it has not arrived sby the time the next chunk. diff --git a/swarm/network/bitvector/bitvector.go b/swarm/network/bitvector/bitvector.go new file mode 100644 index 0000000000..1769fa4f1d --- /dev/null +++ b/swarm/network/bitvector/bitvector.go @@ -0,0 +1,34 @@ +package bitvector + +type BitVector struct { + len int + b []byte +} + +func New(l int) *BitVector { + return NewFromBytes(make([]byte, l/8+1), l) +} + +func NewFromBytes(b []byte, l int) *BitVector { + return &BitVector{ + len: l, + b: b, + } +} + +func (bv *BitVector) Get(i int) bool { + bi := i / 8 + return uint8(bv.b[bi])&0x1>>uint(i%8) != 0 +} + +func (bv *BitVector) Set(i int, v bool) { + bi := i / 8 + cv := bv.Get(i) + if cv != v { + bv.b[bi] ^= 0x1 >> uint8(i%8) + } +} + +func (bv *BitVector) Bytes() []byte { + return bv.b +} diff --git a/swarm/network/priorityqueues/priorityqueues.go b/swarm/network/priorityqueues/priorityqueues.go new file mode 100644 index 0000000000..d64124d6c2 --- /dev/null +++ b/swarm/network/priorityqueues/priorityqueues.go @@ -0,0 +1,95 @@ +// package priority_queues implement a channel based priority queue +// over arbitrary types. It provides an +// an autopop loop applying a function to the items always respecting +// their priority. The structure is only quasi consistent ie., if a lower +// priority item is autopopped, it is guaranteed that there was a point +// when no higher priority item was present, ie. it is not guaranteed +// that there was any point where the lower priority item was present +// but the higher was not + +package priorityqueues + +import ( + "context" + "errors" +) + +var ( + errContention = errors.New("queue contention") + errBadPriority = errors.New("bad priority") + + wakey = struct{}{} +) + +// PriorityQueues is the basic structure +type PriorityQueues struct { + queues []chan interface{} + wakeup chan struct{} +} + +// New is the constructor for PriorityQueues +func New(n int, l int) *PriorityQueues { + var queues = make([]chan interface{}, n) + for i := range queues { + queues[i] = make(chan interface{}, l) + } + return &PriorityQueues{ + queues: queues, + wakeup: make(chan struct{}, 1), + } +} + +// Run is a forever loop popping items from the queues +func (pq *PriorityQueues) Run(ctx context.Context, f func(interface{})) { + top := len(pq.queues) - 1 + p := top + q := pq.queues[p] +READ: + for { + select { + case <-ctx.Done(): + return + case x := <-q: + f(x) + p = top + default: + if p > 0 { + p-- + continue READ + } + p = top + select { + case <-ctx.Done(): + return + case <-pq.wakeup: + } + } + } +} + +// Push pushes an item to the appropriate queue specified in the priority argument +// if context is given it waits until either the item is pushed or the Context aborts +// otherwise returns errContention if the queue is full +func (pq *PriorityQueues) Push(ctx context.Context, x interface{}, p int) error { + if p < 0 || p >= len(pq.queues) { + return errBadPriority + } + if ctx == nil { + select { + case pq.queues[p] <- x: + default: + return errContention + } + } else { + select { + case pq.queues[p] <- x: + case <-ctx.Done(): + return ctx.Err() + } + } + select { + case pq.wakeup <- wakey: + default: + } + return nil +} diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index 79471c8ec6..1f25464b11 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -103,6 +103,7 @@ type BzzConfig struct { // Bzz is the swarm protocol bundle type Bzz struct { + Streamer *Streamer *Hive localAddr *BzzAddr mtx sync.Mutex @@ -116,6 +117,7 @@ type Bzz struct { // * peer store func NewBzz(config *BzzConfig, kad Overlay, store StateStore) *Bzz { return &Bzz{ + Streamer: NewStreamer(), Hive: NewHive(config.HiveParams, kad, store), localAddr: &BzzAddr{config.OverlayAddr, config.UnderlayAddr}, handshakes: make(map[discover.NodeID]*HandshakeMsg), @@ -157,6 +159,14 @@ func (b *Bzz) Protocols() []p2p.Protocol { NodeInfo: b.Hive.NodeInfo, PeerInfo: b.Hive.PeerInfo, }, + { + Name: StreamerSpec.Name, + Version: StreamerSpec.Version, + Length: StreamerSpec.Length(), + Run: b.RunProtocol(StreamerSpec, b.Streamer.Run), + NodeInfo: b.Streamer.NodeInfo, + PeerInfo: b.Streamer.PeerInfo, + }, } } diff --git a/swarm/network/requests.go b/swarm/network/requests.go new file mode 100644 index 0000000000..311acfc927 --- /dev/null +++ b/swarm/network/requests.go @@ -0,0 +1,166 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package network + +import ( + "bytes" + "encoding/binary" + "fmt" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +/* + Retrieve Request and store Request handling +*/ + +// Handler for storage/retrieval related protocol requests +type RequestHandler struct { + netStore *storage.NetStore +} + +// NewEwquestHandler creates a new RequestHandler +// netStore to +func NewRequestHandler(netStore *storage.NetStore) *RequestHandler { + return &RequestHandler{ + netStore: netStore, // entrypoint internal + } +} + +/* +Retrieve request + +MaxSize specifies the maximum size that the peer will accept. This is useful in +particular if we allow storage and delivery of multichunk payload representing +the entire or partial subtree unfolding from the requested root key. +So when only interested in limited part of a stream (infinite trees) or only +testing chunk availability etc etc, we can indicate it by limiting the size here. + +Request ID can be newly generated or kept from the request originator. + +*/ +type retrieveRequestMsgData struct { + Key storage.Key // target Key address of chunk to be retrieved + Id uint64 // request id, request is a lookup if missing or zero + MaxSize uint64 // maximum size of delivery accepted + from Peer // +} + +func (self retrieveRequestMsgData) String() string { + var from string + if self.from == nil { + from = "ourselves" + } else { + from = fmt.Sprintf("%x", self.from.OverlayAddr()) + } + var target []byte + if len(self.Key) > 3 { + target = self.Key[:4] + } + return fmt.Sprintf("Requester: %v, Key: %x; ID: %v, MaxSize: %v", from, target, self.Id, self.MaxSize) +} + +// entrypoint for retrieve requests coming from the bzz wire protocol +// checks swap balance - return if peer has no credit +func (self *RequestHandler) handleRetrieveRequestMsg(msg interface{}, p Peer) error { + req := msg.(*retrieveRequestMsgData) + req.from = p + // TODO: + // swap - record credit for 1 request + // note that only charge actual reqsearches + + // call storage.NetStore#Get which + // blocks until local retrieval finished + // launches cloud retrieval + chunk, _ := self.netStore.Get(req.Key) + rs := chunk.Req + if rs != nil { + rs = storage.NewRequest() + self.addRequester(rs, req) + chunk.Req = rs + } + + // check if we can immediately deliver + if chunk.SData != nil { + if req.MaxSize == 0 || int64(req.MaxSize) >= chunk.Size { + err := self.netStore.Deliver(chunk) + if err != nil { + log.Trace(fmt.Sprintf("%v - content found, delivery error: %v", req.Key.Log(), err)) + return nil + } + log.Trace(fmt.Sprintf("%v - content found, delivering...", req.Key.Log())) + } else { + log.Trace(fmt.Sprintf("%v - content found, not wanted", req.Key.Log())) + } + } else { + log.Trace(fmt.Sprintf("content not found locally, retrieve via bzz", req.Key.Log())) + } + return nil +} + +/* +adds a new peer to an existing open request +only add if less than requesterCount peers forwarded the same request id so far +note this is done irrespective of status (searching or found) +*/ +func (self *RequestHandler) addRequester(rs *storage.RequestStatus, req *retrieveRequestMsgData) { + log.Trace(fmt.Sprintf("Depo.addRequester: key %v - add peer to req.Id %v", req.Key.Log(), req.Id)) + list := rs.Requesters[req.Id] + rs.Requesters[req.Id] = append(list, req) +} + +/* + store requests are put in netstore so they are stored and then + forwarded to the peers in their kademlia proximity bin by the syncer +*/ +type storeRequestMsgData struct { + SData []byte // the stored chunk Data (incl size) + // optional + Id uint64 // request ID. if delivery, the ID is retrieve request ID + from Peer // [not serialised] protocol registers the requester +} + +func (self storeRequestMsgData) String() string { + var from string + if self.from == nil { + from = "self" + } else { + from = fmt.Sprintf("%x", self.from.OverlayAddr()) + } + end := len(self.SData) + if len(self.SData) > 10 { + end = 10 + } + return fmt.Sprintf("from: %v, ID: %v, SData %x", from, self.Id, self.SData[:end]) +} + +// the entrypoint for store requests coming from the bzz wire protocol +// if key found locally, return. otherwise +// remote is untrusted, so hash is verified and chunk passed on to NetStore +func (self *RequestHandler) handleStoreRequestMsg(msg interface{}, p Peer) error { + req := msg.(*storeRequestMsgData) + req.from = p + chunk, err := storage.NewChunkFromData(req.SData) + if err != nil { + return err + } + chunk.Source = p + self.netStore.Put(chunk) + log.Trace(fmt.Sprintf("delivery of %v from %v", chunk, p)) + return nil +} diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go new file mode 100644 index 0000000000..fd437dfac2 --- /dev/null +++ b/swarm/network/streamer.go @@ -0,0 +1,427 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package network + +import ( + "context" + "fmt" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/p2p/protocols" + bv "github.com/ethereum/go-ethereum/swarm/network/bitvector" + pq "github.com/ethereum/go-ethereum/swarm/network/priorityqueues" +) + +const ( + HashSize = 32 + + Low int = iota + Mid + High + Top + PriorityQueues // number of queues + PriorityQueueCap = 3 // queue capacity +) + +// Stream is string descriptor of the stream +type Stream string + +// Handover represents a statement that the upstream peer hands over the stream section +type Handover struct { + Stream Stream // name of stream + Start, End uint64 // index of hashes + Root common.Hash // Root hash for indexed segment inclusion proofs +} + +// HandoverProof represents a signed statement that the upstream peer handed over the stream section +type HandoverProof struct { + Sig []byte // Sign(Hash(Serialisation(Handover))) + *Handover +} + +// Takeover represents a statement that downstream peer took over (stored all data) +// handed over +type Takeover Handover + +// TakeoverProof represents a signed statement that the downstream peer took over +// the stream section +type TakeoverProof struct { + Sig []byte // Sign(Hash(Serialisation(Takeover))) + *Takeover +} + +// TakeoverProofMsg is the protocol msg sent by downstream peer +type TakeoverProofMsg TakeoverProof + +// String pretty prints TakeoverProofMsg +func (self TakeoverProofMsg) String() string { + return fmt.Sprintf("Stream: '%v' [%v-%v], Root: %x, Sig: %x", self.Stream, self.From, self.To, self.Root, self.Sig) +} + +// SubcribeMsg is the protocol msg for requesting a stream(section) +type SubscribeMsg struct { + Stream Stream + From, To uint64 + Priority uint8 // delivered on priority channel +} + +// UnsyncedKeysMsg is the protocol msg for offering to hand over a +// stream section +type UnsyncedKeysMsg struct { + Stream Stream // name of Stream + From, To uint64 // peer and db-specific entry count + Hashes []byte // stream of hashes (128) + *HandoverProof // HandoverProof +} + +// String pretty prints UnsyncedKeysMsg +func (self UnsyncedKeysMsg) String() string { + return fmt.Sprintf("Stream '%v' [%v-%v] (%v)", self.Stream, self.From, self.To, len(self.Hashes)/HashSize) +} + +// WantedKeysMsg is the protocol msg data for signaling which hashes +// offered in UnsyncedKeysMsg downstream peer actually wants sent over +type WantedKeysMsg struct { + Stream Stream // name of stream + Want []byte // bitvector indicating which keys of the batch needed + From, To uint64 // next interval offset - empty if not to be continued +} + +// String pretty prints WantedKeysMsg +func (self WantedKeysMsg) String() string { + return fmt.Sprintf("Stream '%v', Want: %x, Next: [%v-%v]", self.Stream, self.Want, self.From, self.To) +} + +// Streamer registry for outgoing and incoming streamer constructors +type Streamer struct { + incomingLock sync.RWMutex + outgoingLock sync.RWMutex + outgoing map[Stream]func(*StreamerPeer) (OutgoingStreamer, error) + incoming map[Stream]func(*StreamerPeer) (IncomingStreamer, error) +} + +// NewStreamer is Streamer constructor +func NewStreamer() *Streamer { + return &Streamer{ + outgoing: make(map[Stream]func(*StreamerPeer) (OutgoingStreamer, error)), + incoming: make(map[Stream]func(*StreamerPeer) (IncomingStreamer, error)), + } +} + +// RegisterIncomingStreamer registers an incoming streamer constructor +func (self *Streamer) RegisterIncomingStreamer(stream Stream, f func(*StreamerPeer) (IncomingStreamer, error)) { + self.incomingLock.Lock() + defer self.incomingLock.Unlock() + self.incoming[stream] = f +} + +// RegisterOutgoingStreamer registers an outgoing streamer constructor +func (self *Streamer) RegisterOutgoingStreamer(stream Stream, f func(*StreamerPeer) (OutgoingStreamer, error)) { + self.outgoingLock.Lock() + defer self.outgoingLock.Unlock() + self.outgoing[stream] = f +} + +// GetIncomingStreamer accessor for incoming streamer constructors +func (self *Streamer) GetIncomingStreamer(stream Stream) func(*StreamerPeer) (IncomingStreamer, error) { + self.incomingLock.RLock() + defer self.incomingLock.RUnlock() + f := self.incoming[stream] + if f == nil { + return nil, fmt.Errorf("stream %v not registered", s) + } + return f, nil +} + +// GetOutgoingStreamer accessor for incoming streamer constructors +func (self *Streamer) GetOutgoingStreamer(stream Stream) func(*StreamerPeer) (OutgoingStreamer, error) { + self.outgoingLock.RLock() + defer self.outgoingLock.RUnlock() + f := self.outgoing[stream] + if f == nil { + return nil, fmt.Errorf("stream %v not registered", s) + } +} + +// OutgoingStreamer interface for outgoing peer Streamer +type OutgoingStreamer interface { + CurrentBatch() []byte + SetNextBatch(uint64, uint64) ([]byte, uint64, uint64, *HandoverProof) + GetData([]byte) []byte + Priority() int +} + +// IncomingStreamer interface for incoming peer Streamer +type IncomingStreamer interface { + NextBatch(uint64, uint64) (uint64, uint64) + NeedData([]byte) func() + Priority() int +} + +// StreamerPeer is the Peer extention for the streaming protocol +type StreamerPeer struct { + Peer + streamer *Streamer + pq *pq.PriorityQueues + outgoingLock sync.RWMutex + incomingLock sync.RWMutex + outgoing map[Stream]OutgoingStreamer + incoming map[Stream]IncomingStreamer + quit chan struct{} +} + +// NewStreamerPeer is the constructor for StreamerPeer +func NewStreamerPeer(p Peer, streamer *Streamer) *StreamerPeer { + self := &StreamerPeer{ + pq: pq.New(PriorityQueues, PriorityQueueCap), + streamer: streamer, + outgoing: make(map[Stream]OutgoingStreamer), + incoming: make(map[Stream]IncomingStreamer), + quit: make(chan struct{}), + } + ctx, cancel := context.WithCancel(context.Background()) + go self.pq.Run(ctx, func(i interface{}) { p.Send(i) }) + go func() { + <-self.quit + cancel() + }() + return self +} + +func (self *StreamerPeer) getOutgoingStreamer(s Stream) (OutgoingStreamer, error) { + self.outgoingLock.RLock() + defer self.outgoingLock.RUnlock() + streamer := self.outgoing[s] + if streamer == nil { + return nil, fmt.Errorf("stream '%v' not provided", s) + } + return streamer, nil +} + +func (self *StreamerPeer) getIncomingStreamer(s Stream) (IncomingStreamer, error) { + self.incomingLock.RLock() + defer self.incomingLock.RUnlock() + streamer := self.incoming[s] + if streamer == nil { + return nil, fmt.Errorf("stream '%v' not provided", s) + } + return streamer, nil +} + +func (self *StreamerPeer) setOutgoingStreamer(s Stream, o OutgoingStreamer) error { + self.outgoingLock.Lock() + defer self.outgoingLock.Unlock() + if self.outgoing[s] != nil { + return fmt.Errorf("stream %v already registered", s) + } + self.outgoing[s] = o + return nil +} + +func (self *StreamerPeer) setIncomingStreamer(s Stream, i IncomingStreamer) error { + self.incomingLock.Lock() + defer self.incomingLock.Unlock() + if self.incoming[s] != nil { + return fmt.Errorf("stream %v already registered", s) + } + self.incoming[s] = i + return nil +} + +// Subscribe initiates the streamer +func (self *StreamerPeer) Subscribe(s Stream, from, to uint64) error { + f, err := self.streamer.GetIncomingStreamer(s) + if err != nil { + return err + } + is := f(self) + self.setIncomingStreamer(s, is) + msg := &SubscribeMsg{ + Stream: s, + From: from, + To: to, + Priority: uint8(is.Priority()), + } + self.Send(msg, is.Priority()) +} + +func (self *StreamerPeer) handleSubscribeMsg(msg interface{}) error { + req := msg.(*SubscribeMsg) + f, err := self.streamer.getOutgoingStreamer(req.Stream) + if err != nil { + return err + } + s := f(self) + if err := self.setOutgoingStreamer(req.Stream, s); err != nil { + return nil + } + self.UnsyncedKeys(s, req.From, req.To) + return nil +} + +// handleUnsyncedKeysMsg protocol msg handler calls the incoming streamer interface +// Filter method +func (self *StreamerPeer) handleUnsyncedKeysMsg(msg interface{}) error { + req := msg.(*UnsyncedKeysMsg) + req.C = make(chan struct{}) + s, err := self.getIncomingStreamer(req.Stream) + if err != nil { + return err + } + hashes := req.Hashes + want := bv.New(len(hashes) / HashSize) + wg := sync.WaitGroup{} + for i := 0; i < len(hashes)/HashSize; i += HashSize { + hash := hashes[i : i+HashSize] + if wait := s.NeedData(hash); wait != nil { + want.Set(i, true) + wg.Add(1) + // create request and wait until the chunk data arrives and is stored + go func(w func()) { + w() + wg.Done() + }(wait) + } + } + go func() { + wg.Wait() + msg := s.TakeoverProof(req.Stream, req.From, req.Hashes, req.Root) + self.Send(msg, s.Priority()) + }() + // only send wantedKeysMsg if all missing chunks of the previous batch arrived + // except + from, to = s.NextBatch(from, to) + if from == to { + return nil + } + msg = &WantedKeysMsg{ + Stream: req.Stream, + Want: want, + From: from, + To: to, + } + self.Send(msg, s.Priority()) + return nil +} + +// handleWantedKeysMsg protocol msg handler +// * sends the next batch of unsynced keys +// * sends the actual data chunks as per WantedKeysMsg +func (self *StreamerPeer) handleWantedKeysMsg(msg interface{}) error { + req := msg.(*WantedKeysMsg) + s, err := self.getOutgoingStreamer(req.Stream) + if err != nil { + return err + } + hashes := s.CurrentBatch() + // launch in go routine since GetBatch blocks until new hashes arrive + go self.UnsyncedKeys(s, req.From, req.To) + l := len(hashes) / HashSize + want := bv.NewFromBytes(req.Want, l) + for i := 0; i < l; i++ { + if want.Get(i) { + hash := hashes[i*HashSize : (i+1)*HashSize] + data := s.GetData(hash) + if data == nil { + return errNotFound + } + if err := self.Deliver(data, s.Priority()); err != nil { + return err + } + } + } + return nil +} + +func (self *StreamerPeer) handleTakeoverProofMsg(msg interface{}) error { + req := msg.(*TakeoverProofMsg) + s, err := self.getOutgoingStreamer(req.Stream) + if err != nil { + return err + } + // store the strongest takeoverproof for the stream in streamer + return nil +} + +// Deliver sends a storeRequestMsg protocol message to the peer +func (self *StreamerPeer) Deliver(data []byte, priority int) error { + msg := &storeRequestMsg{ + SData: data, + } + return self.pq.Push(nil, msg, priority) +} + +// Deliver sends a storeRequestMsg protocol message to the peer +func (self *StreamerPeer) Send(msg interface{}, priority int) error { + return self.pq.Push(nil, msg, priority) +} + +// UnsyncedKeys sends UnsyncedKeysMsg protocol msg +func (self *StreamerPeer) SendUnsyncedKeys(s OutgoingStreamer, f, t uint64, priority int) { + hashes, from, to, proof := s.SetNextBatch(f, t) + msg := &UnsyncedKeysMsg{ + HandoverProof: proof, + Hashes: hashes, + From: from, + To: to, + } + self.Send(msg, s.Priority()) +} + +// BzzSpec is the spec of the generic swarm handshake +var StrSpec = &protocols.Spec{ + Name: "stream", + Version: 1, + MaxMsgSize: 10 * 1024 * 1024, + Messages: []interface{}{ + HandshakeMsg{}, + UnsyncedKeysMsg{}, + WantedKeysMsg{}, + TakeoverProofMsg{}, + SubscribeMsg{}, + }, +} + +// Run protocol run function +func (s *Streamer) Run(p *bzzPeer) error { + sp := NewStreamerPeer(p, s) + // load saved intervals + defer close(sp.quit) + return sp.Run(sp.HandleMsg) +} + +// HandleMsg is the message handler that delegates incoming messages +func (self *StreamerPeer) HandleMsg(msg interface{}) error { + switch msg := msg.(type) { + + case *SubscribeMsg: + return self.handleSubscribeMsg(msg) + + case *UnsyncedKeysMsg: + return self.handleUnsyncedKeysMsg(msg) + + case *TakeoverProofMsg: + return self.handleTakeoverProofMsg(msg) + + case *WantedKeysMsg: + return self.handleWantedKeysMsg(msg) + + default: + return fmt.Errorf("unknown message type: %T", msg) + } +} diff --git a/swarm/network/syncer.go b/swarm/network/syncer.go new file mode 100644 index 0000000000..84f61af5d6 --- /dev/null +++ b/swarm/network/syncer.go @@ -0,0 +1,266 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package network + +import ( + "bytes" + "fmt" + "io" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +const ( + batchSize = 128 +) + +// wrapper of db-s to provide mockable custom local chunk store access to syncer +type DbAccess struct { + db *storage.DbStore + loc *storage.LocalStore +} + +func NewDbAccess(loc *storage.LocalStore) *DbAccess { + return &DbAccess{loc.DbStore.(*storage.DbStore), loc} +} + +// to obtain the chunks from key or request db entry only +func (self *DbAccess) get(key storage.Key) (*storage.Chunk, error) { + return self.loc.Get(key) +} + +// current storage counter of chunk db +func (self *DbAccess) currentStorageIndex(po int) uint64 { + return self.db.CurrentStorageIndex(po) +} + +// iteration storage counter and proximity order +func (self *DbAccess) iterator(from uint64, to uint64, po uint8, f func(storage.Key, uint64) bool) error { + return self.db.SyncIterator(from, to, po, f) +} + +// OutgoingSwarmSyncer implements an OutgoingStreamer for history syncing on bins +// offered streams: +// * live request delivery with or without checkback +// * (live/non-live historical) chunk syncing per proximity bin +type OutgoingSwarmSyncer struct { + po int + db *DbAccess + sessionAt uint64 + currentBatch []byte + priority int +} + +// NewOutgoingSwarmSyncer is contructor for OutgoingSwarmSyncer +func NewOutgoingSwarmSyncer(po int, db *DbAccess) *OutgoingSwarmSyncer { + self := &OutgoingSwarmSyncer{ + po: po, + db: db, + sessionAt: db.currentStorageIndex(po), + } + return self +} + +func RegisterOutgoingSyncers(streamer *Streamer, db *DbAccess) { + for po := 0; po < maxPO; po++ { + stream := fmt.Sprintf("SYNC-%02d-live", po) + streamer.RegisterOutgoingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) { + return NewOutgoingSwarmSyncer(po, db) + }) + stream = fmt.Sprintf("SYNC-%02d-history", po) + streamer.RegisterOutgoingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) { + return NewOutgoingSwarmSyncer(po, db) + }) + stream = fmt.Sprintf("SYNC-%02d-delete", po) + streamer.RegisterOutgoingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) { + return NewOutgoingProvableSwarmSyncer(po, db) + }) + } +} + +// GetSection retrieves the actual chunk from localstore +func (self *OutgoingSwarmSyncer) GetData(key []byte) []byte { + chunk, err := self.db.get(Key(key)) + if err != nil { + return nil + } + return chunk.SData +} + +func (self *OutgoingSwarmSyncer) CurrentBatch() []byte { + return self.currentBatch +} + +func (self *OutgoingSwarmSyncer) Priority() int { + return self.priority +} + +// GetBatch retrieves the next batch of hashes from the dbstore +func (self *OutgoingSwarmSyncer) SetNextBatch(from, to uint64) []byte { + var batch []byte + i := 0 + err := self.db.iterator(from, to, self.po, func(key storage.Key, idx uint64) bool { + batch = append(batch, key[:]) + i++ + to = idx + return i < batchSize + }) + self.currentBatch = batch + log.Debug("Swarm batch", "po", self.po, "len", i, "from", from, "to", to) + return batch, from, to, proof +} + +// IncomingSwarmSyncer +type IncomingSwarmSyncer struct { + po int + priority int + sessionAt uint64 + nextC chan struct{} + intervals []uint64 + sessionRoot storage.Key + sessionReader storage.LazySectionReader + retrieveC chan storage.Chunk + storeC chan storage.Chunk +} + +// NewIncomingSwarmSyncer is a contructor for provable data exchange syncer +func NewIncomingSwarmSyncer(po int, priority int, sessionAt uint64, intervals []uint64, p Peer) *IncomingSwarmSyncer { + self := &IncomingSwarmSyncer{ + po: po, + priority: priority, + sessionAt: sessionAt, + nextC: make(chan struct{}, 1), + intervals: intervals, + } + return self +} + +// NewIncomingProvableSwarmSyncer is a contructor for provable data exchange syncer +func NewIncomingProvableSwarmSyncer(po int, priority int, index uint64, sessionAt uint64, intervals []uint64, sessionRoot storage.Key, chunker *storage.PyramidChunker, store storage.ChunkStore, p Peer) *IncomingSwarmSyncer { + retrieveC := make(storage.Chunk, chunksCap) + RunChunkRequestor(p, retrieveC) + storeC := make(storage.Chunk, chunksCap) + RunChunkStorer(store, storeC) + self := &IncomingSwarmSyncer{ + po: po, + priority: priority, + sessionAt: sessionAt, + start: index, + end: index, + nextC: make(chan struct{}, 1), + intervals: intervals, + sessionRoot: sessionRoot, + sessionReader: chunker.Join(sessionRoot, retrieveC), + retrieveC: retrieveC, + storeC: storeC, + } + return self +} + +func RegisterIncomingSyncers(streamer *Streamer, db *DbAccess) { + for po := 0; po < maxPO; po++ { + stream := fmt.Sprintf("SYNC-%02d-live", po) + streamer.RegisterIncomingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) { + return NewIncomingSwarmSyncer(po, Mid, sessionAt, nil, p) + }) + stream = fmt.Sprintf("SYNC-%02d-history", po) + streamer.RegisterIncomingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) { + intervals := loadIntervals(p, po, false) + return NewIncomingSwarmSyncer(po, Mid, sessionAt, intervals, p) + }) + stream = fmt.Sprintf("SYNC-%02d-delete", po) + streamer.RegisterIncomingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) { + intervals := loadIntervals(p, po, true) + return NewIncomingSwarmSyncer(po, Mid, sessionAt, intervals, p) + }) + } +} + +// NeedData +func (self *IncomingSwarmSyncer) NeedData(key []byte) func() { + chunk, err := self.store.Get(hash) + if err == nil { + if chunk.SData == nil { + // send a request instead + return nil + } + } + // create request and wait until the chunk data arrives and is stored + return func() { + storedC := <-chunk.storedC + <-storedC + } +} + +// NextBatch adjusts the indexes by inspecting the intervals +func (self *IncomingSwarmSyncer) NextBatch(from, to uint64) (uint64, uint64) { + if from >= self.sessionAt { // live syncing + self.intervals[1] = to + } else if to >= self.sessionAt { // history sync complete + self.intervals = nil + from = 0 + } else if len(intervals) > 2 && to >= self.intervals[2] { // filled a gap in the intervals + self.intervals[1:] = self.intervals[3:] + from = self.intervals[1] + if len(intervals) > 2 { + to = self.intervals[2] + } + } else { + self.intervals[1] = to + } + return from, to +} + +// +func (self *IncomingSwarmSyncer) TakeoverProof(s Stream, from uint64, hashes []byte, root storage.Key) error { + // for provable syncer currentRoot is non-zero length + if self.chunker != nil { + if from > self.sessionAt { // for live syncing currentRoot is always updated + expRoot, err := self.chunker.Append(self.currentRoot, bytes.NewReader(hashes), self.retrieveC, self.storeC) + if err != nil { + return err + } + if !bytes.Equal(root, expRoot) { + return fmt.Errorf("HandoverProof mismatch") + } + self.currentRoot = currentRoot + } else { + expHashes := make([]byte, len(hashes)) + n, err := self.sessionReader.ReadAt(expHashes, int64(self.end*HashSize)) + if err != nil && err != io.EOF { + return err + } + if !bytes.Equal(expHashes, hashes) { + return errInvalidProof + } + } + return nil + } + self.end += len(hashes) / HashSize + takeover := &Takeover{ + Stream: s, + Start: self.start, + End: self.end, + Root: root, + } + // serialise and sign + return &TakeoverProof{ + Takeover: takeover, + Sig: nil, + } +} From ca2882fc816f0b0bff48481a3e7077edba88794d Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Tue, 2 Jan 2018 16:37:04 +0100 Subject: [PATCH 002/210] swarm/network/bitvector: add tests, fix Get method and add Length --- swarm/network/bitvector/bitvector.go | 24 +++++-- swarm/network/bitvector/bitvector_test.go | 88 +++++++++++++++++++++++ 2 files changed, 107 insertions(+), 5 deletions(-) create mode 100644 swarm/network/bitvector/bitvector_test.go diff --git a/swarm/network/bitvector/bitvector.go b/swarm/network/bitvector/bitvector.go index 1769fa4f1d..93e55a9d09 100644 --- a/swarm/network/bitvector/bitvector.go +++ b/swarm/network/bitvector/bitvector.go @@ -1,34 +1,48 @@ package bitvector +import "errors" + +var errInvalidLength = errors.New("invalid length") + type BitVector struct { len int b []byte } -func New(l int) *BitVector { +func New(l int) (bv *BitVector, err error) { return NewFromBytes(make([]byte, l/8+1), l) } -func NewFromBytes(b []byte, l int) *BitVector { +func NewFromBytes(b []byte, l int) (bv *BitVector, err error) { + if l <= 0 { + return nil, errInvalidLength + } + if len(b)*8 < l { + return nil, errInvalidLength + } return &BitVector{ len: l, b: b, - } + }, nil } func (bv *BitVector) Get(i int) bool { bi := i / 8 - return uint8(bv.b[bi])&0x1>>uint(i%8) != 0 + return uint8(bv.b[bi])&(0x1<> uint8(i%8) + bv.b[bi] ^= 0x1 << uint8(i%8) } } func (bv *BitVector) Bytes() []byte { return bv.b } + +func (bv *BitVector) Length() int { + return bv.len +} diff --git a/swarm/network/bitvector/bitvector_test.go b/swarm/network/bitvector/bitvector_test.go new file mode 100644 index 0000000000..ae759404d1 --- /dev/null +++ b/swarm/network/bitvector/bitvector_test.go @@ -0,0 +1,88 @@ +package bitvector + +import "testing" + +func TestBitvectorNew(t *testing.T) { + _, err := New(0) + if err != errInvalidLength { + t.Errorf("expected err %v, got %v", errInvalidLength, err) + } + + _, err = NewFromBytes(nil, 0) + if err != errInvalidLength { + t.Errorf("expected err %v, got %v", errInvalidLength, err) + } + + _, err = NewFromBytes([]byte{0}, 9) + if err != errInvalidLength { + t.Errorf("expected err %v, got %v", errInvalidLength, err) + } + + _, err = NewFromBytes(make([]byte, 8), 8) + if err != nil { + t.Error(err) + } +} + +func TestBitvectorGetSet(t *testing.T) { + for _, length := range []int{ + 1, + 2, + 4, + 8, + 9, + 15, + 16, + } { + bv, err := New(length) + if err != nil { + t.Errorf("error for length %v: %v", length, err) + } + + for i := 0; i < length; i++ { + if bv.Get(i) { + t.Errorf("expected false for element on index %v", i) + } + } + + func() { + defer func() { + if err := recover(); err == nil { + t.Errorf("expecting panic") + } + }() + bv.Get(length + 8) + }() + + for i := 0; i < length; i++ { + bv.Set(i, true) + for j := 0; j < length; j++ { + if j == i { + if bv.Get(j) != true { + t.Errorf("element on index %v is not set to true", i) + } + } else { + if bv.Get(j) != false { + t.Errorf("element on index %v is not false", i) + } + } + } + + bv.Set(i, false) + + if bv.Get(i) != false { + t.Errorf("element on index %v is not set to false", i) + } + } + } +} + +func TestBitvectorNewFromBytesGet(t *testing.T) { + bv, err := NewFromBytes([]byte{8}, 8) + if err != nil { + t.Error(err) + } + if bv.Get(3) != true { + t.Fatalf("element 3 is not set to true: state %08b", bv.b[0]) + } +} From 974536d1cff3bb6429a2721eca624382b5501ea5 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Tue, 2 Jan 2018 17:09:20 +0100 Subject: [PATCH 003/210] Rename priorityqueues to priorityqueue --- .../priorityqueues.go => priorityqueue/priorityqueue.go} | 0 swarm/network/streamer.go | 8 ++++---- 2 files changed, 4 insertions(+), 4 deletions(-) rename swarm/network/{priorityqueues/priorityqueues.go => priorityqueue/priorityqueue.go} (100%) diff --git a/swarm/network/priorityqueues/priorityqueues.go b/swarm/network/priorityqueue/priorityqueue.go similarity index 100% rename from swarm/network/priorityqueues/priorityqueues.go rename to swarm/network/priorityqueue/priorityqueue.go diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go index fd437dfac2..7941a261dd 100644 --- a/swarm/network/streamer.go +++ b/swarm/network/streamer.go @@ -24,7 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/p2p/protocols" bv "github.com/ethereum/go-ethereum/swarm/network/bitvector" - pq "github.com/ethereum/go-ethereum/swarm/network/priorityqueues" + pq "github.com/ethereum/go-ethereum/swarm/network/priorityqueue" ) const ( @@ -34,7 +34,7 @@ const ( Mid High Top - PriorityQueues // number of queues + PriorityQueue // number of queues PriorityQueueCap = 3 // queue capacity ) @@ -177,7 +177,7 @@ type IncomingStreamer interface { type StreamerPeer struct { Peer streamer *Streamer - pq *pq.PriorityQueues + pq *pq.PriorityQueue outgoingLock sync.RWMutex incomingLock sync.RWMutex outgoing map[Stream]OutgoingStreamer @@ -188,7 +188,7 @@ type StreamerPeer struct { // NewStreamerPeer is the constructor for StreamerPeer func NewStreamerPeer(p Peer, streamer *Streamer) *StreamerPeer { self := &StreamerPeer{ - pq: pq.New(PriorityQueues, PriorityQueueCap), + pq: pq.New(PriorityQueue, PriorityQueueCap), streamer: streamer, outgoing: make(map[Stream]OutgoingStreamer), incoming: make(map[Stream]IncomingStreamer), From b5fb850d1bf3a8aa541cdd5dab400f688d5c9c87 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Tue, 2 Jan 2018 17:12:15 +0100 Subject: [PATCH 004/210] Rename priorityqueues to priorityqueue again --- swarm/network/priorityqueue/priorityqueue.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/swarm/network/priorityqueue/priorityqueue.go b/swarm/network/priorityqueue/priorityqueue.go index d64124d6c2..2c6e3e52b0 100644 --- a/swarm/network/priorityqueue/priorityqueue.go +++ b/swarm/network/priorityqueue/priorityqueue.go @@ -1,4 +1,4 @@ -// package priority_queues implement a channel based priority queue +// package priority_queue implement a channel based priority queue // over arbitrary types. It provides an // an autopop loop applying a function to the items always respecting // their priority. The structure is only quasi consistent ie., if a lower @@ -7,7 +7,7 @@ // that there was any point where the lower priority item was present // but the higher was not -package priorityqueues +package priorityqueue import ( "context" @@ -21,26 +21,26 @@ var ( wakey = struct{}{} ) -// PriorityQueues is the basic structure -type PriorityQueues struct { +// PriorityQueue is the basic structure +type PriorityQueue struct { queues []chan interface{} wakeup chan struct{} } -// New is the constructor for PriorityQueues -func New(n int, l int) *PriorityQueues { +// New is the constructor for PriorityQueue +func New(n int, l int) *PriorityQueue { var queues = make([]chan interface{}, n) for i := range queues { queues[i] = make(chan interface{}, l) } - return &PriorityQueues{ + return &PriorityQueue{ queues: queues, wakeup: make(chan struct{}, 1), } } // Run is a forever loop popping items from the queues -func (pq *PriorityQueues) Run(ctx context.Context, f func(interface{})) { +func (pq *PriorityQueue) Run(ctx context.Context, f func(interface{})) { top := len(pq.queues) - 1 p := top q := pq.queues[p] @@ -70,7 +70,7 @@ READ: // Push pushes an item to the appropriate queue specified in the priority argument // if context is given it waits until either the item is pushed or the Context aborts // otherwise returns errContention if the queue is full -func (pq *PriorityQueues) Push(ctx context.Context, x interface{}, p int) error { +func (pq *PriorityQueue) Push(ctx context.Context, x interface{}, p int) error { if p < 0 || p >= len(pq.queues) { return errBadPriority } From aa9a9ff818b18a326b56e7aa4d42fdfe3e8f3cc9 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Tue, 2 Jan 2018 18:11:19 +0100 Subject: [PATCH 005/210] swarm/network/priorotyqueue: add tests and fix Run --- swarm/network/priorityqueue/priorityqueue.go | 2 +- .../priorityqueue/priorityqueue_test.go | 82 +++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 swarm/network/priorityqueue/priorityqueue_test.go diff --git a/swarm/network/priorityqueue/priorityqueue.go b/swarm/network/priorityqueue/priorityqueue.go index 2c6e3e52b0..d2fd97fe10 100644 --- a/swarm/network/priorityqueue/priorityqueue.go +++ b/swarm/network/priorityqueue/priorityqueue.go @@ -43,9 +43,9 @@ func New(n int, l int) *PriorityQueue { func (pq *PriorityQueue) Run(ctx context.Context, f func(interface{})) { top := len(pq.queues) - 1 p := top - q := pq.queues[p] READ: for { + q := pq.queues[p] select { case <-ctx.Done(): return diff --git a/swarm/network/priorityqueue/priorityqueue_test.go b/swarm/network/priorityqueue/priorityqueue_test.go new file mode 100644 index 0000000000..ffb3bbc7db --- /dev/null +++ b/swarm/network/priorityqueue/priorityqueue_test.go @@ -0,0 +1,82 @@ +package priorityqueue + +import ( + "context" + "sync" + "testing" +) + +func Test(t *testing.T) { + var results []string + wg := sync.WaitGroup{} + pq := New(3, 2) + wg.Add(1) + go pq.Run(context.Background(), func(v interface{}) { + results = append(results, v.(string)) + wg.Done() + }) + pq.Push(context.Background(), "2.0", 2) + wg.Wait() + if results[0] != "2.0" { + t.Errorf("expected first result %q, got %q", "2.0", results[0]) + } + +Loop: + for i, tc := range []struct { + priorities []int + values []string + results []string + errors []error + }{ + { + priorities: []int{0}, + values: []string{""}, + results: []string{""}, + }, + { + priorities: []int{0, 1}, + values: []string{"0.0", "1.0"}, + results: []string{"1.0", "0.0"}, + }, + { + priorities: []int{1, 0}, + values: []string{"1.0", "0.0"}, + results: []string{"1.0", "0.0"}, + }, + { + priorities: []int{0, 1, 1}, + values: []string{"0.0", "1.0", "1.1"}, + results: []string{"1.0", "1.1", "0.0"}, + }, + { + priorities: []int{0, 0, 0}, + values: []string{"0.0", "0.0", "0.1"}, + errors: []error{nil, nil, errContention}, + }, + } { + var results []string + wg := sync.WaitGroup{} + pq := New(3, 2) + wg.Add(len(tc.values)) + for j, value := range tc.values { + err := pq.Push(nil, value, tc.priorities[j]) + if tc.errors != nil && err != tc.errors[j] { + t.Errorf("expected push error %v, got %v", tc.errors[j], err) + continue Loop + } + if err != nil { + continue Loop + } + } + go pq.Run(context.Background(), func(v interface{}) { + results = append(results, v.(string)) + wg.Done() + }) + wg.Wait() + for k, result := range tc.results { + if results[k] != result { + t.Errorf("test case %v: expected %v element %q, got %q", i, k, result, results[k]) + } + } + } +} From 9129b6bf44f21b2acbf8c6b7c3f2e1d33c7d7257 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Tue, 2 Jan 2018 18:53:41 +0100 Subject: [PATCH 006/210] swarm/storage: apply changes from swarm-network-rewrite-syncer branch Only changes from swarm/storage from swarm-network-rewrite-syncer branch are merged. Changes to other packages are not merged. --- swarm/storage/common_test.go | 192 +++++++++++----- swarm/storage/dbstore.go | 396 ++++++++++++++++++--------------- swarm/storage/dbstore_test.go | 262 +++++++++++----------- swarm/storage/dpa.go | 16 +- swarm/storage/dpa_test.go | 27 ++- swarm/storage/localstore.go | 4 +- swarm/storage/memstore_test.go | 77 +++++-- swarm/storage/types.go | 75 ++++++- 8 files changed, 643 insertions(+), 406 deletions(-) diff --git a/swarm/storage/common_test.go b/swarm/storage/common_test.go index cd4c2ef139..6fd66a03d9 100644 --- a/swarm/storage/common_test.go +++ b/swarm/storage/common_test.go @@ -19,12 +19,15 @@ package storage import ( "bytes" "crypto/rand" + "encoding/binary" "fmt" + "hash" "io" "sync" "testing" + "time" - "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/crypto/sha3" ) type brokenLimitedReader struct { @@ -42,16 +45,101 @@ func brokenLimitReader(data io.Reader, size int, errAt int) *brokenLimitedReader } } +func mputChunks(store ChunkStore, processors int, n int, chunksize int, hash hash.Hash) (hs []Key) { + f := func(int) *Chunk { + data := make([]byte, chunksize) + rand.Reader.Read(data) + hash.Reset() + hash.Write(data) + h := hash.Sum(nil) + chunk := NewChunk(Key(h), nil) + chunk.SData = data + return chunk + } + return mput(store, processors, n, f) +} + +func mputRandomKey(store ChunkStore, processors int, n int, chunksize int) (hs []Key) { + data := make([]byte, chunksize+8) + binary.LittleEndian.PutUint64(data[0:8], uint64(chunksize)) + + f := func(int) *Chunk { + h := make([]byte, 32) + rand.Reader.Read(h) + chunk := NewChunk(Key(h), nil) + chunk.SData = data + return chunk + } + return mput(store, processors, n, f) +} + +func mput(store ChunkStore, processors int, n int, f func(i int) *Chunk) (hs []Key) { + wg := sync.WaitGroup{} + wg.Add(processors) + c := make(chan *Chunk) + for i := 0; i < processors; i++ { + go func() { + defer wg.Done() + for chunk := range c { + store.Put(chunk) + } + }() + } + for i := 0; i < n; i++ { + chunk := f(i) + hs = append(hs, chunk.Key) + c <- chunk + } + close(c) + wg.Wait() + return hs +} + +func mget(store ChunkStore, hs []Key, f func(h Key, chunk *Chunk) error) error { + wg := sync.WaitGroup{} + wg.Add(len(hs)) + errc := make(chan error) + + for _, k := range hs { + go func(h Key) { + defer wg.Done() + chunk, err := store.Get(h) + if err != nil { + errc <- err + return + } + if f != nil { + err = f(h, chunk) + if err != nil { + errc <- err + return + } + } + }(k) + } + go func() { + wg.Wait() + close(errc) + }() + var err error + select { + case err = <-errc: + case <-time.NewTimer(5 * time.Second).C: + err = fmt.Errorf("timed out after 5 seconds") + } + return err +} + func testDataReader(l int) (r io.Reader) { return io.LimitReader(rand.Reader, int64(l)) } -func (self *brokenLimitedReader) Read(buf []byte) (int, error) { - if self.off+len(buf) > self.errAt { +func (r *brokenLimitedReader) Read(buf []byte) (int, error) { + if r.off+len(buf) > r.errAt { return 0, fmt.Errorf("Broken reader") } - self.off += len(buf) - return self.lr.Read(buf) + r.off += len(buf) + return r.lr.Read(buf) } func testDataReaderAndSlice(l int) (r io.Reader, slice []byte) { @@ -63,54 +151,50 @@ func testDataReaderAndSlice(l int) (r io.Reader, slice []byte) { return } -func testStore(m ChunkStore, l int64, branches int64, t *testing.T) { - - chunkC := make(chan *Chunk) - go func() { - for chunk := range chunkC { - m.Put(chunk) - if chunk.wg != nil { - chunk.wg.Done() - } - } - }() - chunker := NewTreeChunker(&ChunkerParams{ - Branches: branches, - Hash: SHA3Hash, - }) - swg := &sync.WaitGroup{} - key, _ := chunker.Split(rand.Reader, l, chunkC, swg, nil) - swg.Wait() - close(chunkC) - chunkC = make(chan *Chunk) - - quit := make(chan bool) - - go func() { - for ch := range chunkC { - go func(chunk *Chunk) { - storedChunk, err := m.Get(chunk.Key) - if err == notFound { - log.Trace(fmt.Sprintf("chunk '%v' not found", chunk.Key.Log())) - } else if err != nil { - log.Trace(fmt.Sprintf("error retrieving chunk %v: %v", chunk.Key.Log(), err)) - } else { - chunk.SData = storedChunk.SData - chunk.Size = storedChunk.Size - } - log.Trace(fmt.Sprintf("chunk '%v' not found", chunk.Key.Log())) - close(chunk.C) - }(ch) - } - close(quit) - }() - r := chunker.Join(key, chunkC) - - b := make([]byte, l) - n, err := r.ReadAt(b, 0) - if err != io.EOF { - t.Fatalf("read error (%v/%v) %v", n, l, err) +func testStoreRandom(m ChunkStore, processors int, n int, chunksize int, t *testing.T) { + hs := mputRandomKey(m, processors, n, chunksize) + err := mget(m, hs, nil) + if err != nil { + t.Fatalf("testStore failed: %v", err) + } +} + +func testStoreCorrect(m ChunkStore, processors int, n int, chunksize int, t *testing.T) { + hs := mputChunks(m, processors, n, chunksize, sha3.NewKeccak256()) + f := func(h Key, chunk *Chunk) error { + if !bytes.Equal(h, chunk.Key) { + return fmt.Errorf("key does not match retrieved chunk Key") + } + hasher := sha3.NewKeccak256() + hasher.Write(chunk.SData) + exp := hasher.Sum(nil) + if !bytes.Equal(h, exp) { + return fmt.Errorf("key is not hash of chunk data") + } + return nil + } + err := mget(m, hs, f) + if err != nil { + t.Fatalf("testStore failed: %v", err) + } +} + +func benchmarkStorePut(store ChunkStore, processors int, n int, chunksize int, b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + mputRandomKey(store, processors, n, chunksize) + } +} + +func benchmarkStoreGet(store ChunkStore, processors int, n int, chunksize int, b *testing.B) { + hs := mputRandomKey(store, processors, n, chunksize) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := mget(store, hs, nil) + if err != nil { + b.Fatalf("mget failed: %v", err) + } } - close(chunkC) - <-quit } diff --git a/swarm/storage/dbstore.go b/swarm/storage/dbstore.go index 454319f229..c1bef29823 100644 --- a/swarm/storage/dbstore.go +++ b/swarm/storage/dbstore.go @@ -23,19 +23,15 @@ package storage import ( - "archive/tar" "bytes" "encoding/binary" - "encoding/hex" "fmt" - "io" - "io/ioutil" "sync" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/iterator" + "github.com/syndtr/goleveldb/leveldb/opt" ) const ( @@ -51,10 +47,13 @@ const ( ) var ( - keyAccessCnt = []byte{2} - keyEntryCnt = []byte{3} - keyDataIdx = []byte{4} - keyGCPos = []byte{5} + keyOldData = byte(1) + keyAccessCnt = []byte{2} + keyEntryCnt = []byte{3} + keyDataIdx = []byte{4} + keyGCPos = []byte{5} + keyData = byte(6) + keyDistanceCnt = byte(7) ) type gcItem struct { @@ -68,25 +67,29 @@ type DbStore struct { // this should be stored in db, accessed transactionally entryCnt, accessCnt, dataIdx, capacity uint64 + bucketCnt []uint64 gcPos, gcStartPos []byte gcArray []*gcItem hashfunc SwarmHasher - - lock sync.Mutex + po func(Key) uint8 + lock sync.Mutex + trusted bool // if hash integity check is to be performed (for testing only) } -func NewDbStore(path string, hash SwarmHasher, capacity uint64, radius int) (s *DbStore, err error) { +// TODO: Instead of passing the distance function, just pass the address from which distances are calculated +// to avoid the appearance of a pluggable distance metric and opportunities of bugs associated with providing +// a function diferent from the one that is actually used. +func NewDbStore(path string, hash SwarmHasher, capacity uint64, po func(Key) uint8) (s *DbStore, err error) { s = new(DbStore) - s.hashfunc = hash - s.db, err = NewLDBDatabase(path) if err != nil { - return + return nil, err } + s.po = po s.setCapacity(capacity) s.gcStartPos = make([]byte, 1) @@ -95,15 +98,31 @@ func NewDbStore(path string, hash SwarmHasher, capacity uint64, radius int) (s * data, _ := s.db.Get(keyEntryCnt) s.entryCnt = BytesToU64(data) + s.bucketCnt = make([]uint64, 0x100) + for i := 0; i < 0x100; i++ { + k := make([]byte, 2) + k[0] = keyDistanceCnt + k[1] = byte(uint8(i)) + cnt, _ := s.db.Get(k) + s.bucketCnt[i] = BytesToU64(cnt) + } data, _ = s.db.Get(keyAccessCnt) - s.accessCnt = BytesToU64(data) + //s.accessCnt = BytesToU64(data) + if len(data) == 8 { + s.accessCnt = binary.LittleEndian.Uint64(data) + s.accessCnt++ + } data, _ = s.db.Get(keyDataIdx) - s.dataIdx = BytesToU64(data) + if len(data) == 8 { + s.dataIdx = BytesToU64(data) + s.dataIdx++ + } + s.gcPos, _ = s.db.Get(keyGCPos) if s.gcPos == nil { s.gcPos = s.gcStartPos } - return + return s, nil } type dpaDBIndex struct { @@ -115,12 +134,14 @@ func BytesToU64(data []byte) uint64 { if len(data) < 8 { return 0 } - return binary.LittleEndian.Uint64(data) + //return binary.LittleEndian.Uint64(data) + return binary.BigEndian.Uint64(data) } func U64ToBytes(val uint64) []byte { data := make([]byte, 8) - binary.LittleEndian.PutUint64(data, val) + //binary.LittleEndian.PutUint64(data, val) + binary.BigEndian.PutUint64(data, val) return data } @@ -133,38 +154,52 @@ func (s *DbStore) updateIndexAccess(index *dpaDBIndex) { } func getIndexKey(hash Key) []byte { - HashSize := len(hash) - key := make([]byte, HashSize+1) + hashSize := len(hash) + key := make([]byte, hashSize+1) key[0] = 0 copy(key[1:], hash[:]) return key } -func getDataKey(idx uint64) []byte { +func getOldDataKey(idx uint64) []byte { key := make([]byte, 9) - key[0] = 1 + key[0] = keyOldData binary.BigEndian.PutUint64(key[1:9], idx) return key } +func getDataKey(idx uint64, po uint8) []byte { + key := make([]byte, 10) + key[0] = keyData + key[1] = byte(po) + binary.BigEndian.PutUint64(key[2:], idx) + + return key +} + func encodeIndex(index *dpaDBIndex) []byte { data, _ := rlp.EncodeToBytes(index) return data } func encodeData(chunk *Chunk) []byte { - return chunk.SData + return append(chunk.Key[:], chunk.SData...) } -func decodeIndex(data []byte, index *dpaDBIndex) { +func decodeIndex(data []byte, index *dpaDBIndex) error { dec := rlp.NewStream(bytes.NewReader(data), 0) - dec.Decode(index) + return dec.Decode(index) } func decodeData(data []byte, chunk *Chunk) { + chunk.SData = data[32:] + chunk.Size = int64(binary.BigEndian.Uint64(data[32:40])) +} + +func decodeOldData(data []byte, chunk *Chunk) { chunk.SData = data - chunk.Size = int64(binary.LittleEndian.Uint64(data[0:8])) + chunk.Size = int64(binary.BigEndian.Uint64(data[0:8])) } func gcListPartition(list []*gcItem, left int, right int, pivotIndex int) int { @@ -250,98 +285,16 @@ func (s *DbStore) collectGarbage(ratio float32) { cutidx := gcListSelect(s.gcArray, 0, gcnt-1, int(float32(gcnt)*ratio)) cutval := s.gcArray[cutidx].value - // fmt.Print(gcnt, " ", s.entryCnt, " ") - // actual gc for i := 0; i < gcnt; i++ { if s.gcArray[i].value <= cutval { - s.delete(s.gcArray[i].idx, s.gcArray[i].idxKey) + s.delete(s.gcArray[i].idx, s.gcArray[i].idxKey, s.po(Key(s.gcPos[1:]))) } } - // fmt.Println(s.entryCnt) - s.db.Put(keyGCPos, s.gcPos) } -// Export writes all chunks from the store to a tar archive, returning the -// number of chunks written. -func (s *DbStore) Export(out io.Writer) (int64, error) { - tw := tar.NewWriter(out) - defer tw.Close() - - it := s.db.NewIterator() - defer it.Release() - var count int64 - for ok := it.Seek([]byte{kpIndex}); ok; ok = it.Next() { - key := it.Key() - if (key == nil) || (key[0] != kpIndex) { - break - } - - var index dpaDBIndex - decodeIndex(it.Value(), &index) - - data, err := s.db.Get(getDataKey(index.Idx)) - if err != nil { - log.Warn(fmt.Sprintf("Chunk %x found but could not be accessed: %v", key[:], err)) - continue - } - - hdr := &tar.Header{ - Name: hex.EncodeToString(key[1:]), - Mode: 0644, - Size: int64(len(data)), - } - if err := tw.WriteHeader(hdr); err != nil { - return count, err - } - if _, err := tw.Write(data); err != nil { - return count, err - } - count++ - } - - return count, nil -} - -// Import reads chunks into the store from a tar archive, returning the number -// of chunks read. -func (s *DbStore) Import(in io.Reader) (int64, error) { - tr := tar.NewReader(in) - - var count int64 - for { - hdr, err := tr.Next() - if err == io.EOF { - break - } else if err != nil { - return count, err - } - - if len(hdr.Name) != 64 { - log.Warn("ignoring non-chunk file", "name", hdr.Name) - continue - } - - key, err := hex.DecodeString(hdr.Name) - if err != nil { - log.Warn("ignoring invalid chunk file", "name", hdr.Name, "err", err) - continue - } - - data, err := ioutil.ReadAll(tr) - if err != nil { - return count, err - } - - s.Put(&Chunk{Key: key, SData: data}) - count++ - } - - return count, nil -} - func (s *DbStore) Cleanup() { //Iterates over the database and checks that there are no faulty chunks it := s.db.NewIterator() @@ -356,21 +309,23 @@ func (s *DbStore) Cleanup() { } total++ var index dpaDBIndex - decodeIndex(it.Value(), &index) - - data, err := s.db.Get(getDataKey(index.Idx)) + err := decodeIndex(it.Value(), &index) + if err != nil { + it.Next() + continue + } + data, err := s.db.Get(getDataKey(index.Idx, s.po(Key(key[1:])))) if err != nil { log.Warn(fmt.Sprintf("Chunk %x found but could not be accessed: %v", key[:], err)) - s.delete(index.Idx, getIndexKey(key[1:])) + s.delete(index.Idx, getIndexKey(key[1:]), s.po(Key(key[1:]))) errorsFound++ } else { hasher := s.hashfunc() - hasher.Write(data) + hasher.Write(data[32:]) hash := hasher.Sum(nil) if !bytes.Equal(hash, key[1:]) { log.Warn(fmt.Sprintf("Found invalid chunk. Hash mismatch. hash=%x, key=%x", hash, key[:])) - s.delete(index.Idx, getIndexKey(key[1:])) - errorsFound++ + s.delete(index.Idx, getIndexKey(key[1:]), s.po(Key(key[1:]))) } } it.Next() @@ -379,16 +334,90 @@ func (s *DbStore) Cleanup() { log.Warn(fmt.Sprintf("Found %v errors out of %v entries", errorsFound, total)) } -func (s *DbStore) delete(idx uint64, idxKey []byte) { +func (s *DbStore) Dump() { + //Iterates over the database and checks that there are no faulty chunks + it := s.db.NewIterator() + startPosition := []byte{kpIndex} + it.Seek(startPosition) + var key []byte + var total int + for it.Valid() { + key = it.Key() + if (key == nil) || (key[0] != kpIndex) { + break + } + total++ + fmt.Printf("%x\n", key[1:]) + it.Next() + } + it.Release() + log.Warn(fmt.Sprintf("logged %v chunks", total)) +} + +func (s *DbStore) ReIndex() { + //Iterates over the database and checks that there are no faulty chunks + it := s.db.NewIterator() + startPosition := []byte{keyOldData} + it.Seek(startPosition) + var key []byte + var errorsFound, total int + for it.Valid() { + key = it.Key() + if (key == nil) || (key[0] != keyOldData) { + break + } + data := it.Value() + hasher := s.hashfunc() + hasher.Write(data) + hash := hasher.Sum(nil) + + newKey := make([]byte, 10) + oldCntKey := make([]byte, 2) + newCntKey := make([]byte, 2) + oldCntKey[0] = keyDistanceCnt + newCntKey[0] = keyDistanceCnt + key[0] = keyData + key[1] = byte(s.po(Key(key[1:]))) + oldCntKey[1] = key[1] + newCntKey[1] = byte(s.po(Key(newKey[1:]))) + copy(newKey[2:], key[1:]) + newValue := append(hash, data...) + + batch := new(leveldb.Batch) + batch.Delete(key) + s.bucketCnt[oldCntKey[1]]-- + batch.Put(oldCntKey, U64ToBytes(s.bucketCnt[oldCntKey[1]])) + batch.Put(newKey, newValue) + s.bucketCnt[newCntKey[1]]++ + batch.Put(newCntKey, U64ToBytes(s.bucketCnt[newCntKey[1]])) + s.db.Write(batch) + it.Next() + } + it.Release() + log.Warn(fmt.Sprintf("Found %v errors out of %v entries", errorsFound, total)) +} + +func (s *DbStore) delete(idx uint64, idxKey []byte, po uint8) { batch := new(leveldb.Batch) batch.Delete(idxKey) - batch.Delete(getDataKey(idx)) + batch.Delete(getDataKey(idx, po)) s.entryCnt-- + s.bucketCnt[po]-- + cntKey := make([]byte, 2) + cntKey[0] = keyDistanceCnt + cntKey[1] = po batch.Put(keyEntryCnt, U64ToBytes(s.entryCnt)) + batch.Put(cntKey, U64ToBytes(s.bucketCnt[po])) s.db.Write(batch) } -func (s *DbStore) Counter() uint64 { +func (s *DbStore) Size() uint64 { + s.lock.Lock() + defer s.lock.Unlock() + return s.entryCnt +} + +func (s *DbStore) CurrentStorageIndex() uint64 { s.lock.Lock() defer s.lock.Unlock() return s.dataIdx @@ -410,7 +439,6 @@ func (s *DbStore) Put(chunk *Chunk) { } data := encodeData(chunk) - //data := ethutil.Encode([]interface{}{entry}) if s.entryCnt >= s.capacity { s.collectGarbage(gcArrayFreeRatio) @@ -418,7 +446,9 @@ func (s *DbStore) Put(chunk *Chunk) { batch := new(leveldb.Batch) - batch.Put(getDataKey(s.dataIdx), data) + po := s.po(chunk.Key) + t_datakey := getDataKey(s.dataIdx, po) + batch.Put(t_datakey, data) index.Idx = s.dataIdx s.updateIndexAccess(&index) @@ -430,9 +460,17 @@ func (s *DbStore) Put(chunk *Chunk) { s.entryCnt++ batch.Put(keyDataIdx, U64ToBytes(s.dataIdx)) s.dataIdx++ - batch.Put(keyAccessCnt, U64ToBytes(s.accessCnt)) + accesscnt := make([]byte, 8) + binary.LittleEndian.PutUint64(accesscnt, s.accessCnt) + batch.Put(keyAccessCnt, accesscnt) s.accessCnt++ + s.bucketCnt[po]++ + cntKey := make([]byte, 2) + cntKey[0] = keyDistanceCnt + cntKey[1] = po + batch.Put(cntKey, U64ToBytes(s.bucketCnt[po])) + s.db.Write(batch) if chunk.dbStored != nil { close(chunk.dbStored) @@ -450,7 +488,10 @@ func (s *DbStore) tryAccessIdx(ikey []byte, index *dpaDBIndex) bool { batch := new(leveldb.Batch) - batch.Put(keyAccessCnt, U64ToBytes(s.accessCnt)) + accesscnt := make([]byte, 8) + binary.LittleEndian.PutUint64(accesscnt, s.accessCnt) + batch.Put(keyAccessCnt, accesscnt) + s.accessCnt++ s.updateIndexAccess(index) idata = encodeIndex(index) @@ -464,24 +505,34 @@ func (s *DbStore) tryAccessIdx(ikey []byte, index *dpaDBIndex) bool { func (s *DbStore) Get(key Key) (chunk *Chunk, err error) { s.lock.Lock() defer s.lock.Unlock() + return s.get(key) +} - var index dpaDBIndex +func (s *DbStore) get(key Key) (chunk *Chunk, err error) { + var indx dpaDBIndex - if s.tryAccessIdx(getIndexKey(key), &index) { + if s.tryAccessIdx(getIndexKey(key), &indx) { var data []byte - data, err = s.db.Get(getDataKey(index.Idx)) + + proximity := s.po(key) + datakey := getDataKey(indx.Idx, proximity) + data, err = s.db.Get(datakey) + log.Trace(fmt.Sprintf("DBStore: Chunk %v indexkey %v datakey %x proximity %d", key.Log(), indx.Idx, datakey, proximity)) if err != nil { log.Trace(fmt.Sprintf("DBStore: Chunk %v found but could not be accessed: %v", key.Log(), err)) - s.delete(index.Idx, getIndexKey(key)) + s.delete(indx.Idx, getIndexKey(key), s.po(key)) return } - if s.hashfunc != nil { + if !s.trusted { + data_mod := data[32:] hasher := s.hashfunc() - hasher.Write(data) + hasher.Write(data_mod) hash := hasher.Sum(nil) + if !bytes.Equal(hash, key) { - s.delete(index.Idx, getIndexKey(key)) + log.Trace(fmt.Sprintf("Apparent key/hash mismatch. Hash %x, key %v", hash, key[:])) + s.delete(indx.Idx, getIndexKey(key), s.po(key)) log.Warn("Invalid Chunk in Database. Please repair with command: 'swarm cleandb'") } } @@ -516,7 +567,8 @@ func (s *DbStore) setCapacity(c uint64) { s.capacity = c if s.entryCnt > c { - ratio := float32(1.01) - float32(c)/float32(s.entryCnt) + var ratio float32 + ratio = float32(1.01) - float32(c)/float32(s.entryCnt) if ratio < gcArrayFreeRatio { ratio = gcArrayFreeRatio } @@ -533,62 +585,40 @@ func (s *DbStore) Close() { s.db.Close() } -// describes a section of the DbStore representing the unsynced -// domain relevant to a peer -// Start - Stop designate a continuous area Keys in an address space -// typically the addresses closer to us than to the peer but not closer -// another closer peer in between -// From - To designates a time interval typically from the last disconnect -// till the latest connection (real time traffic is relayed) -type DbSyncState struct { - Start, Stop Key - First, Last uint64 -} - -// implements the syncer iterator interface -// iterates by storage index (~ time of storage = first entry to db) -type dbSyncIterator struct { - it iterator.Iterator - DbSyncState -} - // initialises a sync iterator from a syncToken (passed in with the handshake) -func (self *DbStore) NewSyncIterator(state DbSyncState) (si *dbSyncIterator, err error) { - if state.First > state.Last { - return nil, fmt.Errorf("no entries found") - } - si = &dbSyncIterator{ - it: self.db.NewIterator(), - DbSyncState: state, - } - si.it.Seek(getIndexKey(state.Start)) - return si, nil -} +func (s *DbStore) SyncIterator(since uint64, until uint64, po uint8, f func(Key, uint64) bool) error { + s.lock.Lock() + defer s.lock.Unlock() + untilkey := getDataKey(until, po) -// walk the area from Start to Stop and returns items within time interval -// First to Last -func (self *dbSyncIterator) Next() (key Key) { - for self.it.Valid() { - dbkey := self.it.Key() - if dbkey[0] != 0 { + it := s.db.NewIterator() + seek := getDataKey(since, po) + it.Seek(seek) + defer it.Release() + for it.Valid() { + dbkey := it.Key() + if dbkey[0] != keyData || dbkey[1] != byte(po) || bytes.Compare(untilkey, dbkey) < 0 { break } - key = Key(make([]byte, len(dbkey)-1)) - copy(key[:], dbkey[1:]) - if bytes.Compare(key[:], self.Start) <= 0 { - self.it.Next() - continue - } - if bytes.Compare(key[:], self.Stop) > 0 { + + key := make([]byte, 32) + copy(key, it.Value()[:32]) + if !f(Key(key), binary.BigEndian.Uint64(dbkey[2:])) { break } - var index dpaDBIndex - decodeIndex(self.it.Value(), &index) - self.it.Next() - if (index.Idx >= self.First) && (index.Idx < self.Last) { - return - } + it.Next() } - self.it.Release() return nil } + +func databaseExists(path string) bool { + o := &opt.Options{ + ErrorIfMissing: true, + } + tdb, err := leveldb.OpenFile(path, o) + if err != nil { + return false + } + defer tdb.Close() + return true +} diff --git a/swarm/storage/dbstore_test.go b/swarm/storage/dbstore_test.go index dd165b5768..27bab975a6 100644 --- a/swarm/storage/dbstore_test.go +++ b/swarm/storage/dbstore_test.go @@ -18,174 +18,174 @@ package storage import ( "bytes" + "fmt" "io/ioutil" + "os" "testing" - "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" ) -func initDbStore(t *testing.T) *DbStore { +type testDbStore struct { + *DbStore + dir string +} + +func newTestDbStore() (*testDbStore, error) { dir, err := ioutil.TempDir("", "bzz-storage-test") if err != nil { - t.Fatal(err) + return nil, err } - m, err := NewDbStore(dir, MakeHashFunc(SHA3Hash), defaultDbCapacity, defaultRadius) + basekey := make([]byte, 32) + db, err := NewDbStore(dir, MakeHashFunc(SHA3Hash), defaultDbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) }) + + return &testDbStore{db, dir}, err +} + +func (db *testDbStore) close() { + db.Close() + err := os.RemoveAll(db.dir) if err != nil { - t.Fatal("can't create store:", err) + panic(err) } - return m } -func testDbStore(l int64, branches int64, t *testing.T) { - m := initDbStore(t) - defer m.Close() - testStore(m, l, branches, t) +func testDbStoreRandom(n int, processors int, chunksize int, t *testing.T) { + db, err := newTestDbStore() + if err != nil { + t.Fatalf("init dbStore failed: %v", err) + } + defer db.close() + db.trusted = true + testStoreRandom(db, processors, n, chunksize, t) } -func TestDbStore128_0x1000000(t *testing.T) { - testDbStore(0x1000000, 128, t) +func testDbStoreCorrect(n int, processors int, chunksize int, t *testing.T) { + db, err := newTestDbStore() + if err != nil { + t.Fatalf("init dbStore failed: %v", err) + } + defer db.close() + testStoreCorrect(db, processors, n, chunksize, t) } -func TestDbStore128_10000_(t *testing.T) { - testDbStore(10000, 128, t) +func TestDbStoreRandom_1(t *testing.T) { + testDbStoreRandom(1, 1, 0, t) } -func TestDbStore128_1000_(t *testing.T) { - testDbStore(1000, 128, t) +func TestDbStoreCorrect_1(t *testing.T) { + testDbStoreCorrect(1, 1, 4096, t) } -func TestDbStore128_100_(t *testing.T) { - testDbStore(100, 128, t) +func TestDbStoreRandom_1_5k(t *testing.T) { + testDbStoreRandom(8, 5000, 0, t) } -func TestDbStore2_100_(t *testing.T) { - testDbStore(100, 2, t) +func TestDbStoreRandom_8_5k(t *testing.T) { + testDbStoreRandom(8, 5000, 0, t) +} + +func TestDbStoreCorrect_1_5k(t *testing.T) { + testDbStoreCorrect(1, 5000, 4096, t) +} + +func TestDbStoreCorrect_8_5k(t *testing.T) { + testDbStoreCorrect(8, 5000, 4096, t) } func TestDbStoreNotFound(t *testing.T) { - m := initDbStore(t) - defer m.Close() - _, err := m.Get(ZeroKey) + db, err := newTestDbStore() + if err != nil { + t.Fatalf("init dbStore failed: %v", err) + } + defer db.close() + + _, err = db.Get(ZeroKey) if err != notFound { t.Errorf("Expected notFound, got %v", err) } } -func TestDbStoreSyncIterator(t *testing.T) { - m := initDbStore(t) - defer m.Close() - keys := []Key{ - Key(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000")), - Key(common.Hex2Bytes("4000000000000000000000000000000000000000000000000000000000000000")), - Key(common.Hex2Bytes("5000000000000000000000000000000000000000000000000000000000000000")), - Key(common.Hex2Bytes("3000000000000000000000000000000000000000000000000000000000000000")), - Key(common.Hex2Bytes("2000000000000000000000000000000000000000000000000000000000000000")), - Key(common.Hex2Bytes("1000000000000000000000000000000000000000000000000000000000000000")), - } - for _, key := range keys { - m.Put(NewChunk(key, nil)) - } - it, err := m.NewSyncIterator(DbSyncState{ - Start: Key(common.Hex2Bytes("1000000000000000000000000000000000000000000000000000000000000000")), - Stop: Key(common.Hex2Bytes("4000000000000000000000000000000000000000000000000000000000000000")), - First: 2, - Last: 4, - }) +func TestIterator(t *testing.T) { + var chunkcount int = 32 + var i int + var poc uint + chunkkeys := NewKeyCollection(chunkcount) + chunkkeys_results := NewKeyCollection(chunkcount) + chunks := make([]Chunk, chunkcount) + + db, err := newTestDbStore() if err != nil { - t.Fatalf("unexpected error creating NewSyncIterator") + t.Fatalf("init dbStore failed: %v", err) + } + defer db.close() + + FakeChunk(getDefaultChunkSize(), chunkcount, chunks) + + for i = 0; i < len(chunks); i++ { + db.Put(&chunks[i]) + chunkkeys[i] = chunks[i].Key } - var chunk Key - var res []Key - for { - chunk = it.Next() - if chunk == nil { - break + //testSplit(m, l, 128, chunkkeys, t) + + for i = 0; i < len(chunkkeys); i++ { + log.Trace(fmt.Sprintf("Chunk array pos %d/%d: '%v'", i, chunkcount, chunkkeys[i])) + } + + i = 0 + for poc = 0; poc <= 255; poc++ { + err := db.SyncIterator(0, uint64(chunkkeys.Len()), uint8(poc), func(k Key, n uint64) bool { + log.Trace(fmt.Sprintf("Got key %v number %d poc %d", k, n, uint8(poc))) + chunkkeys_results[n] = k + i++ + return true + }) + if err != nil { + t.Fatalf("Iterator call failed: %v", err) } - res = append(res, chunk) - } - if len(res) != 1 { - t.Fatalf("Expected 1 chunk, got %v: %v", len(res), res) - } - if !bytes.Equal(res[0][:], keys[3]) { - t.Fatalf("Expected %v chunk, got %v", keys[3], res[0]) } - if err != nil { - t.Fatalf("unexpected error creating NewSyncIterator") - } - - it, err = m.NewSyncIterator(DbSyncState{ - Start: Key(common.Hex2Bytes("1000000000000000000000000000000000000000000000000000000000000000")), - Stop: Key(common.Hex2Bytes("5000000000000000000000000000000000000000000000000000000000000000")), - First: 2, - Last: 4, - }) - - res = nil - for { - chunk = it.Next() - if chunk == nil { - break + for i = 0; i < chunkcount; i++ { + if bytes.Compare(chunkkeys[i], chunkkeys_results[i]) != 0 { + t.Fatalf("Chunk put #%d key '%v' does not match iterator's key '%v'", i, chunkkeys[i], chunkkeys_results[i]) } - res = append(res, chunk) - } - if len(res) != 2 { - t.Fatalf("Expected 2 chunk, got %v: %v", len(res), res) - } - if !bytes.Equal(res[0][:], keys[3]) { - t.Fatalf("Expected %v chunk, got %v", keys[3], res[0]) - } - if !bytes.Equal(res[1][:], keys[2]) { - t.Fatalf("Expected %v chunk, got %v", keys[2], res[1]) } - if err != nil { - t.Fatalf("unexpected error creating NewSyncIterator") - } - - it, _ = m.NewSyncIterator(DbSyncState{ - Start: Key(common.Hex2Bytes("1000000000000000000000000000000000000000000000000000000000000000")), - Stop: Key(common.Hex2Bytes("4000000000000000000000000000000000000000000000000000000000000000")), - First: 2, - Last: 5, - }) - res = nil - for { - chunk = it.Next() - if chunk == nil { - break - } - res = append(res, chunk) - } - if len(res) != 2 { - t.Fatalf("Expected 2 chunk, got %v", len(res)) - } - if !bytes.Equal(res[0][:], keys[4]) { - t.Fatalf("Expected %v chunk, got %v", keys[4], res[0]) - } - if !bytes.Equal(res[1][:], keys[3]) { - t.Fatalf("Expected %v chunk, got %v", keys[3], res[1]) - } - - it, _ = m.NewSyncIterator(DbSyncState{ - Start: Key(common.Hex2Bytes("2000000000000000000000000000000000000000000000000000000000000000")), - Stop: Key(common.Hex2Bytes("4000000000000000000000000000000000000000000000000000000000000000")), - First: 2, - Last: 5, - }) - res = nil - for { - chunk = it.Next() - if chunk == nil { - break - } - res = append(res, chunk) - } - if len(res) != 1 { - t.Fatalf("Expected 1 chunk, got %v", len(res)) - } - if !bytes.Equal(res[0][:], keys[3]) { - t.Fatalf("Expected %v chunk, got %v", keys[3], res[0]) - } +} + +func benchmarkDbStorePut(n int, processors int, chunksize int, b *testing.B) { + db, err := newTestDbStore() + if err != nil { + b.Fatalf("init dbStore failed: %v", err) + } + defer db.close() + db.trusted = true + benchmarkStorePut(db, processors, n, chunksize, b) +} + +func benchmarkDbStoreGet(n int, processors int, chunksize int, b *testing.B) { + db, err := newTestDbStore() + if err != nil { + b.Fatalf("init dbStore failed: %v", err) + } + defer db.close() + db.trusted = true + benchmarkStoreGet(db, processors, n, chunksize, b) +} + +func BenchmarkDbStorePut_1_5k(b *testing.B) { + benchmarkDbStorePut(5000, 1, 4096, b) +} + +func BenchmarkDbStorePut_8_5k(b *testing.B) { + benchmarkDbStorePut(5000, 8, 4096, b) +} + +func BenchmarkDbStoreGet_1_5k(b *testing.B) { + benchmarkDbStoreGet(5000, 1, 4096, b) +} + +func BenchmarkDbStoreGet_8_5k(b *testing.B) { + benchmarkDbStoreGet(5000, 8, 4096, b) } diff --git a/swarm/storage/dpa.go b/swarm/storage/dpa.go index 49a362555e..b8f7f5fd8f 100644 --- a/swarm/storage/dpa.go +++ b/swarm/storage/dpa.go @@ -59,15 +59,16 @@ type DPA struct { lock sync.Mutex running bool + wg *sync.WaitGroup quitC chan bool } // for testing locally -func NewLocalDPA(datadir string) (*DPA, error) { +func NewLocalDPA(datadir string, basekey []byte) (*DPA, error) { hash := MakeHashFunc("SHA3") - dbStore, err := NewDbStore(datadir, hash, singletonSwarmDbCapacity, 0) + dbStore, err := NewDbStore(datadir, hash, singletonSwarmDbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) }) if err != nil { return nil, err } @@ -135,11 +136,8 @@ func (self *DPA) retrieveLoop() { func (self *DPA) retrieveWorker() { for chunk := range self.retrieveC { - log.Trace(fmt.Sprintf("dpa: retrieve loop : chunk %v", chunk.Key.Log())) storedChunk, err := self.Get(chunk.Key) - if err == notFound { - log.Trace(fmt.Sprintf("chunk %v not found", chunk.Key.Log())) - } else if err != nil { + if err != nil { log.Trace(fmt.Sprintf("error retrieving chunk %v: %v", chunk.Key.Log(), err)) } else { chunk.SData = storedChunk.SData @@ -169,9 +167,7 @@ func (self *DPA) storeWorker() { for chunk := range self.storeC { self.Put(chunk) if chunk.wg != nil { - log.Trace(fmt.Sprintf("dpa: store processor %v", chunk.Key.Log())) chunk.wg.Done() - } select { case <-self.quitC: @@ -200,7 +196,6 @@ func NewDpaChunkStore(localStore, netStore ChunkStore) *dpaChunkStore { // waits for response or times out func (self *dpaChunkStore) Get(key Key) (chunk *Chunk, err error) { chunk, err = self.netStore.Get(key) - // timeout := time.Now().Add(searchTimeout) if chunk.SData != nil { log.Trace(fmt.Sprintf("DPA.Get: %v found locally, %d bytes", key.Log(), len(chunk.SData))) return @@ -238,4 +233,5 @@ func (self *dpaChunkStore) Put(entry *Chunk) { } // Close chunk store -func (self *dpaChunkStore) Close() {} +func (self *dpaChunkStore) Close() { +} diff --git a/swarm/storage/dpa_test.go b/swarm/storage/dpa_test.go index a23b9efebe..3bccd82d54 100644 --- a/swarm/storage/dpa_test.go +++ b/swarm/storage/dpa_test.go @@ -28,12 +28,17 @@ import ( const testDataSize = 0x1000000 func TestDPArandom(t *testing.T) { - dbStore := initDbStore(t) - dbStore.setCapacity(50000) - memStore := NewMemStore(dbStore, defaultCacheCapacity) + tdb, err := newTestDbStore() + if err != nil { + t.Fatalf("init dbStore failed: %v", err) + } + defer tdb.close() + db := tdb.DbStore + db.setCapacity(50000) + memStore := NewMemStore(db, defaultCacheCapacity) localStore := &LocalStore{ memStore, - dbStore, + db, } chunker := NewTreeChunker(NewChunkerParams()) dpa := &DPA{ @@ -65,7 +70,7 @@ func TestDPArandom(t *testing.T) { } ioutil.WriteFile("/tmp/slice.bzz.16M", slice, 0666) ioutil.WriteFile("/tmp/result.bzz.16M", resultSlice, 0666) - localStore.memStore = NewMemStore(dbStore, defaultCacheCapacity) + localStore.memStore = NewMemStore(db, defaultCacheCapacity) resultReader = dpa.Retrieve(key) for i := range resultSlice { resultSlice[i] = 0 @@ -83,13 +88,17 @@ func TestDPArandom(t *testing.T) { } func TestDPA_capacity(t *testing.T) { - dbStore := initDbStore(t) - memStore := NewMemStore(dbStore, defaultCacheCapacity) + tdb, err := newTestDbStore() + if err != nil { + t.Fatalf("init dbStore failed: %v", err) + } + defer tdb.close() + db := tdb.DbStore + memStore := NewMemStore(db, 0) localStore := &LocalStore{ memStore, - dbStore, + db, } - memStore.setCapacity(0) chunker := NewTreeChunker(NewChunkerParams()) dpa := &DPA{ Chunker: chunker, diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index bf9eeb2e77..2ed9fb305a 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -28,8 +28,8 @@ type LocalStore struct { } // This constructor uses MemStore and DbStore as components -func NewLocalStore(hash SwarmHasher, params *StoreParams) (*LocalStore, error) { - dbStore, err := NewDbStore(params.ChunkDbPath, hash, params.DbCapacity, params.Radius) +func NewLocalStore(hash SwarmHasher, params *StoreParams, basekey []byte) (*LocalStore, error) { + dbStore, err := NewDbStore(params.ChunkDbPath, hash, params.DbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) }) if err != nil { return nil, err } diff --git a/swarm/storage/memstore_test.go b/swarm/storage/memstore_test.go index 2e0ab535af..6b4bc0da56 100644 --- a/swarm/storage/memstore_test.go +++ b/swarm/storage/memstore_test.go @@ -16,35 +16,82 @@ package storage -import ( - "testing" -) +import "testing" -func testMemStore(l int64, branches int64, t *testing.T) { - m := NewMemStore(nil, defaultCacheCapacity) - testStore(m, l, branches, t) +func newTestMemStore() *MemStore { + return NewMemStore(nil, defaultCacheCapacity) } -func TestMemStore128_10000(t *testing.T) { - testMemStore(10000, 128, t) +func testMemStoreRandom(n int, processors int, chunksize int, t *testing.T) { + m := newTestMemStore() + defer m.Close() + testStoreRandom(m, processors, n, chunksize, t) } -func TestMemStore128_1000(t *testing.T) { - testMemStore(1000, 128, t) +func testMemStoreCorrect(n int, processors int, chunksize int, t *testing.T) { + m := newTestMemStore() + defer m.Close() + testStoreCorrect(m, processors, n, chunksize, t) } -func TestMemStore128_100(t *testing.T) { - testMemStore(100, 128, t) +func TestMemStoreRandom_1(t *testing.T) { + testMemStoreRandom(1, 1, 0, t) } -func TestMemStore2_100(t *testing.T) { - testMemStore(100, 2, t) +func TestMemStoreCorrect_1(t *testing.T) { + testMemStoreCorrect(1, 1, 4104, t) +} + +func TestMemStoreRandom_1_10k(t *testing.T) { + testMemStoreRandom(1, 5000, 0, t) +} + +func TestMemStoreCorrect_1_10k(t *testing.T) { + testMemStoreCorrect(1, 5000, 4096, t) +} + +func TestMemStoreRandom_8_10k(t *testing.T) { + testMemStoreRandom(8, 5000, 0, t) +} + +func TestMemStoreCorrect_8_10k(t *testing.T) { + testMemStoreCorrect(8, 5000, 4096, t) } func TestMemStoreNotFound(t *testing.T) { - m := NewMemStore(nil, defaultCacheCapacity) + m := newTestMemStore() + defer m.Close() + _, err := m.Get(ZeroKey) if err != notFound { t.Errorf("Expected notFound, got %v", err) } } + +func benchmarkMemStorePut(n int, processors int, chunksize int, b *testing.B) { + m := newTestMemStore() + defer m.Close() + benchmarkStorePut(m, processors, n, chunksize, b) +} + +func benchmarkMemStoreGet(n int, processors int, chunksize int, b *testing.B) { + m := newTestMemStore() + defer m.Close() + benchmarkStoreGet(m, processors, n, chunksize, b) +} + +func BenchmarkMemStorePut_1_5k(b *testing.B) { + benchmarkMemStorePut(5000, 1, 4096, b) +} + +func BenchmarkMemStorePut_8_5k(b *testing.B) { + benchmarkMemStorePut(5000, 8, 4096, b) +} + +func BenchmarkMemStoreGet_1_5k(b *testing.B) { + benchmarkMemStoreGet(5000, 1, 4096, b) +} + +func BenchmarkMemStoreGet_8_5k(b *testing.B) { + benchmarkMemStoreGet(5000, 8, 4096, b) +} diff --git a/swarm/storage/types.go b/swarm/storage/types.go index d35f1f9294..e2c111f7b1 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -19,6 +19,8 @@ package storage import ( "bytes" "crypto" + "crypto/rand" + "encoding/binary" "fmt" "hash" "io" @@ -29,6 +31,8 @@ import ( "github.com/ethereum/go-ethereum/crypto/sha3" ) +const MaxPO = 7 + type Hasher func() hash.Hash type SwarmHasher func() SwarmHash @@ -73,6 +77,26 @@ func (h Key) bits(i, j uint) uint { return res } +func Proximity(one, other []byte) (ret int) { + b := (MaxPO-1)/8 + 1 + if b > len(one) { + b = len(one) + } + m := 8 + for i := 0; i < b; i++ { + oxo := one[i] ^ other[i] + if i == b-1 { + m = MaxPO % 8 + } + for j := 0; j < m; j++ { + if (uint8(oxo)>>uint8(7-j))&0x01 != 0 { + return i*8 + j + } + } + } + return MaxPO +} + func IsZeroKey(key Key) bool { return len(key) == 0 || bytes.Equal(key, ZeroKey) } @@ -100,10 +124,10 @@ func (key Key) Hex() string { } func (key Key) Log() string { - if len(key[:]) < 4 { + if len(key[:]) < 8 { return fmt.Sprintf("%x", []byte(key[:])) } - return fmt.Sprintf("%08x", []byte(key[:4])) + return fmt.Sprintf("%016x", []byte(key[:8])) } func (key Key) String() string { @@ -122,6 +146,27 @@ func (key *Key) UnmarshalJSON(value []byte) error { return nil } +type KeyCollection []Key + +func NewKeyCollection(l int) KeyCollection { + return make(KeyCollection, l) +} + +func (c KeyCollection) Len() int { + return len(c) +} + +func (c KeyCollection) Less(i, j int) bool { + if bytes.Compare(c[i], c[j]) == -1 { + return true + } + return false +} + +func (c KeyCollection) Swap(i, j int) { + c[i], c[j] = c[j], c[i] +} + // each chunk when first requested opens a record associated with the request // next time a request for the same chunk arrives, this record is updated // this request status keeps track of the request ID-s as well as the requesting @@ -163,6 +208,32 @@ func NewChunk(key Key, rs *RequestStatus) *Chunk { return &Chunk{Key: key, Req: rs} } +func FakeChunk(size int64, count int, chunks []Chunk) int { + var i int + hasher := MakeHashFunc(SHA3Hash)() + chunksize := getDefaultChunkSize() + if size > chunksize { + size = chunksize + } + + for i = 0; i < count; i++ { + hasher.Reset() + chunks[i].SData = make([]byte, size) + rand.Read(chunks[i].SData) + binary.LittleEndian.PutUint64(chunks[i].SData[:8], uint64(size)) + hasher.Write(chunks[i].SData) + chunks[i].Key = make([]byte, 32) + copy(chunks[i].Key, hasher.Sum(nil)) + } + + return i +} + +func getDefaultChunkSize() int64 { + return DefaultBranches * int64(MakeHashFunc(SHA3Hash)().Size()) + +} + /* The ChunkStore interface is implemented by : From 95d01cb354c57013025a7fa43fe05d9ce22c936c Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Wed, 3 Jan 2018 17:45:51 +0100 Subject: [PATCH 007/210] cmd/swarm, swarm: merge important parts from swarm-gateways-db-fixes --- cmd/swarm/db.go | 23 +-- cmd/swarm/hash.go | 2 +- swarm/api/api.go | 22 +-- swarm/api/api_test.go | 2 +- swarm/api/filesystem.go | 7 +- swarm/api/http/server.go | 2 +- swarm/api/http/server_test.go | 8 +- swarm/api/manifest.go | 10 +- swarm/api/storage.go | 2 +- swarm/network/requests.go | 46 +++-- swarm/network/streamer.go | 108 +++++++---- swarm/network/syncer.go | 186 ++++++++++--------- swarm/pss/pss.go | 4 +- swarm/storage/chunker.go | 67 +++---- swarm/storage/common_test.go | 9 +- swarm/storage/dbstore.go | 326 +++++++++++++++++++++++++--------- swarm/storage/dbstore_test.go | 8 +- swarm/storage/dpa.go | 8 +- swarm/storage/dpa_test.go | 11 +- swarm/storage/localstore.go | 1 - swarm/storage/memstore.go | 10 +- swarm/storage/netstore.go | 2 +- swarm/storage/pyramid.go | 10 +- swarm/storage/types.go | 14 +- swarm/swarm.go | 28 +-- 25 files changed, 535 insertions(+), 381 deletions(-) diff --git a/cmd/swarm/db.go b/cmd/swarm/db.go index dfd2d069b9..82db713f14 100644 --- a/cmd/swarm/db.go +++ b/cmd/swarm/db.go @@ -23,6 +23,7 @@ import ( "path/filepath" "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/storage" "gopkg.in/urfave/cli.v1" @@ -30,11 +31,11 @@ import ( func dbExport(ctx *cli.Context) { args := ctx.Args() - if len(args) != 2 { - utils.Fatalf("invalid arguments, please specify both (path to a local chunk database) and (path to write the tar archive to, - for stdout)") + if len(args) != 3 { + utils.Fatalf("invalid arguments, please specify both (path to a local chunk database), (path to write the tar archive to, - for stdout) and the base key") } - store, err := openDbStore(args[0]) + store, err := openDbStore(args[0], common.Hex2Bytes(args[2])) if err != nil { utils.Fatalf("error opening local chunk database: %s", err) } @@ -62,11 +63,11 @@ func dbExport(ctx *cli.Context) { func dbImport(ctx *cli.Context) { args := ctx.Args() - if len(args) != 2 { - utils.Fatalf("invalid arguments, please specify both (path to a local chunk database) and (path to read the tar archive from, - for stdin)") + if len(args) != 3 { + utils.Fatalf("invalid arguments, please specify both (path to a local chunk database), (path to read the tar archive from, - for stdin) and the base key") } - store, err := openDbStore(args[0]) + store, err := openDbStore(args[0], common.Hex2Bytes(args[2])) if err != nil { utils.Fatalf("error opening local chunk database: %s", err) } @@ -94,11 +95,11 @@ func dbImport(ctx *cli.Context) { func dbClean(ctx *cli.Context) { args := ctx.Args() - if len(args) != 1 { - utils.Fatalf("invalid arguments, please specify (path to a local chunk database)") + if len(args) != 2 { + utils.Fatalf("invalid arguments, please specify (path to a local chunk database) and the base key") } - store, err := openDbStore(args[0]) + store, err := openDbStore(args[0], common.Hex2Bytes(args[1])) if err != nil { utils.Fatalf("error opening local chunk database: %s", err) } @@ -107,10 +108,10 @@ func dbClean(ctx *cli.Context) { store.Cleanup() } -func openDbStore(path string) (*storage.DbStore, error) { +func openDbStore(path string, basekey []byte) (*storage.DbStore, error) { if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil { return nil, fmt.Errorf("invalid chunkdb path: %s", err) } hash := storage.MakeHashFunc("SHA3") - return storage.NewDbStore(path, hash, 10000000, 0) + return storage.NewDbStore(path, hash, 10000000, func(k storage.Key) (ret uint8) { return uint8(storage.Proximity(basekey[:], k[:])) }) } diff --git a/cmd/swarm/hash.go b/cmd/swarm/hash.go index 792e8d0d7a..a6e6a6ba76 100644 --- a/cmd/swarm/hash.go +++ b/cmd/swarm/hash.go @@ -39,7 +39,7 @@ func hash(ctx *cli.Context) { stat, _ := f.Stat() chunker := storage.NewTreeChunker(storage.NewChunkerParams()) - key, err := chunker.Split(f, stat.Size(), nil, nil, nil) + key, _, err := chunker.Split(f, stat.Size(), nil) if err != nil { utils.Fatalf("%v\n", err) } else { diff --git a/swarm/api/api.go b/swarm/api/api.go index 8c4bca2ec0..e187b748d6 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -22,7 +22,6 @@ import ( "net/http" "regexp" "strings" - "sync" "bytes" "mime" @@ -71,8 +70,8 @@ func (self *Api) Retrieve(key storage.Key) storage.LazySectionReader { return self.dpa.Retrieve(key) } -func (self *Api) Store(data io.Reader, size int64, wg *sync.WaitGroup) (key storage.Key, err error) { - return self.dpa.Store(data, size, wg, nil) +func (self *Api) Store(data io.Reader, size int64) (key storage.Key, wait func(), err error) { + return self.dpa.Store(data, size) } type ErrResolve error @@ -109,21 +108,22 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) { } // Put provides singleton manifest creation on top of dpa store -func (self *Api) Put(content, contentType string) (storage.Key, error) { +func (self *Api) Put(content, contentType string) (k storage.Key, wait func(), err error) { r := strings.NewReader(content) - wg := &sync.WaitGroup{} - key, err := self.dpa.Store(r, int64(len(content)), wg, nil) + key, waitContent, err := self.dpa.Store(r, int64(len(content))) if err != nil { - return nil, err + return nil, nil, err } manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType) r = strings.NewReader(manifest) - key, err = self.dpa.Store(r, int64(len(manifest)), wg, nil) + key, waitManifest, err := self.dpa.Store(r, int64(len(manifest))) if err != nil { - return nil, err + return nil, nil, err } - wg.Wait() - return key, nil + return key, func() { + waitContent() + waitManifest() + }, nil } // Get uses iterative manifest retrieval and prefix matching diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index e673f76c42..44bf8aadc2 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -110,7 +110,7 @@ func TestApiPut(t *testing.T) { content := "hello" exp := expResponse(content, "text/plain", 0) // exp := expResponse([]byte(content), "text/plain", 0) - key, err := api.Put(content, exp.MimeType) + key, _, err := api.Put(content, exp.MimeType) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/swarm/api/filesystem.go b/swarm/api/filesystem.go index f5dc90e2e5..b6a2de8862 100644 --- a/swarm/api/filesystem.go +++ b/swarm/api/filesystem.go @@ -43,6 +43,7 @@ func NewFileSystem(api *Api) *FileSystem { // Upload replicates a local directory as a manifest file and uploads it // using dpa store +// This function waits the chunks to be stored. // TODO: localpath should point to a manifest // // DEPRECATED: Use the HTTP API instead @@ -112,12 +113,12 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) { if err == nil { stat, _ := f.Stat() var hash storage.Key - wg := &sync.WaitGroup{} - hash, err = self.api.dpa.Store(f, stat.Size(), wg, nil) + var wait func() + hash, wait, err = self.api.dpa.Store(f, stat.Size()) if hash != nil { list[i].Hash = hash.String() } - wg.Wait() + wait() awg.Done() if err == nil { first512 := make([]byte, 512) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 74341899d2..7adddd9ff4 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -99,7 +99,7 @@ func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) { return } - key, err := s.api.Store(r.Body, r.ContentLength, nil) + key, _, err := s.api.Store(r.Body, r.ContentLength) if err != nil { s.Error(w, r, err) return diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 305d5cf7db..6a35d2c785 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -23,7 +23,6 @@ import ( "io/ioutil" "net/http" "strings" - "sync" "testing" "github.com/ethereum/go-ethereum/common" @@ -59,15 +58,14 @@ func TestBzzGetPath(t *testing.T) { srv := testutil.NewTestSwarmServer(t) defer srv.Close() - wg := &sync.WaitGroup{} - for i, mf := range testmanifest { reader[i] = bytes.NewReader([]byte(mf)) - key[i], err = srv.Dpa.Store(reader[i], int64(len(mf)), wg, nil) + var wait func() + key[i], wait, err = srv.Dpa.Store(reader[i], int64(len(mf))) if err != nil { t.Fatal(err) } - wg.Wait() + wait() } _, err = http.Get(srv.URL + "/bzz-raw:/" + common.ToHex(key[0])[2:] + "/a") diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index 685a300fca..f6279a4ced 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -24,7 +24,6 @@ import ( "io" "net/http" "strings" - "sync" "time" "github.com/ethereum/go-ethereum/common" @@ -65,7 +64,8 @@ func (a *Api) NewManifest() (storage.Key, error) { if err != nil { return nil, err } - return a.Store(bytes.NewReader(data), int64(len(data)), &sync.WaitGroup{}) + key, _, err := a.Store(bytes.NewReader(data), int64(len(data))) + return key, err } // ManifestWriter is used to add and remove entries from an underlying manifest @@ -85,7 +85,7 @@ func (a *Api) NewManifestWriter(key storage.Key, quitC chan bool) (*ManifestWrit // AddEntry stores the given data and adds the resulting key to the manifest func (m *ManifestWriter) AddEntry(data io.Reader, e *ManifestEntry) (storage.Key, error) { - key, err := m.api.Store(data, e.Size, nil) + key, _, err := m.api.Store(data, e.Size) if err != nil { return nil, err } @@ -351,9 +351,7 @@ func (self *manifestTrie) recalcAndStore() error { } sr := bytes.NewReader(manifest) - wg := &sync.WaitGroup{} - key, err2 := self.dpa.Store(sr, int64(len(manifest)), wg, nil) - wg.Wait() + key, _, err2 := self.dpa.Store(sr, int64(len(manifest))) self.hash = key return err2 } diff --git a/swarm/api/storage.go b/swarm/api/storage.go index 0e3abecfe4..ae94e15cb9 100644 --- a/swarm/api/storage.go +++ b/swarm/api/storage.go @@ -42,7 +42,7 @@ func NewStorage(api *Api) *Storage { // // DEPRECATED: Use the HTTP API instead func (self *Storage) Put(content, contentType string) (string, error) { - key, err := self.api.Put(content, contentType) + key, _, err := self.api.Put(content, contentType) if err != nil { return "", err } diff --git a/swarm/network/requests.go b/swarm/network/requests.go index 311acfc927..27e0ea2853 100644 --- a/swarm/network/requests.go +++ b/swarm/network/requests.go @@ -17,8 +17,6 @@ package network import ( - "bytes" - "encoding/binary" "fmt" "github.com/ethereum/go-ethereum/log" @@ -54,19 +52,19 @@ testing chunk availability etc etc, we can indicate it by limiting the size here Request ID can be newly generated or kept from the request originator. */ -type retrieveRequestMsgData struct { - Key storage.Key // target Key address of chunk to be retrieved - Id uint64 // request id, request is a lookup if missing or zero - MaxSize uint64 // maximum size of delivery accepted - from Peer // +type retrieveRequestMsg struct { + Key storage.Key // target Key address of chunk to be retrieved + Id uint64 // request id, request is a lookup if missing or zero + MaxSize uint64 // maximum size of delivery accepted + from *StreamerPeer // } -func (self retrieveRequestMsgData) String() string { +func (self retrieveRequestMsg) String() string { var from string if self.from == nil { from = "ourselves" } else { - from = fmt.Sprintf("%x", self.from.OverlayAddr()) + from = fmt.Sprintf("%x", self.from.Over()) } var target []byte if len(self.Key) > 3 { @@ -77,9 +75,9 @@ func (self retrieveRequestMsgData) String() string { // entrypoint for retrieve requests coming from the bzz wire protocol // checks swap balance - return if peer has no credit -func (self *RequestHandler) handleRetrieveRequestMsg(msg interface{}, p Peer) error { - req := msg.(*retrieveRequestMsgData) - req.from = p +func (self *StreamerPeer) handleRetrieveRequestMsg(msg interface{}) error { + req := msg.(*retrieveRequestMsg) + req.from = self // TODO: // swap - record credit for 1 request // note that only charge actual reqsearches @@ -90,15 +88,15 @@ func (self *RequestHandler) handleRetrieveRequestMsg(msg interface{}, p Peer) er chunk, _ := self.netStore.Get(req.Key) rs := chunk.Req if rs != nil { - rs = storage.NewRequest() - self.addRequester(rs, req) + rs = storage.NewRequestStatus(req.Key) + addRequester(rs, req) chunk.Req = rs } // check if we can immediately deliver if chunk.SData != nil { if req.MaxSize == 0 || int64(req.MaxSize) >= chunk.Size { - err := self.netStore.Deliver(chunk) + err := self.Deliver(chunk, Top) if err != nil { log.Trace(fmt.Sprintf("%v - content found, delivery error: %v", req.Key.Log(), err)) return nil @@ -118,7 +116,7 @@ adds a new peer to an existing open request only add if less than requesterCount peers forwarded the same request id so far note this is done irrespective of status (searching or found) */ -func (self *RequestHandler) addRequester(rs *storage.RequestStatus, req *retrieveRequestMsgData) { +func addRequester(rs *storage.RequestStatus, req *retrieveRequestMsg) { log.Trace(fmt.Sprintf("Depo.addRequester: key %v - add peer to req.Id %v", req.Key.Log(), req.Id)) list := rs.Requesters[req.Id] rs.Requesters[req.Id] = append(list, req) @@ -128,19 +126,20 @@ func (self *RequestHandler) addRequester(rs *storage.RequestStatus, req *retriev store requests are put in netstore so they are stored and then forwarded to the peers in their kademlia proximity bin by the syncer */ -type storeRequestMsgData struct { +type storeRequestMsg struct { + Key storage.Key SData []byte // the stored chunk Data (incl size) // optional Id uint64 // request ID. if delivery, the ID is retrieve request ID from Peer // [not serialised] protocol registers the requester } -func (self storeRequestMsgData) String() string { +func (self storeRequestMsg) String() string { var from string if self.from == nil { from = "self" } else { - from = fmt.Sprintf("%x", self.from.OverlayAddr()) + from = fmt.Sprintf("%x", self.from.Over()) } end := len(self.SData) if len(self.SData) > 10 { @@ -153,12 +152,11 @@ func (self storeRequestMsgData) String() string { // if key found locally, return. otherwise // remote is untrusted, so hash is verified and chunk passed on to NetStore func (self *RequestHandler) handleStoreRequestMsg(msg interface{}, p Peer) error { - req := msg.(*storeRequestMsgData) + req := msg.(*storeRequestMsg) req.from = p - chunk, err := storage.NewChunkFromData(req.SData) - if err != nil { - return err - } + // TODO: chunk validation + chunk := storage.NewChunk(req.Key, nil) + chunk.SData = req.SData chunk.Source = p self.netStore.Put(chunk) log.Trace(fmt.Sprintf("delivery of %v from %v", chunk, p)) diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go index 7941a261dd..0207f3a0fc 100644 --- a/swarm/network/streamer.go +++ b/swarm/network/streamer.go @@ -18,13 +18,15 @@ package network import ( "context" + "errors" "fmt" "sync" - "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/protocols" bv "github.com/ethereum/go-ethereum/swarm/network/bitvector" pq "github.com/ethereum/go-ethereum/swarm/network/priorityqueue" + "github.com/ethereum/go-ethereum/swarm/storage" ) const ( @@ -43,9 +45,9 @@ type Stream string // Handover represents a statement that the upstream peer hands over the stream section type Handover struct { - Stream Stream // name of stream - Start, End uint64 // index of hashes - Root common.Hash // Root hash for indexed segment inclusion proofs + Stream Stream // name of stream + Start, End uint64 // index of hashes + Root []byte // Root hash for indexed segment inclusion proofs } // HandoverProof represents a signed statement that the upstream peer handed over the stream section @@ -70,7 +72,7 @@ type TakeoverProofMsg TakeoverProof // String pretty prints TakeoverProofMsg func (self TakeoverProofMsg) String() string { - return fmt.Sprintf("Stream: '%v' [%v-%v], Root: %x, Sig: %x", self.Stream, self.From, self.To, self.Root, self.Sig) + return fmt.Sprintf("Stream: '%v' [%v-%v], Root: %x, Sig: %x", self.Stream, self.Start, self.End, self.Root, self.Sig) } // SubcribeMsg is the protocol msg for requesting a stream(section) @@ -138,37 +140,46 @@ func (self *Streamer) RegisterOutgoingStreamer(stream Stream, f func(*StreamerPe } // GetIncomingStreamer accessor for incoming streamer constructors -func (self *Streamer) GetIncomingStreamer(stream Stream) func(*StreamerPeer) (IncomingStreamer, error) { +func (self *Streamer) GetIncomingStreamer(stream Stream) (func(*StreamerPeer) (IncomingStreamer, error), error) { self.incomingLock.RLock() defer self.incomingLock.RUnlock() f := self.incoming[stream] if f == nil { - return nil, fmt.Errorf("stream %v not registered", s) + return nil, fmt.Errorf("stream %v not registered", stream) } return f, nil } // GetOutgoingStreamer accessor for incoming streamer constructors -func (self *Streamer) GetOutgoingStreamer(stream Stream) func(*StreamerPeer) (OutgoingStreamer, error) { +func (self *Streamer) GetOutgoingStreamer(stream Stream) (func(*StreamerPeer) (OutgoingStreamer, error), error) { self.outgoingLock.RLock() defer self.outgoingLock.RUnlock() f := self.outgoing[stream] if f == nil { - return nil, fmt.Errorf("stream %v not registered", s) + return nil, fmt.Errorf("stream %v not registered", stream) } + return f, nil +} + +func (self *Streamer) NodeInfo() interface{} { + return nil +} + +func (self *Streamer) PeerInfo(id discover.NodeID) interface{} { + return nil } // OutgoingStreamer interface for outgoing peer Streamer type OutgoingStreamer interface { CurrentBatch() []byte - SetNextBatch(uint64, uint64) ([]byte, uint64, uint64, *HandoverProof) + SetNextBatch(uint64, uint64) ([]byte, uint64, uint64, *HandoverProof, error) GetData([]byte) []byte Priority() int } // IncomingStreamer interface for incoming peer Streamer type IncomingStreamer interface { - NextBatch(uint64, uint64) (uint64, uint64) + NextBatch(uint64) (uint64, uint64) NeedData([]byte) func() Priority() int } @@ -178,6 +189,7 @@ type StreamerPeer struct { Peer streamer *Streamer pq *pq.PriorityQueue + netStore storage.ChunkStore outgoingLock sync.RWMutex incomingLock sync.RWMutex outgoing map[Stream]OutgoingStreamer @@ -249,7 +261,10 @@ func (self *StreamerPeer) Subscribe(s Stream, from, to uint64) error { if err != nil { return err } - is := f(self) + is, err := f(self) + if err != nil { + return err + } self.setIncomingStreamer(s, is) msg := &SubscribeMsg{ Stream: s, @@ -257,20 +272,24 @@ func (self *StreamerPeer) Subscribe(s Stream, from, to uint64) error { To: to, Priority: uint8(is.Priority()), } - self.Send(msg, is.Priority()) + self.SendPriority(msg, is.Priority()) + return nil } func (self *StreamerPeer) handleSubscribeMsg(msg interface{}) error { req := msg.(*SubscribeMsg) - f, err := self.streamer.getOutgoingStreamer(req.Stream) + f, err := self.streamer.GetOutgoingStreamer(req.Stream) + if err != nil { + return err + } + s, err := f(self) if err != nil { return err } - s := f(self) if err := self.setOutgoingStreamer(req.Stream, s); err != nil { return nil } - self.UnsyncedKeys(s, req.From, req.To) + self.SendUnsyncedKeys(s, req.From, req.To, int(req.Priority)) return nil } @@ -278,13 +297,15 @@ func (self *StreamerPeer) handleSubscribeMsg(msg interface{}) error { // Filter method func (self *StreamerPeer) handleUnsyncedKeysMsg(msg interface{}) error { req := msg.(*UnsyncedKeysMsg) - req.C = make(chan struct{}) s, err := self.getIncomingStreamer(req.Stream) if err != nil { return err } hashes := req.Hashes - want := bv.New(len(hashes) / HashSize) + want, err := bv.New(len(hashes) / HashSize) + if err != nil { + return err + } wg := sync.WaitGroup{} for i := 0; i < len(hashes)/HashSize; i += HashSize { hash := hashes[i : i+HashSize] @@ -298,24 +319,24 @@ func (self *StreamerPeer) handleUnsyncedKeysMsg(msg interface{}) error { }(wait) } } - go func() { - wg.Wait() - msg := s.TakeoverProof(req.Stream, req.From, req.Hashes, req.Root) - self.Send(msg, s.Priority()) - }() + // go func() { + // wg.Wait() + // msg := s.TakeoverProof(req.Stream, req.From, req.Hashes, req.Root) + // self.Send(msg, s.Priority()) + // }() // only send wantedKeysMsg if all missing chunks of the previous batch arrived // except - from, to = s.NextBatch(from, to) + from, to := s.NextBatch(req.To) if from == to { return nil } msg = &WantedKeysMsg{ Stream: req.Stream, - Want: want, + Want: want.Bytes(), From: from, To: to, } - self.Send(msg, s.Priority()) + self.SendPriority(msg, s.Priority()) return nil } @@ -330,17 +351,22 @@ func (self *StreamerPeer) handleWantedKeysMsg(msg interface{}) error { } hashes := s.CurrentBatch() // launch in go routine since GetBatch blocks until new hashes arrive - go self.UnsyncedKeys(s, req.From, req.To) + go self.SendUnsyncedKeys(s, req.From, req.To, s.Priority()) l := len(hashes) / HashSize - want := bv.NewFromBytes(req.Want, l) + want, err := bv.NewFromBytes(req.Want, l) + if err != nil { + return err + } for i := 0; i < l; i++ { if want.Get(i) { hash := hashes[i*HashSize : (i+1)*HashSize] data := s.GetData(hash) if data == nil { - return errNotFound + return errors.New("not found") } - if err := self.Deliver(data, s.Priority()); err != nil { + chunk := storage.NewChunk(hash, nil) + chunk.SData = data + if err := self.Deliver(chunk, s.Priority()); err != nil { return err } } @@ -350,7 +376,7 @@ func (self *StreamerPeer) handleWantedKeysMsg(msg interface{}) error { func (self *StreamerPeer) handleTakeoverProofMsg(msg interface{}) error { req := msg.(*TakeoverProofMsg) - s, err := self.getOutgoingStreamer(req.Stream) + _, err := self.getOutgoingStreamer(req.Stream) if err != nil { return err } @@ -359,32 +385,36 @@ func (self *StreamerPeer) handleTakeoverProofMsg(msg interface{}) error { } // Deliver sends a storeRequestMsg protocol message to the peer -func (self *StreamerPeer) Deliver(data []byte, priority int) error { +func (self *StreamerPeer) Deliver(chunk *storage.Chunk, priority int) error { msg := &storeRequestMsg{ - SData: data, + Key: chunk.Key, + SData: chunk.SData, } return self.pq.Push(nil, msg, priority) } // Deliver sends a storeRequestMsg protocol message to the peer -func (self *StreamerPeer) Send(msg interface{}, priority int) error { +func (self *StreamerPeer) SendPriority(msg interface{}, priority int) error { return self.pq.Push(nil, msg, priority) } // UnsyncedKeys sends UnsyncedKeysMsg protocol msg -func (self *StreamerPeer) SendUnsyncedKeys(s OutgoingStreamer, f, t uint64, priority int) { - hashes, from, to, proof := s.SetNextBatch(f, t) +func (self *StreamerPeer) SendUnsyncedKeys(s OutgoingStreamer, f, t uint64, priority int) error { + hashes, from, to, proof, err := s.SetNextBatch(f, t) + if err != nil { + return err + } msg := &UnsyncedKeysMsg{ HandoverProof: proof, Hashes: hashes, From: from, To: to, } - self.Send(msg, s.Priority()) + return self.SendPriority(msg, s.Priority()) } -// BzzSpec is the spec of the generic swarm handshake -var StrSpec = &protocols.Spec{ +// StreamerSpec is the spec of the streamer protocol. +var StreamerSpec = &protocols.Spec{ Name: "stream", Version: 1, MaxMsgSize: 10 * 1024 * 1024, diff --git a/swarm/network/syncer.go b/swarm/network/syncer.go index 84f61af5d6..028945384e 100644 --- a/swarm/network/syncer.go +++ b/swarm/network/syncer.go @@ -18,6 +18,7 @@ package network import ( "bytes" + "errors" "fmt" "io" @@ -45,8 +46,8 @@ func (self *DbAccess) get(key storage.Key) (*storage.Chunk, error) { } // current storage counter of chunk db -func (self *DbAccess) currentStorageIndex(po int) uint64 { - return self.db.CurrentStorageIndex(po) +func (self *DbAccess) currentBucketStorageIndex(po uint8) uint64 { + return self.db.CurrentBucketStorageIndex(po) } // iteration storage counter and proximity order @@ -59,7 +60,7 @@ func (self *DbAccess) iterator(from uint64, to uint64, po uint8, f func(storage. // * live request delivery with or without checkback // * (live/non-live historical) chunk syncing per proximity bin type OutgoingSwarmSyncer struct { - po int + po uint8 db *DbAccess sessionAt uint64 currentBatch []byte @@ -67,35 +68,37 @@ type OutgoingSwarmSyncer struct { } // NewOutgoingSwarmSyncer is contructor for OutgoingSwarmSyncer -func NewOutgoingSwarmSyncer(po int, db *DbAccess) *OutgoingSwarmSyncer { +func NewOutgoingSwarmSyncer(po uint8, db *DbAccess) (*OutgoingSwarmSyncer, error) { self := &OutgoingSwarmSyncer{ po: po, db: db, - sessionAt: db.currentStorageIndex(po), + sessionAt: db.currentBucketStorageIndex(po), } - return self + return self, nil } +const maxPO = 32 + func RegisterOutgoingSyncers(streamer *Streamer, db *DbAccess) { - for po := 0; po < maxPO; po++ { - stream := fmt.Sprintf("SYNC-%02d-live", po) + for po := uint8(0); po < maxPO; po++ { + stream := Stream(fmt.Sprintf("SYNC-%02d-live", po)) streamer.RegisterOutgoingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) { return NewOutgoingSwarmSyncer(po, db) }) - stream = fmt.Sprintf("SYNC-%02d-history", po) + stream = Stream(fmt.Sprintf("SYNC-%02d-history", po)) streamer.RegisterOutgoingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) { return NewOutgoingSwarmSyncer(po, db) }) - stream = fmt.Sprintf("SYNC-%02d-delete", po) - streamer.RegisterOutgoingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) { - return NewOutgoingProvableSwarmSyncer(po, db) - }) + // stream = Stream(fmt.Sprintf("SYNC-%02d-delete", po)) + // streamer.RegisterOutgoingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) { + // return NewOutgoingProvableSwarmSyncer(po, db) + // }) } } // GetSection retrieves the actual chunk from localstore func (self *OutgoingSwarmSyncer) GetData(key []byte) []byte { - chunk, err := self.db.get(Key(key)) + chunk, err := self.db.get(storage.Key(key)) if err != nil { return nil } @@ -111,89 +114,100 @@ func (self *OutgoingSwarmSyncer) Priority() int { } // GetBatch retrieves the next batch of hashes from the dbstore -func (self *OutgoingSwarmSyncer) SetNextBatch(from, to uint64) []byte { +func (self *OutgoingSwarmSyncer) SetNextBatch(from, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) { var batch []byte i := 0 err := self.db.iterator(from, to, self.po, func(key storage.Key, idx uint64) bool { - batch = append(batch, key[:]) + batch = append(batch, key[:]...) i++ to = idx return i < batchSize }) + if err != nil { + return nil, 0, 0, nil, err + } self.currentBatch = batch log.Debug("Swarm batch", "po", self.po, "len", i, "from", from, "to", to) - return batch, from, to, proof + return batch, from, to, nil, nil } // IncomingSwarmSyncer type IncomingSwarmSyncer struct { - po int + po uint8 priority int sessionAt uint64 nextC chan struct{} intervals []uint64 sessionRoot storage.Key sessionReader storage.LazySectionReader - retrieveC chan storage.Chunk - storeC chan storage.Chunk + retrieveC chan *storage.Chunk + storeC chan *storage.Chunk + store storage.ChunkStore + chunker storage.Chunker + currentRoot storage.Key + end, start uint64 } // NewIncomingSwarmSyncer is a contructor for provable data exchange syncer -func NewIncomingSwarmSyncer(po int, priority int, sessionAt uint64, intervals []uint64, p Peer) *IncomingSwarmSyncer { +func NewIncomingSwarmSyncer(po uint8, priority int, intervals []uint64, p Peer, store storage.ChunkStore, chunker storage.Chunker) (*IncomingSwarmSyncer, error) { self := &IncomingSwarmSyncer{ po: po, priority: priority, - sessionAt: sessionAt, - nextC: make(chan struct{}, 1), intervals: intervals, + store: store, + chunker: chunker, } - return self + return self, nil } -// NewIncomingProvableSwarmSyncer is a contructor for provable data exchange syncer -func NewIncomingProvableSwarmSyncer(po int, priority int, index uint64, sessionAt uint64, intervals []uint64, sessionRoot storage.Key, chunker *storage.PyramidChunker, store storage.ChunkStore, p Peer) *IncomingSwarmSyncer { - retrieveC := make(storage.Chunk, chunksCap) - RunChunkRequestor(p, retrieveC) - storeC := make(storage.Chunk, chunksCap) - RunChunkStorer(store, storeC) - self := &IncomingSwarmSyncer{ - po: po, - priority: priority, - sessionAt: sessionAt, - start: index, - end: index, - nextC: make(chan struct{}, 1), - intervals: intervals, - sessionRoot: sessionRoot, - sessionReader: chunker.Join(sessionRoot, retrieveC), - retrieveC: retrieveC, - storeC: storeC, - } - return self +func (s *IncomingSwarmSyncer) Priority() int { + return s.priority } +// // NewIncomingProvableSwarmSyncer is a contructor for provable data exchange syncer +// func NewIncomingProvableSwarmSyncer(po int, priority int, index uint64, sessionAt uint64, intervals []uint64, sessionRoot storage.Key, chunker *storage.PyramidChunker, store storage.ChunkStore, p Peer) *IncomingSwarmSyncer { +// retrieveC := make(storage.Chunk, chunksCap) +// RunChunkRequestor(p, retrieveC) +// storeC := make(storage.Chunk, chunksCap) +// RunChunkStorer(store, storeC) +// self := &IncomingSwarmSyncer{ +// po: po, +// priority: priority, +// sessionAt: sessionAt, +// start: index, +// end: index, +// nextC: make(chan struct{}, 1), +// intervals: intervals, +// sessionRoot: sessionRoot, +// sessionReader: chunker.Join(sessionRoot, retrieveC), +// retrieveC: retrieveC, +// storeC: storeC, +// } +// return self +// } + func RegisterIncomingSyncers(streamer *Streamer, db *DbAccess) { - for po := 0; po < maxPO; po++ { - stream := fmt.Sprintf("SYNC-%02d-live", po) - streamer.RegisterIncomingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) { - return NewIncomingSwarmSyncer(po, Mid, sessionAt, nil, p) + for po := uint8(0); po < maxPO; po++ { + stream := Stream(fmt.Sprintf("SYNC-%02d-live", po)) + streamer.RegisterIncomingStreamer(stream, func(p *StreamerPeer) (IncomingStreamer, error) { + return NewIncomingSwarmSyncer(po, High, nil, p, nil, nil) }) - stream = fmt.Sprintf("SYNC-%02d-history", po) - streamer.RegisterIncomingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) { - intervals := loadIntervals(p, po, false) - return NewIncomingSwarmSyncer(po, Mid, sessionAt, intervals, p) - }) - stream = fmt.Sprintf("SYNC-%02d-delete", po) - streamer.RegisterIncomingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) { - intervals := loadIntervals(p, po, true) - return NewIncomingSwarmSyncer(po, Mid, sessionAt, intervals, p) + stream = Stream(fmt.Sprintf("SYNC-%02d-history", po)) + streamer.RegisterIncomingStreamer(stream, func(p *StreamerPeer) (IncomingStreamer, error) { + //intervals := loadIntervals(p, po, false) + return NewIncomingSwarmSyncer(po, Mid, nil, p, nil, nil) }) + // stream = fmt.Sprintf("SYNC-%02d-delete", po) + // streamer.RegisterIncomingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) { + // intervals := loadIntervals(p, po, true) + // return NewIncomingSwarmSyncer(po, Mid, sessionAt, intervals, p) + // }) } } // NeedData func (self *IncomingSwarmSyncer) NeedData(key []byte) func() { - chunk, err := self.store.Get(hash) + chunk, err := self.store.Get(key) if err == nil { if chunk.SData == nil { // send a request instead @@ -201,57 +215,59 @@ func (self *IncomingSwarmSyncer) NeedData(key []byte) func() { } } // create request and wait until the chunk data arrives and is stored - return func() { - storedC := <-chunk.storedC - <-storedC - } + return chunk.WaitToStore } // NextBatch adjusts the indexes by inspecting the intervals -func (self *IncomingSwarmSyncer) NextBatch(from, to uint64) (uint64, uint64) { - if from >= self.sessionAt { // live syncing - self.intervals[1] = to - } else if to >= self.sessionAt { // history sync complete +func (self *IncomingSwarmSyncer) NextBatch(from uint64) (nextFrom uint64, nextTo uint64) { + if self.intervals[0] >= self.sessionAt { // live syncing + nextFrom = from + self.intervals[1] = from + } else if from >= self.sessionAt { // history sync complete self.intervals = nil - from = 0 - } else if len(intervals) > 2 && to >= self.intervals[2] { // filled a gap in the intervals - self.intervals[1:] = self.intervals[3:] - from = self.intervals[1] - if len(intervals) > 2 { - to = self.intervals[2] + } else if len(self.intervals) > 2 && from >= self.intervals[2] { // filled a gap in the intervals + self.intervals = append(self.intervals[:1], self.intervals[3:]...) + nextFrom = self.intervals[1] + if len(self.intervals) > 2 { + nextTo = self.intervals[2] + } else { + nextTo = self.sessionAt } } else { - self.intervals[1] = to + nextFrom = from + self.intervals[1] = from + nextTo = self.sessionAt } - return from, to + return nextFrom, nextTo } // -func (self *IncomingSwarmSyncer) TakeoverProof(s Stream, from uint64, hashes []byte, root storage.Key) error { +func (self *IncomingSwarmSyncer) TakeoverProof(s Stream, from uint64, hashes []byte, root storage.Key) (*TakeoverProof, error) { // for provable syncer currentRoot is non-zero length if self.chunker != nil { if from > self.sessionAt { // for live syncing currentRoot is always updated - expRoot, err := self.chunker.Append(self.currentRoot, bytes.NewReader(hashes), self.retrieveC, self.storeC) + //expRoot, err := self.chunker.Append(self.currentRoot, bytes.NewReader(hashes), self.retrieveC, self.storeC) + expRoot, err := self.chunker.Append(self.currentRoot, bytes.NewReader(hashes), self.retrieveC) if err != nil { - return err + return nil, err } if !bytes.Equal(root, expRoot) { - return fmt.Errorf("HandoverProof mismatch") + return nil, fmt.Errorf("HandoverProof mismatch") } - self.currentRoot = currentRoot + self.currentRoot = root } else { expHashes := make([]byte, len(hashes)) - n, err := self.sessionReader.ReadAt(expHashes, int64(self.end*HashSize)) + _, err := self.sessionReader.ReadAt(expHashes, int64(self.end*HashSize)) if err != nil && err != io.EOF { - return err + return nil, err } if !bytes.Equal(expHashes, hashes) { - return errInvalidProof + return nil, errors.New("invalid proof") } } - return nil + return nil, nil } - self.end += len(hashes) / HashSize + self.end += uint64(len(hashes)) / HashSize takeover := &Takeover{ Stream: s, Start: self.start, @@ -262,5 +278,5 @@ func (self *IncomingSwarmSyncer) TakeoverProof(s Stream, from uint64, hashes []b return &TakeoverProof{ Takeover: takeover, Sig: nil, - } + }, nil } diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index 507b4ab655..4a7564d34c 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -777,10 +777,8 @@ func (self *Pss) checkFwdCache(addr []byte, digest pssDigest) bool { // DPA storage handler for message cache func (self *Pss) storeMsg(msg *PssMsg) (pssDigest, error) { - swg := &sync.WaitGroup{} - wwg := &sync.WaitGroup{} buf := bytes.NewReader(msg.serialize()) - key, err := self.dpa.Store(buf, int64(buf.Len()), swg, wwg) + key, _, err := self.dpa.Store(buf, int64(buf.Len())) if err != nil { log.Warn("Could not store in swarm", "err", err) return pssDigest{}, err diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index 98cd6e75ea..f049d39f7e 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -118,23 +118,19 @@ func (self *TreeChunker) decrementWorkerCount() { self.workerCount -= 1 } -func (self *TreeChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, swg, wwg *sync.WaitGroup) (Key, error) { +func (self *TreeChunker) Split(data io.Reader, size int64, chunkC chan *Chunk) (k Key, wait func(), err error) { if self.chunkSize <= 0 { panic("chunker must be initialised") } jobC := make(chan *hashJob, 2*ChunkProcessors) wg := &sync.WaitGroup{} + storeWg := &sync.WaitGroup{} errC := make(chan error) quitC := make(chan bool) - // wwg = workers waitgroup keeps track of hashworkers spawned by this split call - if wwg != nil { - wwg.Add(1) - } - self.incrementWorkerCount() - go self.hashWorker(jobC, chunkC, errC, quitC, swg, wwg) + go self.hashWorker(jobC, chunkC, errC, quitC, storeWg) depth := 0 treeSize := self.chunkSize @@ -149,16 +145,12 @@ func (self *TreeChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, s // this waitgroup member is released after the root hash is calculated wg.Add(1) //launch actual recursive function passing the waitgroups - go self.split(depth, treeSize/self.branches, key, data, size, jobC, chunkC, errC, quitC, wg, swg, wwg) + go self.split(depth, treeSize/self.branches, key, data, size, jobC, chunkC, errC, quitC, wg, storeWg) // closes internal error channel if all subprocesses in the workgroup finished go func() { // waiting for all threads to finish wg.Wait() - // if storage waitgroup is non-nil, we wait for storage to finish too - if swg != nil { - swg.Wait() - } close(errC) }() @@ -166,16 +158,16 @@ func (self *TreeChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, s select { case err := <-errC: if err != nil { - return nil, err + return nil, nil, err } case <-time.NewTimer(splitTimeout).C: - return nil, errOperationTimedOut + return nil, nil, errOperationTimedOut } - return key, nil + return key, storeWg.Wait, nil } -func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reader, size int64, jobC chan *hashJob, chunkC chan *Chunk, errC chan error, quitC chan bool, parentWg, swg, wwg *sync.WaitGroup) { +func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reader, size int64, jobC chan *hashJob, chunkC chan *Chunk, errC chan error, quitC chan bool, parentWg, storeWg *sync.WaitGroup) { // @@ -225,7 +217,7 @@ func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reade subTreeKey := chunk[8+i*self.hashSize : 8+(i+1)*self.hashSize] childrenWg.Add(1) - self.split(depth-1, treeSize/self.branches, subTreeKey, data, secSize, jobC, chunkC, errC, quitC, childrenWg, swg, wwg) + self.split(depth-1, treeSize/self.branches, subTreeKey, data, secSize, jobC, chunkC, errC, quitC, childrenWg, storeWg) i++ pos += treeSize @@ -237,11 +229,8 @@ func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reade worker := self.getWorkerCount() if int64(len(jobC)) > worker && worker < ChunkProcessors { - if wwg != nil { - wwg.Add(1) - } self.incrementWorkerCount() - go self.hashWorker(jobC, chunkC, errC, quitC, swg, wwg) + go self.hashWorker(jobC, chunkC, errC, quitC, storeWg) } select { @@ -250,13 +239,10 @@ func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reade } } -func (self *TreeChunker) hashWorker(jobC chan *hashJob, chunkC chan *Chunk, errC chan error, quitC chan bool, swg, wwg *sync.WaitGroup) { +func (self *TreeChunker) hashWorker(jobC chan *hashJob, chunkC chan *Chunk, errC chan error, quitC chan bool, storeWg *sync.WaitGroup) { defer self.decrementWorkerCount() hasher := self.hashFunc() - if wwg != nil { - defer wwg.Done() - } for { select { @@ -265,7 +251,7 @@ func (self *TreeChunker) hashWorker(jobC chan *hashJob, chunkC chan *Chunk, errC return } // now we got the hashes in the chunk, then hash the chunks - self.hashChunk(hasher, job, chunkC, swg) + self.hashChunk(hasher, job, chunkC, storeWg) case <-quitC: return } @@ -275,34 +261,31 @@ func (self *TreeChunker) hashWorker(jobC chan *hashJob, chunkC chan *Chunk, errC // The treeChunkers own Hash hashes together // - the size (of the subtree encoded in the Chunk) // - the Chunk, ie. the contents read from the input reader -func (self *TreeChunker) hashChunk(hasher SwarmHash, job *hashJob, chunkC chan *Chunk, swg *sync.WaitGroup) { +func (self *TreeChunker) hashChunk(hasher SwarmHash, job *hashJob, chunkC chan *Chunk, storeWg *sync.WaitGroup) { hasher.ResetWithLength(job.chunk[:8]) // 8 bytes of length hasher.Write(job.chunk[8:]) // minus 8 []byte length h := hasher.Sum(nil) - newChunk := &Chunk{ - Key: h, - SData: job.chunk, - Size: job.size, - wg: swg, - } + newChunk := NewChunk(h, nil) + newChunk.SData = job.chunk + newChunk.Size = job.size // report hash of this chunk one level up (keys corresponds to the proper subslice of the parent chunk) copy(job.key, h) // send off new chunk to storage - if chunkC != nil { - if swg != nil { - swg.Add(1) - } - } job.parentWg.Done() if chunkC != nil { chunkC <- newChunk + storeWg.Add(1) + go func() { + defer storeWg.Done() + <-newChunk.dbStored + }() } } -func (self *TreeChunker) Append(key Key, data io.Reader, chunkC chan *Chunk, swg, wwg *sync.WaitGroup) (Key, error) { +func (self *TreeChunker) Append(key Key, data io.Reader, chunkC chan *Chunk) (Key, error) { return nil, errAppendOppNotSuported } @@ -456,10 +439,8 @@ func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, tr // block until they time out or arrive // abort if quitC is readable func retrieve(key Key, chunkC chan *Chunk, quitC chan bool) *Chunk { - chunk := &Chunk{ - Key: key, - C: make(chan bool), // close channel to signal data delivery - } + chunk := NewChunk(key, nil) + chunk.C = make(chan bool) // submit chunk for retrieval select { case chunkC <- chunk: // submit retrieval request, someone should be listening on the other side (or we will time out globally) diff --git a/swarm/storage/common_test.go b/swarm/storage/common_test.go index 6fd66a03d9..bdc4814d51 100644 --- a/swarm/storage/common_test.go +++ b/swarm/storage/common_test.go @@ -81,7 +81,14 @@ func mput(store ChunkStore, processors int, n int, f func(i int) *Chunk) (hs []K go func() { defer wg.Done() for chunk := range c { - store.Put(chunk) + wg.Add(1) + chunk := chunk + go func() { + defer wg.Done() + + store.Put(chunk) + <-chunk.dbStored + }() } }() } diff --git a/swarm/storage/dbstore.go b/swarm/storage/dbstore.go index c1bef29823..01a6b79f7f 100644 --- a/swarm/storage/dbstore.go +++ b/swarm/storage/dbstore.go @@ -23,9 +23,13 @@ package storage import ( + "archive/tar" "bytes" "encoding/binary" + "encoding/hex" "fmt" + "io" + "io/ioutil" "sync" "github.com/ethereum/go-ethereum/log" @@ -74,7 +78,12 @@ type DbStore struct { hashfunc SwarmHasher po func(Key) uint8 - lock sync.Mutex + + batchC chan bool + quit chan struct{} + batchesC chan struct{} + batch *leveldb.Batch + lock sync.RWMutex trusted bool // if hash integity check is to be performed (for testing only) } @@ -84,6 +93,13 @@ type DbStore struct { func NewDbStore(path string, hash SwarmHasher, capacity uint64, po func(Key) uint8) (s *DbStore, err error) { s = new(DbStore) s.hashfunc = hash + + s.batchC = make(chan bool) + s.quit = make(chan struct{}) + s.batchesC = make(chan struct{}, 1) + go s.writeBatches() + s.batch = new(leveldb.Batch) + s.db, err = NewLDBDatabase(path) if err != nil { return nil, err @@ -96,8 +112,6 @@ func NewDbStore(path string, hash SwarmHasher, capacity uint64, po func(Key) uin s.gcStartPos[0] = kpIndex s.gcArray = make([]*gcItem, gcArraySize) - data, _ := s.db.Get(keyEntryCnt) - s.entryCnt = BytesToU64(data) s.bucketCnt = make([]uint64, 0x100) for i := 0; i < 0x100; i++ { k := make([]byte, 2) @@ -105,18 +119,17 @@ func NewDbStore(path string, hash SwarmHasher, capacity uint64, po func(Key) uin k[1] = byte(uint8(i)) cnt, _ := s.db.Get(k) s.bucketCnt[i] = BytesToU64(cnt) + s.bucketCnt[i]++ } + data, _ := s.db.Get(keyEntryCnt) + s.entryCnt = BytesToU64(data) + s.entryCnt++ data, _ = s.db.Get(keyAccessCnt) - //s.accessCnt = BytesToU64(data) - if len(data) == 8 { - s.accessCnt = binary.LittleEndian.Uint64(data) - s.accessCnt++ - } + s.accessCnt = BytesToU64(data) + s.accessCnt++ data, _ = s.db.Get(keyDataIdx) - if len(data) == 8 { - s.dataIdx = BytesToU64(data) - s.dataIdx++ - } + s.dataIdx = BytesToU64(data) + s.dataIdx++ s.gcPos, _ = s.db.Get(keyGCPos) if s.gcPos == nil { @@ -295,6 +308,92 @@ func (s *DbStore) collectGarbage(ratio float32) { s.db.Put(keyGCPos, s.gcPos) } +// Export writes all chunks from the store to a tar archive, returning the +// number of chunks written. +func (s *DbStore) Export(out io.Writer) (int64, error) { + tw := tar.NewWriter(out) + defer tw.Close() + + it := s.db.NewIterator() + defer it.Release() + var count int64 + for ok := it.Seek([]byte{kpIndex}); ok; ok = it.Next() { + key := it.Key() + if (key == nil) || (key[0] != kpIndex) { + break + } + + var index dpaDBIndex + decodeIndex(it.Value(), &index) + + hash := key[1:] + + data, err := s.db.Get(getDataKey(index.Idx, s.po(hash))) + if err != nil { + log.Warn(fmt.Sprintf("Chunk %x found but could not be accessed: %v", key[:], err)) + continue + } + + hdr := &tar.Header{ + Name: hex.EncodeToString(hash), + Mode: 0644, + Size: int64(len(data)), + } + if err := tw.WriteHeader(hdr); err != nil { + return count, err + } + if _, err := tw.Write(data); err != nil { + return count, err + } + count++ + } + + return count, nil +} + +// of chunks read. +func (s *DbStore) Import(in io.Reader) (int64, error) { + tr := tar.NewReader(in) + + var count int64 + var wg sync.WaitGroup + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } else if err != nil { + return count, err + } + + if len(hdr.Name) != 64 { + log.Warn("ignoring non-chunk file", "name", hdr.Name) + continue + } + + key, err := hex.DecodeString(hdr.Name) + if err != nil { + log.Warn("ignoring invalid chunk file", "name", hdr.Name, "err", err) + continue + } + + data, err := ioutil.ReadAll(tr) + if err != nil { + return count, err + } + chunk := NewChunk(key, nil) + chunk.SData = data + s.Put(chunk) + wg.Add(1) + go func() { + defer wg.Done() + <-chunk.dbStored + }() + count++ + } + wg.Wait() + return count, nil +} + func (s *DbStore) Cleanup() { //Iterates over the database and checks that there are no faulty chunks it := s.db.NewIterator() @@ -334,26 +433,6 @@ func (s *DbStore) Cleanup() { log.Warn(fmt.Sprintf("Found %v errors out of %v entries", errorsFound, total)) } -func (s *DbStore) Dump() { - //Iterates over the database and checks that there are no faulty chunks - it := s.db.NewIterator() - startPosition := []byte{kpIndex} - it.Seek(startPosition) - var key []byte - var total int - for it.Valid() { - key = it.Key() - if (key == nil) || (key[0] != kpIndex) { - break - } - total++ - fmt.Printf("%x\n", key[1:]) - it.Next() - } - it.Release() - log.Warn(fmt.Sprintf("logged %v chunks", total)) -} - func (s *DbStore) ReIndex() { //Iterates over the database and checks that there are no faulty chunks it := s.db.NewIterator() @@ -411,6 +490,13 @@ func (s *DbStore) delete(idx uint64, idxKey []byte, po uint8) { s.db.Write(batch) } +func (s *DbStore) CurrentBucketStorageIndex(po uint8) uint64 { + s.lock.RLock() + defer s.lock.RUnlock() + + return s.bucketCnt[po] +} + func (s *DbStore) Size() uint64 { s.lock.Lock() defer s.lock.Unlock() @@ -418,11 +504,64 @@ func (s *DbStore) Size() uint64 { } func (s *DbStore) CurrentStorageIndex() uint64 { - s.lock.Lock() - defer s.lock.Unlock() + s.lock.RLock() + defer s.lock.RUnlock() return s.dataIdx } +// TODO: remove the old code for Put +// func (s *DbStore) Put(chunk *Chunk) { +// s.lock.Lock() +// defer s.lock.Unlock() + +// ikey := getIndexKey(chunk.Key) +// var index dpaDBIndex + +// if s.tryAccessIdx(ikey, &index) { +// if chunk.dbStored != nil { +// close(chunk.dbStored) +// } +// log.Trace(fmt.Sprintf("Storing to DB: chunk already exists, only update access")) +// return // already exists, only update access +// } + +// data := encodeData(chunk) + +// if s.entryCnt >= s.capacity { +// s.collectGarbage(gcArrayFreeRatio) +// } + +// po := s.po(chunk.Key) +// t_datakey := getDataKey(s.dataIdx, po) +// s.batch.Put(t_datakey, data) + +// index.Idx = s.dataIdx +// s.updateIndexAccess(&index) + +// idata := encodeIndex(&index) +// s.batch.Put(ikey, idata) + +// s.batch.Put(keyEntryCnt, U64ToBytes(s.entryCnt)) +// s.entryCnt++ +// s.batch.Put(keyDataIdx, U64ToBytes(s.dataIdx)) +// s.dataIdx++ +// accesscnt := make([]byte, 8) +// binary.LittleEndian.PutUint64(accesscnt, s.accessCnt) +// s.batch.Put(keyAccessCnt, accesscnt) +// s.accessCnt++ + +// s.bucketCnt[po]++ +// cntKey := make([]byte, 2) +// cntKey[0] = keyDistanceCnt +// cntKey[1] = po +// s.batch.Put(cntKey, U64ToBytes(s.bucketCnt[po])) + +// if chunk.dbStored != nil { +// close(chunk.dbStored) +// } +// log.Trace(fmt.Sprintf("DbStore.Put: %v. db storage counter: %v ", chunk.Key.Log(), s.dataIdx)) +// } + func (s *DbStore) Put(chunk *Chunk) { s.lock.Lock() defer s.lock.Unlock() @@ -430,54 +569,84 @@ func (s *DbStore) Put(chunk *Chunk) { ikey := getIndexKey(chunk.Key) var index dpaDBIndex - if s.tryAccessIdx(ikey, &index) { - if chunk.dbStored != nil { - close(chunk.dbStored) - } - log.Trace(fmt.Sprintf("Storing to DB: chunk already exists, only update access")) - return // already exists, only update access - } - - data := encodeData(chunk) - - if s.entryCnt >= s.capacity { - s.collectGarbage(gcArrayFreeRatio) - } - - batch := new(leveldb.Batch) - po := s.po(chunk.Key) - t_datakey := getDataKey(s.dataIdx, po) - batch.Put(t_datakey, data) - index.Idx = s.dataIdx - s.updateIndexAccess(&index) - - idata := encodeIndex(&index) - batch.Put(ikey, idata) - - batch.Put(keyEntryCnt, U64ToBytes(s.entryCnt)) - s.entryCnt++ - batch.Put(keyDataIdx, U64ToBytes(s.dataIdx)) - s.dataIdx++ - accesscnt := make([]byte, 8) - binary.LittleEndian.PutUint64(accesscnt, s.accessCnt) - batch.Put(keyAccessCnt, accesscnt) + idata, err := s.db.Get(ikey) + if err != nil { + s.doPut(chunk, ikey, &index, po) + } else { + log.Trace(fmt.Sprintf("DbStore: chunk already exists, only update access")) + decodeIndex(idata, &index) + close(chunk.dbStored) + } + index.Access = s.accessCnt s.accessCnt++ + idata = encodeIndex(&index) + s.batch.Put(ikey, idata) + select { + case <-s.quit: + case s.batchesC <- struct{}{}: + default: + } +} + +// force putting into db, does not check access index +func (s *DbStore) doPut(chunk *Chunk, ikey []byte, index *dpaDBIndex, po uint8) { + data := encodeData(chunk) + s.batch.Put(getDataKey(s.dataIdx, po), data) + index.Idx = s.dataIdx + s.entryCnt++ + s.dataIdx++ s.bucketCnt[po]++ cntKey := make([]byte, 2) cntKey[0] = keyDistanceCnt cntKey[1] = po - batch.Put(cntKey, U64ToBytes(s.bucketCnt[po])) + s.batch.Put(cntKey, U64ToBytes(s.bucketCnt[po])) - s.db.Write(batch) - if chunk.dbStored != nil { + batchC := s.batchC + go func() { + <-batchC close(chunk.dbStored) - } + }() + log.Trace(fmt.Sprintf("DbStore.Put: %v. db storage counter: %v ", chunk.Key.Log(), s.dataIdx)) } +func (s *DbStore) writeBatches() { + for range s.batchesC { + s.lock.Lock() + b := s.batch + e := s.entryCnt + d := s.dataIdx + a := s.accessCnt + c := s.batchC + s.batchC = make(chan bool) + s.batch = new(leveldb.Batch) + s.lock.Unlock() + log.Trace(fmt.Sprintf("DbStore: spawn batch write (%d chunks) ", b.Len())) + s.writeBatch(b, e, d, a) + close(c) + if e >= s.capacity { + log.Trace(fmt.Sprintf("DbStore: collecting garbage...(%d chunks)", e)) + s.collectGarbage(gcArrayFreeRatio) + } + } + log.Trace(fmt.Sprintf("DbStore: quit batch write loop")) +} + +// must be called non concurrently +func (s *DbStore) writeBatch(b *leveldb.Batch, entryCnt, dataIdx, accessCnt uint64) { + b.Put(keyEntryCnt, U64ToBytes(entryCnt)) + b.Put(keyDataIdx, U64ToBytes(dataIdx)) + b.Put(keyAccessCnt, U64ToBytes(accessCnt)) + l := s.batch.Len() + if err := s.db.Write(b); err != nil { + log.Error(fmt.Sprintf("unable to write batch: %v", err)) + } + log.Trace(fmt.Sprintf("DbStore: batch write (%d chunks) complete", l)) +} + // try to find index; if found, update access cnt and return true func (s *DbStore) tryAccessIdx(ikey []byte, index *dpaDBIndex) bool { idata, err := s.db.Get(ikey) @@ -485,20 +654,11 @@ func (s *DbStore) tryAccessIdx(ikey []byte, index *dpaDBIndex) bool { return false } decodeIndex(idata, index) - - batch := new(leveldb.Batch) - - accesscnt := make([]byte, 8) - binary.LittleEndian.PutUint64(accesscnt, s.accessCnt) - batch.Put(keyAccessCnt, accesscnt) - + s.batch.Put(keyAccessCnt, U64ToBytes(s.accessCnt)) s.accessCnt++ - s.updateIndexAccess(index) + index.Access = s.accessCnt idata = encodeIndex(index) - batch.Put(ikey, idata) - - s.db.Write(batch) - + s.batch.Put(ikey, idata) return true } @@ -537,9 +697,7 @@ func (s *DbStore) get(key Key) (chunk *Chunk, err error) { } } - chunk = &Chunk{ - Key: key, - } + chunk = NewChunk(key, nil) decodeData(data, chunk) } else { err = notFound diff --git a/swarm/storage/dbstore_test.go b/swarm/storage/dbstore_test.go index 27bab975a6..ea250bfb07 100644 --- a/swarm/storage/dbstore_test.go +++ b/swarm/storage/dbstore_test.go @@ -112,7 +112,11 @@ func TestIterator(t *testing.T) { var poc uint chunkkeys := NewKeyCollection(chunkcount) chunkkeys_results := NewKeyCollection(chunkcount) - chunks := make([]Chunk, chunkcount) + var chunks []*Chunk + + for i := 0; i < chunkcount; i++ { + chunks = append(chunks, NewChunk(nil, nil)) + } db, err := newTestDbStore() if err != nil { @@ -123,7 +127,7 @@ func TestIterator(t *testing.T) { FakeChunk(getDefaultChunkSize(), chunkcount, chunks) for i = 0; i < len(chunks); i++ { - db.Put(&chunks[i]) + db.Put(chunks[i]) chunkkeys[i] = chunks[i].Key } diff --git a/swarm/storage/dpa.go b/swarm/storage/dpa.go index b8f7f5fd8f..354ff32c15 100644 --- a/swarm/storage/dpa.go +++ b/swarm/storage/dpa.go @@ -97,8 +97,8 @@ func (self *DPA) Retrieve(key Key) LazySectionReader { // Public API. Main entry point for document storage directly. Used by the // FS-aware API and httpaccess -func (self *DPA) Store(data io.Reader, size int64, swg *sync.WaitGroup, wwg *sync.WaitGroup) (key Key, err error) { - return self.Chunker.Split(data, size, self.storeC, swg, wwg) +func (self *DPA) Store(data io.Reader, size int64) (key Key, wait func(), err error) { + return self.Chunker.Split(data, size, self.storeC) } func (self *DPA) Start() { @@ -163,12 +163,8 @@ func (self *DPA) storeLoop() { } func (self *DPA) storeWorker() { - for chunk := range self.storeC { self.Put(chunk) - if chunk.wg != nil { - chunk.wg.Done() - } select { case <-self.quitC: return diff --git a/swarm/storage/dpa_test.go b/swarm/storage/dpa_test.go index 3bccd82d54..391418674b 100644 --- a/swarm/storage/dpa_test.go +++ b/swarm/storage/dpa_test.go @@ -21,7 +21,6 @@ import ( "io" "io/ioutil" "os" - "sync" "testing" ) @@ -50,12 +49,11 @@ func TestDPArandom(t *testing.T) { defer os.RemoveAll("/tmp/bzz") reader, slice := testDataReaderAndSlice(testDataSize) - wg := &sync.WaitGroup{} - key, err := dpa.Store(reader, testDataSize, wg, nil) + key, wait, err := dpa.Store(reader, testDataSize) if err != nil { t.Errorf("Store error: %v", err) } - wg.Wait() + wait() resultReader := dpa.Retrieve(key) resultSlice := make([]byte, len(slice)) n, err := resultReader.ReadAt(resultSlice, 0) @@ -106,12 +104,11 @@ func TestDPA_capacity(t *testing.T) { } dpa.Start() reader, slice := testDataReaderAndSlice(testDataSize) - wg := &sync.WaitGroup{} - key, err := dpa.Store(reader, testDataSize, wg, nil) + key, wait, err := dpa.Store(reader, testDataSize) if err != nil { t.Errorf("Store error: %v", err) } - wg.Wait() + wait() resultReader := dpa.Retrieve(key) resultSlice := make([]byte, len(slice)) n, err := resultReader.ReadAt(resultSlice, 0) diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index 2ed9fb305a..ddc10934d8 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -42,7 +42,6 @@ func NewLocalStore(hash SwarmHasher, params *StoreParams, basekey []byte) (*Loca // LocalStore is itself a chunk store // unsafe, in that the data is not integrity checked func (self *LocalStore) Put(chunk *Chunk) { - chunk.dbStored = make(chan bool) self.memStore.Put(chunk) if chunk.wg != nil { chunk.wg.Add(1) diff --git a/swarm/storage/memstore.go b/swarm/storage/memstore.go index 3cb25ac625..fed1ae0094 100644 --- a/swarm/storage/memstore.go +++ b/swarm/storage/memstore.go @@ -280,13 +280,9 @@ func (s *MemStore) removeOldest() { } - if node.entry.dbStored != nil { - log.Trace(fmt.Sprintf("Memstore Clean: Waiting for chunk %v to be saved", node.entry.Key.Log())) - <-node.entry.dbStored - log.Trace(fmt.Sprintf("Memstore Clean: Chunk %v saved to DBStore. Ready to clear from mem.", node.entry.Key.Log())) - } else { - log.Trace(fmt.Sprintf("Memstore Clean: Chunk %v already in DB. Ready to delete.", node.entry.Key.Log())) - } + log.Trace(fmt.Sprintf("Memstore Clean: Waiting for chunk %v to be saved", node.entry.Key.Log())) + <-node.entry.dbStored + log.Trace(fmt.Sprintf("Memstore Clean: Chunk %v saved to DBStore. Ready to clear from mem.", node.entry.Key.Log())) if node.entry.SData != nil { node.entry = nil diff --git a/swarm/storage/netstore.go b/swarm/storage/netstore.go index 5d4f17deb1..2afba75811 100644 --- a/swarm/storage/netstore.go +++ b/swarm/storage/netstore.go @@ -131,7 +131,7 @@ func (self *NetStore) Get(key Key) (*Chunk, error) { } // no data and no request status log.Trace(fmt.Sprintf("NetStore.Get: %v not found locally. open new request", key)) - chunk = NewChunk(key, newRequestStatus(key)) + chunk = NewChunk(key, NewRequestStatus(key)) self.localStore.memStore.Put(chunk) go self.cloud.Retrieve(chunk) return chunk, nil diff --git a/swarm/storage/pyramid.go b/swarm/storage/pyramid.go index 19d493405a..2ee3d5b58a 100644 --- a/swarm/storage/pyramid.go +++ b/swarm/storage/pyramid.go @@ -271,12 +271,10 @@ func (self *PyramidChunker) processChunk(id int64, hasher SwarmHash, job *chunkJ hasher.Write(job.chunk[8:]) // minus 8 []byte length h := hasher.Sum(nil) - newChunk := &Chunk{ - Key: h, - SData: job.chunk, - Size: job.size, - wg: swg, - } + newChunk := NewChunk(h, nil) + newChunk.SData = job.chunk + newChunk.Size = job.size + newChunk.wg = swg // report hash of this chunk one level up (keys corresponds to the proper subslice of the parent chunk) copy(job.key, h) diff --git a/swarm/storage/types.go b/swarm/storage/types.go index e2c111f7b1..3cfe8a6f11 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -180,7 +180,7 @@ type RequestStatus struct { Requesters map[uint64][]interface{} } -func newRequestStatus(key Key) *RequestStatus { +func NewRequestStatus(key Key) *RequestStatus { return &RequestStatus{ Key: key, Requesters: make(map[uint64][]interface{}), @@ -205,10 +205,14 @@ type Chunk struct { } func NewChunk(key Key, rs *RequestStatus) *Chunk { - return &Chunk{Key: key, Req: rs} + return &Chunk{Key: key, Req: rs, dbStored: make(chan bool)} } -func FakeChunk(size int64, count int, chunks []Chunk) int { +func (c *Chunk) WaitToStore() { + <-c.dbStored +} + +func FakeChunk(size int64, count int, chunks []*Chunk) int { var i int hasher := MakeHashFunc(SHA3Hash)() chunksize := getDefaultChunkSize() @@ -269,14 +273,14 @@ type Splitter interface { The caller gets returned an error channel, if an error is encountered during splitting, it is fed to errC error channel. A closed error signals process completion at which point the key can be considered final if there were no errors. */ - Split(io.Reader, int64, chan *Chunk, *sync.WaitGroup, *sync.WaitGroup) (Key, error) + Split(io.Reader, int64, chan *Chunk) (Key, func(), error) /* This is the first step in making files mutable (not chunks).. Append allows adding more data chunks to the end of the already existsing file. The key for the root chunk is supplied to load the respective tree. Rest of the parameters behave like Split. */ - Append(Key, io.Reader, chan *Chunk, *sync.WaitGroup, *sync.WaitGroup) (Key, error) + Append(Key, io.Reader, chan *Chunk) (Key, error) } type Joiner interface { diff --git a/swarm/swarm.go b/swarm/swarm.go index 3061d07a8a..b2349c0fc4 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -97,7 +97,7 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e log.Debug(fmt.Sprintf("Setting up Swarm service components")) hash := storage.MakeHashFunc(config.ChunkerParams.Hash) - self.lstore, err = storage.NewLocalStore(hash, config.StoreParams) + self.lstore, err = storage.NewLocalStore(hash, config.StoreParams, common.Hex2Bytes(config.BzzKey)) if err != nil { return } @@ -346,32 +346,6 @@ func (self *Swarm) SetChequebook(ctx context.Context) error { return nil } -// Local swarm without netStore -func NewLocalSwarm(datadir, port string) (self *Swarm, err error) { - - prvKey, err := crypto.GenerateKey() - if err != nil { - return - } - - config := api.NewConfig() - config.Path = datadir - config.Port = port - config.Init(prvKey) - - dpa, err := storage.NewLocalDPA(datadir) - if err != nil { - return - } - - self = &Swarm{ - api: api.NewApi(dpa, nil), - config: config, - } - - return -} - // serialisable info about swarm type Info struct { *api.Config From f973b68d65db2758cd7bc8adff3c74a9d24f029a Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Wed, 3 Jan 2018 18:37:38 +0100 Subject: [PATCH 008/210] swarm/storage: attempt to fix storage tests --- swarm/storage/chunker.go | 4 +- swarm/storage/chunker_test.go | 70 ++++++++++++++++------------------- swarm/storage/pyramid.go | 61 +++++++++++------------------- swarm/storage/types.go | 2 +- 4 files changed, 56 insertions(+), 81 deletions(-) diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index f049d39f7e..dde975c1a0 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -285,8 +285,8 @@ func (self *TreeChunker) hashChunk(hasher SwarmHash, job *hashJob, chunkC chan * } } -func (self *TreeChunker) Append(key Key, data io.Reader, chunkC chan *Chunk) (Key, error) { - return nil, errAppendOppNotSuported +func (self *TreeChunker) Append(key Key, data io.Reader, chunkC chan *Chunk) (Key, func(), error) { + return nil, nil, errAppendOppNotSuported } // LazyChunkReader implements LazySectionReader diff --git a/swarm/storage/chunker_test.go b/swarm/storage/chunker_test.go index 6b828970b6..abfcbbed9f 100644 --- a/swarm/storage/chunker_test.go +++ b/swarm/storage/chunker_test.go @@ -23,7 +23,6 @@ import ( "errors" "fmt" "io" - "sync" "testing" "time" @@ -45,7 +44,7 @@ type chunkerTester struct { t test } -func (self *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, chunkC chan *Chunk, swg *sync.WaitGroup, expectedError error) (key Key, err error) { +func (self *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, chunkC chan *Chunk, expectedError error) (key Key, wait func(), err error) { // reset self.chunks = make(map[string]*Chunk) @@ -66,30 +65,27 @@ func (self *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, c case chunk := <-chunkC: // self.chunks = append(self.chunks, chunk) self.chunks[chunk.Key.String()] = chunk - if chunk.wg != nil { - chunk.wg.Done() - } + close(chunk.dbStored) } } }() } - key, err = chunker.Split(data, size, chunkC, swg, nil) + key, wait, err = chunker.Split(data, size, chunkC) if err != nil && expectedError == nil { err = fmt.Errorf("Split error: %v", err) } if chunkC != nil { - if swg != nil { - swg.Wait() - } close(quitC) + } else { + wait = func() {} } - return key, err + return key, wait, err } -func (self *chunkerTester) Append(chunker Splitter, rootKey Key, data io.Reader, chunkC chan *Chunk, swg *sync.WaitGroup, expectedError error) (key Key, err error) { +func (self *chunkerTester) Append(chunker Splitter, rootKey Key, data io.Reader, chunkC chan *Chunk, expectedError error) (key Key, wait func(), err error) { quitC := make(chan bool) timeout := time.After(60 * time.Second) if chunkC != nil { @@ -106,13 +102,11 @@ func (self *chunkerTester) Append(chunker Splitter, rootKey Key, data io.Reader, if !success { // Requesting data self.chunks[chunk.Key.String()] = chunk - if chunk.wg != nil { - chunk.wg.Done() - } } else { // getting data chunk.SData = stored.SData chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) + close(chunk.dbStored) close(chunk.C) } } @@ -121,25 +115,22 @@ func (self *chunkerTester) Append(chunker Splitter, rootKey Key, data io.Reader, }() } - key, err = chunker.Append(rootKey, data, chunkC, swg, nil) + key, wait, err = chunker.Append(rootKey, data, chunkC) if err != nil && expectedError == nil { err = fmt.Errorf("Append error: %v", err) } if chunkC != nil { - if swg != nil { - swg.Wait() - } close(quitC) + } else { + wait = func() {} } - return key, err + return key, wait, err } func (self *chunkerTester) Join(chunker Chunker, key Key, c int, chunkC chan *Chunk, quitC chan bool) LazySectionReader { // reset but not the chunks - reader := chunker.Join(key, chunkC) - timeout := time.After(600 * time.Second) i := 0 go func() error { @@ -164,6 +155,8 @@ func (self *chunkerTester) Join(chunker Chunker, key Key, c int, chunkC chan *Ch } } }() + + reader := chunker.Join(key, chunkC) return reader } @@ -181,10 +174,9 @@ func testRandomBrokenData(splitter Splitter, n int, tester *chunkerTester) { brokendata = brokenLimitReader(data, n, n/2) chunkC := make(chan *Chunk, 1000) - swg := &sync.WaitGroup{} expectedError := fmt.Errorf("Broken reader") - key, err := tester.Split(splitter, brokendata, int64(n), chunkC, swg, expectedError) + key, _, err := tester.Split(splitter, brokendata, int64(n), chunkC, expectedError) if err == nil || err.Error() != expectedError.Error() { tester.t.Fatalf("Not receiving the correct error! Expected %v, received %v", expectedError, err) } @@ -205,9 +197,8 @@ func testRandomData(splitter Splitter, n int, tester *chunkerTester) Key { } chunkC := make(chan *Chunk, 1000) - swg := &sync.WaitGroup{} - key, err := tester.Split(splitter, data, int64(n), chunkC, swg, nil) + key, _, err := tester.Split(splitter, data, int64(n), chunkC, nil) if err != nil { tester.t.Fatalf(err.Error()) } @@ -248,12 +239,12 @@ func testRandomDataAppend(splitter Splitter, n, m int, tester *chunkerTester) { } chunkC := make(chan *Chunk, 1000) - swg := &sync.WaitGroup{} - key, err := tester.Split(splitter, data, int64(n), chunkC, swg, nil) + key, wait, err := tester.Split(splitter, data, int64(n), chunkC, nil) if err != nil { tester.t.Fatalf(err.Error()) } + wait() tester.t.Logf(" Key = %v\n", key) //create a append data stream @@ -267,12 +258,12 @@ func testRandomDataAppend(splitter Splitter, n, m int, tester *chunkerTester) { } chunkC = make(chan *Chunk, 1000) - swg = &sync.WaitGroup{} - newKey, err := tester.Append(splitter, key, appendData, chunkC, swg, nil) + newKey, wait, err := tester.Append(splitter, key, appendData, chunkC, nil) if err != nil { tester.t.Fatalf(err.Error()) } + wait() tester.t.Logf(" NewKey = %v\n", newKey) chunkC = make(chan *Chunk, 1000) @@ -324,6 +315,8 @@ func TestSha3ForCorrectness(t *testing.T) { } func TestDataAppend(t *testing.T) { + t.Skip("Skip until append chunks are fixed") + sizes := []int{1, 1, 1, 4095, 4096, 4097, 1, 1, 1, 123456, 2345678, 2345678} appendSizes := []int{4095, 4096, 4097, 1, 1, 1, 8191, 8192, 8193, 9000, 3000, 5000} @@ -388,12 +381,12 @@ func benchmarkJoin(n int, t *testing.B) { data := testDataReader(n) chunkC := make(chan *Chunk, 1000) - swg := &sync.WaitGroup{} - key, err := tester.Split(chunker, data, int64(n), chunkC, swg, nil) + key, wait, err := tester.Split(chunker, data, int64(n), chunkC, nil) if err != nil { tester.t.Fatalf(err.Error()) } + wait() chunkC = make(chan *Chunk, 1000) quitC := make(chan bool) reader := tester.Join(chunker, key, i, chunkC, quitC) @@ -409,7 +402,7 @@ func benchmarkSplitTreeSHA3(n int, t *testing.B) { chunker := NewTreeChunker(NewChunkerParams()) tester := &chunkerTester{t: t} data := testDataReader(n) - _, err := tester.Split(chunker, data, int64(n), nil, nil, nil) + _, _, err := tester.Split(chunker, data, int64(n), nil, nil) if err != nil { tester.t.Fatalf(err.Error()) } @@ -424,7 +417,7 @@ func benchmarkSplitTreeBMT(n int, t *testing.B) { chunker := NewTreeChunker(cp) tester := &chunkerTester{t: t} data := testDataReader(n) - _, err := tester.Split(chunker, data, int64(n), nil, nil, nil) + _, _, err := tester.Split(chunker, data, int64(n), nil, nil) if err != nil { tester.t.Fatalf(err.Error()) } @@ -437,10 +430,11 @@ func benchmarkSplitPyramidSHA3(n int, t *testing.B) { splitter := NewPyramidChunker(NewChunkerParams()) tester := &chunkerTester{t: t} data := testDataReader(n) - _, err := tester.Split(splitter, data, int64(n), nil, nil, nil) + _, _, err := tester.Split(splitter, data, int64(n), nil, nil) if err != nil { tester.t.Fatalf(err.Error()) } + } } @@ -452,7 +446,7 @@ func benchmarkSplitPyramidBMT(n int, t *testing.B) { splitter := NewPyramidChunker(cp) tester := &chunkerTester{t: t} data := testDataReader(n) - _, err := tester.Split(splitter, data, int64(n), nil, nil, nil) + _, _, err := tester.Split(splitter, data, int64(n), nil, nil) if err != nil { tester.t.Fatalf(err.Error()) } @@ -468,16 +462,14 @@ func benchmarkAppendPyramid(n, m int, t *testing.B) { data1 := testDataReader(m) chunkC := make(chan *Chunk, 1000) - swg := &sync.WaitGroup{} - key, err := tester.Split(chunker, data, int64(n), chunkC, swg, nil) + key, _, err := tester.Split(chunker, data, int64(n), chunkC, nil) if err != nil { tester.t.Fatalf(err.Error()) } chunkC = make(chan *Chunk, 1000) - swg = &sync.WaitGroup{} - _, err = tester.Append(chunker, key, data1, chunkC, swg, nil) + _, _, err = tester.Append(chunker, key, data1, chunkC, nil) if err != nil { tester.t.Fatalf(err.Error()) } diff --git a/swarm/storage/pyramid.go b/swarm/storage/pyramid.go index 2ee3d5b58a..005e2dca89 100644 --- a/swarm/storage/pyramid.go +++ b/swarm/storage/pyramid.go @@ -164,16 +164,17 @@ func (self *PyramidChunker) decrementWorkerCount() { self.workerCount -= 1 } -func (self *PyramidChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, storageWG, processorWG *sync.WaitGroup) (Key, error) { +func (self *PyramidChunker) Split(data io.Reader, size int64, chunkC chan *Chunk) (k Key, wait func(), err error) { jobC := make(chan *chunkJob, 2*ChunkProcessors) wg := &sync.WaitGroup{} + storageWG := &sync.WaitGroup{} errC := make(chan error) quitC := make(chan bool) rootKey := make([]byte, self.hashSize) chunkLevel := make([][]*TreeEntry, self.branches) wg.Add(1) - go self.prepareChunks(false, chunkLevel, data, rootKey, quitC, wg, jobC, processorWG, chunkC, errC, storageWG) + go self.prepareChunks(false, chunkLevel, data, rootKey, quitC, wg, jobC, chunkC, errC, storageWG) // closes internal error channel if all subprocesses in the workgroup finished go func() { @@ -181,10 +182,6 @@ func (self *PyramidChunker) Split(data io.Reader, size int64, chunkC chan *Chunk // waiting for all chunks to finish wg.Wait() - // if storage waitgroup is non-nil, we wait for storage to finish too - if storageWG != nil { - storageWG.Wait() - } //We close errC here because this is passed down to 8 parallel routines underneath. // if a error happens in one of them.. that particular routine raises error... // once they all complete successfully, the control comes back and we can safely close this here. @@ -196,15 +193,15 @@ func (self *PyramidChunker) Split(data io.Reader, size int64, chunkC chan *Chunk select { case err := <-errC: if err != nil { - return nil, err + return nil, nil, err } case <-time.NewTimer(splitTimeout).C: } - return rootKey, nil + return rootKey, storageWG.Wait, nil } -func (self *PyramidChunker) Append(key Key, data io.Reader, chunkC chan *Chunk, storageWG, processorWG *sync.WaitGroup) (Key, error) { +func (self *PyramidChunker) Append(key Key, data io.Reader, chunkC chan *Chunk) (k Key, wait func(), err error) { quitC := make(chan bool) rootKey := make([]byte, self.hashSize) chunkLevel := make([][]*TreeEntry, self.branches) @@ -216,8 +213,10 @@ func (self *PyramidChunker) Append(key Key, data io.Reader, chunkC chan *Chunk, wg := &sync.WaitGroup{} errC := make(chan error) + storageWG := &sync.WaitGroup{} + wg.Add(1) - go self.prepareChunks(true, chunkLevel, data, rootKey, quitC, wg, jobC, processorWG, chunkC, errC, storageWG) + go self.prepareChunks(true, chunkLevel, data, rootKey, quitC, wg, jobC, chunkC, errC, storageWG) // closes internal error channel if all subprocesses in the workgroup finished go func() { @@ -225,10 +224,6 @@ func (self *PyramidChunker) Append(key Key, data io.Reader, chunkC chan *Chunk, // waiting for all chunks to finish wg.Wait() - // if storage waitgroup is non-nil, we wait for storage to finish too - if storageWG != nil { - storageWG.Wait() - } close(errC) }() @@ -237,21 +232,18 @@ func (self *PyramidChunker) Append(key Key, data io.Reader, chunkC chan *Chunk, select { case err := <-errC: if err != nil { - return nil, err + return nil, nil, err } case <-time.NewTimer(splitTimeout).C: } - return rootKey, nil + return rootKey, storageWG.Wait, nil } -func (self *PyramidChunker) processor(id int64, jobC chan *chunkJob, chunkC chan *Chunk, errC chan error, quitC chan bool, swg, wwg *sync.WaitGroup) { +func (self *PyramidChunker) processor(id int64, jobC chan *chunkJob, chunkC chan *Chunk, errC chan error, quitC chan bool, storageWG *sync.WaitGroup) { defer self.decrementWorkerCount() hasher := self.hashFunc() - if wwg != nil { - defer wwg.Done() - } for { select { @@ -259,14 +251,14 @@ func (self *PyramidChunker) processor(id int64, jobC chan *chunkJob, chunkC chan if !ok { return } - self.processChunk(id, hasher, job, chunkC, swg) + self.processChunk(id, hasher, job, chunkC, storageWG) case <-quitC: return } } } -func (self *PyramidChunker) processChunk(id int64, hasher SwarmHash, job *chunkJob, chunkC chan *Chunk, swg *sync.WaitGroup) { +func (self *PyramidChunker) processChunk(id int64, hasher SwarmHash, job *chunkJob, chunkC chan *Chunk, storageWG *sync.WaitGroup) { hasher.ResetWithLength(job.chunk[:8]) // 8 bytes of length hasher.Write(job.chunk[8:]) // minus 8 []byte length h := hasher.Sum(nil) @@ -274,21 +266,20 @@ func (self *PyramidChunker) processChunk(id int64, hasher SwarmHash, job *chunkJ newChunk := NewChunk(h, nil) newChunk.SData = job.chunk newChunk.Size = job.size - newChunk.wg = swg // report hash of this chunk one level up (keys corresponds to the proper subslice of the parent chunk) copy(job.key, h) // send off new chunk to storage - if chunkC != nil { - if swg != nil { - swg.Add(1) - } - } job.parentWg.Done() if chunkC != nil { chunkC <- newChunk + storageWG.Add(1) + go func() { + defer storageWG.Done() + <-newChunk.dbStored + }() } } @@ -372,19 +363,14 @@ func (self *PyramidChunker) loadTree(chunkLevel [][]*TreeEntry, key Key, chunkC return nil } -func (self *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEntry, data io.Reader, rootKey []byte, quitC chan bool, wg *sync.WaitGroup, jobC chan *chunkJob, processorWG *sync.WaitGroup, chunkC chan *Chunk, errC chan error, storageWG *sync.WaitGroup) { +func (self *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEntry, data io.Reader, rootKey []byte, quitC chan bool, wg *sync.WaitGroup, jobC chan *chunkJob, chunkC chan *Chunk, errC chan error, storageWG *sync.WaitGroup) { defer wg.Done() chunkWG := &sync.WaitGroup{} totalDataSize := 0 - // processorWG keeps track of workers spawned for hashing chunks - if processorWG != nil { - processorWG.Add(1) - } - self.incrementWorkerCount() - go self.processor(self.workerCount, jobC, chunkC, errC, quitC, storageWG, processorWG) + go self.processor(self.workerCount, jobC, chunkC, errC, quitC, storageWG) parent := NewTreeEntry(self) var unFinishedChunk *Chunk @@ -484,11 +470,8 @@ func (self *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEnt workers := self.getWorkerCount() if int64(len(jobC)) > workers && workers < ChunkProcessors { - if processorWG != nil { - processorWG.Add(1) - } self.incrementWorkerCount() - go self.processor(self.workerCount, jobC, chunkC, errC, quitC, storageWG, processorWG) + go self.processor(self.workerCount, jobC, chunkC, errC, quitC, storageWG) } } diff --git a/swarm/storage/types.go b/swarm/storage/types.go index 3cfe8a6f11..000c4a056b 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -280,7 +280,7 @@ type Splitter interface { The key for the root chunk is supplied to load the respective tree. Rest of the parameters behave like Split. */ - Append(Key, io.Reader, chan *Chunk) (Key, error) + Append(Key, io.Reader, chan *Chunk) (Key, func(), error) } type Joiner interface { From 53c7fb46c7daa2d630be5bfbcb82e4cef5ea79ce Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 4 Jan 2018 02:24:49 +0100 Subject: [PATCH 009/210] swarm/storage, swarm/network: fix tests - all tests pass - chunker tester split/append should wait before closing quit chan - synciterator now increments first - kademlia hive pretty print test broke on 2017 vs 2018 year in date ;) - append tester needs to wait for dbstore chan in other conditional branch - mput should enforce dbstored chan closed when chunk created for memstore - ... --- swarm/network/kademlia_test.go | 3 ++- swarm/network/syncer.go | 2 +- swarm/storage/chunker_test.go | 25 +++++++++++++++---------- swarm/storage/common_test.go | 11 ++++++++++- swarm/storage/dbstore_test.go | 12 ++++++++++-- 5 files changed, 38 insertions(+), 15 deletions(-) diff --git a/swarm/network/kademlia_test.go b/swarm/network/kademlia_test.go index 5c09133f19..20bfd7daaf 100644 --- a/swarm/network/kademlia_test.go +++ b/swarm/network/kademlia_test.go @@ -399,9 +399,10 @@ func TestPruning(t *testing.T) { func TestKademliaHiveString(t *testing.T) { k := newTestKademlia("00000000").On("01000000", "00100000").Register("10000000", "10000001") + k.MaxProxDisplay = 8 h := k.String() expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), MinProxBinSize: 2, MinBinSize: 1, MaxBinSize: 4\n000 0 | 2 8100 (0) 8000 (0)\n============ DEPTH: 1 ==========================================\n001 1 4000 | 1 4000 (0)\n002 1 2000 | 1 2000 (0)\n003 0 | 0\n004 0 | 0\n005 0 | 0\n006 0 | 0\n007 0 | 0\n=========================================================================" - if expH[100:] != h[100:] { + if expH[104:] != h[104:] { t.Fatalf("incorrect hive output. expected %v, got %v", expH, h) } } diff --git a/swarm/network/syncer.go b/swarm/network/syncer.go index 028945384e..1d85116c99 100644 --- a/swarm/network/syncer.go +++ b/swarm/network/syncer.go @@ -247,7 +247,7 @@ func (self *IncomingSwarmSyncer) TakeoverProof(s Stream, from uint64, hashes []b if self.chunker != nil { if from > self.sessionAt { // for live syncing currentRoot is always updated //expRoot, err := self.chunker.Append(self.currentRoot, bytes.NewReader(hashes), self.retrieveC, self.storeC) - expRoot, err := self.chunker.Append(self.currentRoot, bytes.NewReader(hashes), self.retrieveC) + expRoot, _, err := self.chunker.Append(self.currentRoot, bytes.NewReader(hashes), self.retrieveC) if err != nil { return nil, err } diff --git a/swarm/storage/chunker_test.go b/swarm/storage/chunker_test.go index abfcbbed9f..b6eb9ba6f8 100644 --- a/swarm/storage/chunker_test.go +++ b/swarm/storage/chunker_test.go @@ -72,13 +72,16 @@ func (self *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, c }() } - key, wait, err = chunker.Split(data, size, chunkC) + var w func() + key, w, err = chunker.Split(data, size, chunkC) if err != nil && expectedError == nil { err = fmt.Errorf("Split error: %v", err) } - if chunkC != nil { - close(quitC) + wait = func() { + w() + close(quitC) + } } else { wait = func() {} } @@ -102,6 +105,7 @@ func (self *chunkerTester) Append(chunker Splitter, rootKey Key, data io.Reader, if !success { // Requesting data self.chunks[chunk.Key.String()] = chunk + close(chunk.dbStored) } else { // getting data chunk.SData = stored.SData @@ -114,14 +118,17 @@ func (self *chunkerTester) Append(chunker Splitter, rootKey Key, data io.Reader, } }() } - - key, wait, err = chunker.Append(rootKey, data, chunkC) + var w func() + key, w, err = chunker.Append(rootKey, data, chunkC) if err != nil && expectedError == nil { err = fmt.Errorf("Append error: %v", err) } if chunkC != nil { - close(quitC) + wait = func() { + w() + close(quitC) + } } else { wait = func() {} } @@ -198,12 +205,12 @@ func testRandomData(splitter Splitter, n int, tester *chunkerTester) Key { chunkC := make(chan *Chunk, 1000) - key, _, err := tester.Split(splitter, data, int64(n), chunkC, nil) + key, wait, err := tester.Split(splitter, data, int64(n), chunkC, nil) if err != nil { tester.t.Fatalf(err.Error()) } tester.t.Logf(" Key = %v\n", key) - + wait() chunkC = make(chan *Chunk, 1000) quitC := make(chan bool) @@ -315,8 +322,6 @@ func TestSha3ForCorrectness(t *testing.T) { } func TestDataAppend(t *testing.T) { - t.Skip("Skip until append chunks are fixed") - sizes := []int{1, 1, 1, 4095, 4096, 4097, 1, 1, 1, 123456, 2345678, 2345678} appendSizes := []int{4095, 4096, 4097, 1, 1, 1, 8191, 8192, 8193, 9000, 3000, 5000} diff --git a/swarm/storage/common_test.go b/swarm/storage/common_test.go index bdc4814d51..700ba8dac0 100644 --- a/swarm/storage/common_test.go +++ b/swarm/storage/common_test.go @@ -87,13 +87,22 @@ func mput(store ChunkStore, processors int, n int, f func(i int) *Chunk) (hs []K defer wg.Done() store.Put(chunk) + <-chunk.dbStored }() } }() } + fa := f + if _, ok := store.(*MemStore); ok { + fa = func(i int) *Chunk { + chunk := f(i) + close(chunk.dbStored) + return chunk + } + } for i := 0; i < n; i++ { - chunk := f(i) + chunk := fa(i) hs = append(hs, chunk.Key) c <- chunk } diff --git a/swarm/storage/dbstore_test.go b/swarm/storage/dbstore_test.go index ea250bfb07..dc234fd195 100644 --- a/swarm/storage/dbstore_test.go +++ b/swarm/storage/dbstore_test.go @@ -21,6 +21,7 @@ import ( "fmt" "io/ioutil" "os" + "sync" "testing" "github.com/ethereum/go-ethereum/log" @@ -126,9 +127,16 @@ func TestIterator(t *testing.T) { FakeChunk(getDefaultChunkSize(), chunkcount, chunks) + wg := &sync.WaitGroup{} + wg.Add(len(chunks)) for i = 0; i < len(chunks); i++ { db.Put(chunks[i]) chunkkeys[i] = chunks[i].Key + j := i + go func() { + defer wg.Done() + <-chunks[j].dbStored + }() } //testSplit(m, l, 128, chunkkeys, t) @@ -136,12 +144,12 @@ func TestIterator(t *testing.T) { for i = 0; i < len(chunkkeys); i++ { log.Trace(fmt.Sprintf("Chunk array pos %d/%d: '%v'", i, chunkcount, chunkkeys[i])) } - + wg.Wait() i = 0 for poc = 0; poc <= 255; poc++ { err := db.SyncIterator(0, uint64(chunkkeys.Len()), uint8(poc), func(k Key, n uint64) bool { log.Trace(fmt.Sprintf("Got key %v number %d poc %d", k, n, uint8(poc))) - chunkkeys_results[n] = k + chunkkeys_results[n-1] = k i++ return true }) From 4e0303f7821b030e01bbfa21162fdbe490c225cf Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Thu, 4 Jan 2018 10:33:21 +0100 Subject: [PATCH 010/210] Remove commented code --- swarm/storage/dbstore.go | 53 ---------------------------------------- 1 file changed, 53 deletions(-) diff --git a/swarm/storage/dbstore.go b/swarm/storage/dbstore.go index 01a6b79f7f..d559a7b27b 100644 --- a/swarm/storage/dbstore.go +++ b/swarm/storage/dbstore.go @@ -509,59 +509,6 @@ func (s *DbStore) CurrentStorageIndex() uint64 { return s.dataIdx } -// TODO: remove the old code for Put -// func (s *DbStore) Put(chunk *Chunk) { -// s.lock.Lock() -// defer s.lock.Unlock() - -// ikey := getIndexKey(chunk.Key) -// var index dpaDBIndex - -// if s.tryAccessIdx(ikey, &index) { -// if chunk.dbStored != nil { -// close(chunk.dbStored) -// } -// log.Trace(fmt.Sprintf("Storing to DB: chunk already exists, only update access")) -// return // already exists, only update access -// } - -// data := encodeData(chunk) - -// if s.entryCnt >= s.capacity { -// s.collectGarbage(gcArrayFreeRatio) -// } - -// po := s.po(chunk.Key) -// t_datakey := getDataKey(s.dataIdx, po) -// s.batch.Put(t_datakey, data) - -// index.Idx = s.dataIdx -// s.updateIndexAccess(&index) - -// idata := encodeIndex(&index) -// s.batch.Put(ikey, idata) - -// s.batch.Put(keyEntryCnt, U64ToBytes(s.entryCnt)) -// s.entryCnt++ -// s.batch.Put(keyDataIdx, U64ToBytes(s.dataIdx)) -// s.dataIdx++ -// accesscnt := make([]byte, 8) -// binary.LittleEndian.PutUint64(accesscnt, s.accessCnt) -// s.batch.Put(keyAccessCnt, accesscnt) -// s.accessCnt++ - -// s.bucketCnt[po]++ -// cntKey := make([]byte, 2) -// cntKey[0] = keyDistanceCnt -// cntKey[1] = po -// s.batch.Put(cntKey, U64ToBytes(s.bucketCnt[po])) - -// if chunk.dbStored != nil { -// close(chunk.dbStored) -// } -// log.Trace(fmt.Sprintf("DbStore.Put: %v. db storage counter: %v ", chunk.Key.Log(), s.dataIdx)) -// } - func (s *DbStore) Put(chunk *Chunk) { s.lock.Lock() defer s.lock.Unlock() From 772153a4b41f1e75f4c2b1db2db5d4bf1e1ea480 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Thu, 4 Jan 2018 12:20:31 +0100 Subject: [PATCH 011/210] Remove unnecessary wg from Chunk --- swarm/storage/localstore.go | 6 ------ swarm/storage/types.go | 16 +++++++--------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index ddc10934d8..c70271a09a 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -43,14 +43,8 @@ func NewLocalStore(hash SwarmHasher, params *StoreParams, basekey []byte) (*Loca // unsafe, in that the data is not integrity checked func (self *LocalStore) Put(chunk *Chunk) { self.memStore.Put(chunk) - if chunk.wg != nil { - chunk.wg.Add(1) - } go func() { self.DbStore.Put(chunk) - if chunk.wg != nil { - chunk.wg.Done() - } }() } diff --git a/swarm/storage/types.go b/swarm/storage/types.go index 000c4a056b..833a1b4262 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -24,7 +24,6 @@ import ( "fmt" "hash" "io" - "sync" "github.com/ethereum/go-ethereum/bmt" "github.com/ethereum/go-ethereum/common" @@ -194,14 +193,13 @@ func NewRequestStatus(key Key) *RequestStatus { // but the size of the subtree encoded in the chunk // 0 if request, to be supplied by the dpa type Chunk struct { - Key Key // always - SData []byte // nil if request, to be supplied by dpa - Size int64 // size of the data covered by the subtree encoded in this chunk - Source Peer // peer - C chan bool // to signal data delivery by the dpa - Req *RequestStatus // request Status needed by netStore - wg *sync.WaitGroup // wg to synchronize - dbStored chan bool // never remove a chunk from memStore before it is written to dbStore + Key Key // always + SData []byte // nil if request, to be supplied by dpa + Size int64 // size of the data covered by the subtree encoded in this chunk + Source Peer // peer + C chan bool // to signal data delivery by the dpa + Req *RequestStatus // request Status needed by netStore + dbStored chan bool // never remove a chunk from memStore before it is written to dbStore } func NewChunk(key Key, rs *RequestStatus) *Chunk { From 755cf50d6ffe12e349a9d494aa32fe9631dced24 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Thu, 4 Jan 2018 12:21:40 +0100 Subject: [PATCH 012/210] Wait for Split and Append in benchmark test --- swarm/storage/chunker_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/swarm/storage/chunker_test.go b/swarm/storage/chunker_test.go index b6eb9ba6f8..5cb1125bf5 100644 --- a/swarm/storage/chunker_test.go +++ b/swarm/storage/chunker_test.go @@ -467,18 +467,18 @@ func benchmarkAppendPyramid(n, m int, t *testing.B) { data1 := testDataReader(m) chunkC := make(chan *Chunk, 1000) - key, _, err := tester.Split(chunker, data, int64(n), chunkC, nil) + key, wait, err := tester.Split(chunker, data, int64(n), chunkC, nil) if err != nil { tester.t.Fatalf(err.Error()) } - + wait() chunkC = make(chan *Chunk, 1000) - _, _, err = tester.Append(chunker, key, data1, chunkC, nil) + _, wait, err = tester.Append(chunker, key, data1, chunkC, nil) if err != nil { tester.t.Fatalf(err.Error()) } - + wait() close(chunkC) } } From 1ad50b1563e3ea11ca7016dfdeaecc3fede0cd24 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Thu, 4 Jan 2018 12:49:03 +0100 Subject: [PATCH 013/210] Fix storageWG in pyramid chunker storageWG did not work because it was possible to start to wait on earlier than the first Add happened Also go routine for prepareChunks in Split and Append was unnecessary, because the processors are started in goroutines anyway --- swarm/storage/pyramid.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/swarm/storage/pyramid.go b/swarm/storage/pyramid.go index 005e2dca89..28736cf319 100644 --- a/swarm/storage/pyramid.go +++ b/swarm/storage/pyramid.go @@ -168,13 +168,14 @@ func (self *PyramidChunker) Split(data io.Reader, size int64, chunkC chan *Chunk jobC := make(chan *chunkJob, 2*ChunkProcessors) wg := &sync.WaitGroup{} storageWG := &sync.WaitGroup{} + storageWG.Add(1) errC := make(chan error) quitC := make(chan bool) rootKey := make([]byte, self.hashSize) chunkLevel := make([][]*TreeEntry, self.branches) wg.Add(1) - go self.prepareChunks(false, chunkLevel, data, rootKey, quitC, wg, jobC, chunkC, errC, storageWG) + self.prepareChunks(false, chunkLevel, data, rootKey, quitC, wg, jobC, chunkC, errC, storageWG) // closes internal error channel if all subprocesses in the workgroup finished go func() { @@ -214,9 +215,10 @@ func (self *PyramidChunker) Append(key Key, data io.Reader, chunkC chan *Chunk) errC := make(chan error) storageWG := &sync.WaitGroup{} + storageWG.Add(1) wg.Add(1) - go self.prepareChunks(true, chunkLevel, data, rootKey, quitC, wg, jobC, chunkC, errC, storageWG) + self.prepareChunks(true, chunkLevel, data, rootKey, quitC, wg, jobC, chunkC, errC, storageWG) // closes internal error channel if all subprocesses in the workgroup finished go func() { @@ -242,7 +244,7 @@ func (self *PyramidChunker) Append(key Key, data io.Reader, chunkC chan *Chunk) func (self *PyramidChunker) processor(id int64, jobC chan *chunkJob, chunkC chan *Chunk, errC chan error, quitC chan bool, storageWG *sync.WaitGroup) { defer self.decrementWorkerCount() - + defer storageWG.Done() hasher := self.hashFunc() for { select { @@ -370,6 +372,7 @@ func (self *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEnt totalDataSize := 0 self.incrementWorkerCount() + go self.processor(self.workerCount, jobC, chunkC, errC, quitC, storageWG) parent := NewTreeEntry(self) @@ -471,6 +474,7 @@ func (self *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEnt workers := self.getWorkerCount() if int64(len(jobC)) > workers && workers < ChunkProcessors { self.incrementWorkerCount() + storageWG.Add(1) go self.processor(self.workerCount, jobC, chunkC, errC, quitC, storageWG) } From f3fdcb2064d8c48488a6b02492db3277c0680773 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Thu, 4 Jan 2018 12:59:04 +0100 Subject: [PATCH 014/210] Fis storageWG in chunker storeWG did not work because it was possible to start to wait on earlier than the first Add happened --- swarm/storage/chunker.go | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index dde975c1a0..be142f227b 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -130,7 +130,7 @@ func (self *TreeChunker) Split(data io.Reader, size int64, chunkC chan *Chunk) ( quitC := make(chan bool) self.incrementWorkerCount() - go self.hashWorker(jobC, chunkC, errC, quitC, storeWg) + self.runHashWorker(jobC, chunkC, errC, quitC, storeWg) depth := 0 treeSize := self.chunkSize @@ -230,7 +230,7 @@ func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reade worker := self.getWorkerCount() if int64(len(jobC)) > worker && worker < ChunkProcessors { self.incrementWorkerCount() - go self.hashWorker(jobC, chunkC, errC, quitC, storeWg) + self.runHashWorker(jobC, chunkC, errC, quitC, storeWg) } select { @@ -239,23 +239,27 @@ func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reade } } -func (self *TreeChunker) hashWorker(jobC chan *hashJob, chunkC chan *Chunk, errC chan error, quitC chan bool, storeWg *sync.WaitGroup) { - defer self.decrementWorkerCount() +func (self *TreeChunker) runHashWorker(jobC chan *hashJob, chunkC chan *Chunk, errC chan error, quitC chan bool, storeWg *sync.WaitGroup) { + storeWg.Add(1) - hasher := self.hashFunc() - for { - select { + go func() { + defer self.decrementWorkerCount() + defer storeWg.Done() + hasher := self.hashFunc() + for { + select { - case job, ok := <-jobC: - if !ok { + case job, ok := <-jobC: + if !ok { + return + } + // now we got the hashes in the chunk, then hash the chunks + self.hashChunk(hasher, job, chunkC, storeWg) + case <-quitC: return } - // now we got the hashes in the chunk, then hash the chunks - self.hashChunk(hasher, job, chunkC, storeWg) - case <-quitC: - return } - } + }() } // The treeChunkers own Hash hashes together From c2bedb54fe13b272320bb458db1e1f86fef0deb7 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Thu, 4 Jan 2018 16:51:11 +0100 Subject: [PATCH 015/210] Some draft stuff for Janos --- swarm/network/requests.go | 27 ----------------------- swarm/network/streamer.go | 46 +++++++++++++++++++++++++++++++++++++-- swarm/network/syncer.go | 4 ++++ 3 files changed, 48 insertions(+), 29 deletions(-) diff --git a/swarm/network/requests.go b/swarm/network/requests.go index 27e0ea2853..ef6ae18450 100644 --- a/swarm/network/requests.go +++ b/swarm/network/requests.go @@ -122,18 +122,6 @@ func addRequester(rs *storage.RequestStatus, req *retrieveRequestMsg) { rs.Requesters[req.Id] = append(list, req) } -/* - store requests are put in netstore so they are stored and then - forwarded to the peers in their kademlia proximity bin by the syncer -*/ -type storeRequestMsg struct { - Key storage.Key - SData []byte // the stored chunk Data (incl size) - // optional - Id uint64 // request ID. if delivery, the ID is retrieve request ID - from Peer // [not serialised] protocol registers the requester -} - func (self storeRequestMsg) String() string { var from string if self.from == nil { @@ -147,18 +135,3 @@ func (self storeRequestMsg) String() string { } return fmt.Sprintf("from: %v, ID: %v, SData %x", from, self.Id, self.SData[:end]) } - -// the entrypoint for store requests coming from the bzz wire protocol -// if key found locally, return. otherwise -// remote is untrusted, so hash is verified and chunk passed on to NetStore -func (self *RequestHandler) handleStoreRequestMsg(msg interface{}, p Peer) error { - req := msg.(*storeRequestMsg) - req.from = p - // TODO: chunk validation - chunk := storage.NewChunk(req.Key, nil) - chunk.SData = req.SData - chunk.Source = p - self.netStore.Put(chunk) - log.Trace(fmt.Sprintf("delivery of %v from %v", chunk, p)) - return nil -} diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go index 0207f3a0fc..3c7314ceef 100644 --- a/swarm/network/streamer.go +++ b/swarm/network/streamer.go @@ -22,6 +22,7 @@ import ( "fmt" "sync" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/protocols" bv "github.com/ethereum/go-ethereum/swarm/network/bitvector" @@ -91,6 +92,18 @@ type UnsyncedKeysMsg struct { *HandoverProof // HandoverProof } +/* + store requests are put in netstore so they are stored and then + forwarded to the peers in their kademlia proximity bin by the syncer +*/ +type ChunkDeliveryMsg struct { + Key storage.Key + SData []byte // the stored chunk Data (incl size) + // optional + Id uint64 // request ID. if delivery, the ID is retrieve request ID + from Peer // [not serialised] protocol registers the requester +} + // String pretty prints UnsyncedKeysMsg func (self UnsyncedKeysMsg) String() string { return fmt.Sprintf("Stream '%v' [%v-%v] (%v)", self.Stream, self.From, self.To, len(self.Hashes)/HashSize) @@ -170,7 +183,7 @@ func (self *Streamer) PeerInfo(id discover.NodeID) interface{} { } // OutgoingStreamer interface for outgoing peer Streamer -type OutgoingStreamer interface { +type OutgoingStreamerBackend interface { CurrentBatch() []byte SetNextBatch(uint64, uint64) ([]byte, uint64, uint64, *HandoverProof, error) GetData([]byte) []byte @@ -178,7 +191,7 @@ type OutgoingStreamer interface { } // IncomingStreamer interface for incoming peer Streamer -type IncomingStreamer interface { +type IncomingStreamerBackend interface { NextBatch(uint64) (uint64, uint64) NeedData([]byte) func() Priority() int @@ -197,6 +210,16 @@ type StreamerPeer struct { quit chan struct{} } +type IncomingStreamer struct { + priority uint8 + peer *StreamerPeer +} + +type OutgoingStreamer struct { + priority uint8 + peer *StreamerPeer +} + // NewStreamerPeer is the constructor for StreamerPeer func NewStreamerPeer(p Peer, streamer *Streamer) *StreamerPeer { self := &StreamerPeer{ @@ -255,6 +278,10 @@ func (self *StreamerPeer) setIncomingStreamer(s Stream, i IncomingStreamer) erro return nil } +func (self *OutgoingStreamer) Subscribe(s Stream) OutgoingStreamerBackend { + +} + // Subscribe initiates the streamer func (self *StreamerPeer) Subscribe(s Stream, from, to uint64) error { f, err := self.streamer.GetIncomingStreamer(s) @@ -384,6 +411,18 @@ func (self *StreamerPeer) handleTakeoverProofMsg(msg interface{}) error { return nil } +func (self *StreamerPeer) handleChunkDeliveryMsg(msg interface{}) error { + req := msg.(*chunkDeliveryMsg) + req.from = self + // TODO: chunk validation + chunk := storage.NewChunk(req.Key, nil) + chunk.SData = req.SData + chunk.Source = p + self.netStore.Put(chunk) + log.Trace(fmt.Sprintf("delivery of %v from %v", chunk, p)) + return nil +} + // Deliver sends a storeRequestMsg protocol message to the peer func (self *StreamerPeer) Deliver(chunk *storage.Chunk, priority int) error { msg := &storeRequestMsg{ @@ -451,6 +490,9 @@ func (self *StreamerPeer) HandleMsg(msg interface{}) error { case *WantedKeysMsg: return self.handleWantedKeysMsg(msg) + case *ChunkDeliveryMsg: + return self.handleChunkDeliveryMsg(msg) + default: return fmt.Errorf("unknown message type: %T", msg) } diff --git a/swarm/network/syncer.go b/swarm/network/syncer.go index 1d85116c99..fe212ff988 100644 --- a/swarm/network/syncer.go +++ b/swarm/network/syncer.go @@ -148,6 +148,10 @@ type IncomingSwarmSyncer struct { end, start uint64 } +type { + Subscribe() +} + // NewIncomingSwarmSyncer is a contructor for provable data exchange syncer func NewIncomingSwarmSyncer(po uint8, priority int, intervals []uint64, p Peer, store storage.ChunkStore, chunker storage.Chunker) (*IncomingSwarmSyncer, error) { self := &IncomingSwarmSyncer{ From 2ea9cf58f23926a3318cca8282b3ddbdae8d4e5c Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Fri, 5 Jan 2018 17:07:07 +0100 Subject: [PATCH 016/210] swarm: request streamer implementation --- swarm/network/protocol.go | 4 +- swarm/network/requests.go | 206 +++++++++++++++++------------------ swarm/network/streamer.go | 211 +++++++++++++++++++++++++++++------- swarm/network/syncer.go | 39 ++++--- swarm/storage/dpa.go | 57 ++++------ swarm/storage/forwarder.go | 16 --- swarm/storage/localstore.go | 23 ++++ swarm/storage/memstore.go | 4 +- swarm/storage/netstore.go | 157 +++++++++++++-------------- swarm/storage/types.go | 39 ++----- swarm/swarm.go | 21 ++-- 11 files changed, 442 insertions(+), 335 deletions(-) delete mode 100644 swarm/storage/forwarder.go diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index 1f25464b11..9374d844c8 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -115,9 +115,9 @@ type Bzz struct { // * bzz config // * overlay driver // * peer store -func NewBzz(config *BzzConfig, kad Overlay, store StateStore) *Bzz { +func NewBzz(config *BzzConfig, kad Overlay, store StateStore, streamer *Streamer) *Bzz { return &Bzz{ - Streamer: NewStreamer(), + Streamer: streamer, Hive: NewHive(config.HiveParams, kad, store), localAddr: &BzzAddr{config.OverlayAddr, config.UnderlayAddr}, handshakes: make(map[discover.NodeID]*HandshakeMsg), diff --git a/swarm/network/requests.go b/swarm/network/requests.go index ef6ae18450..f5cea0ea44 100644 --- a/swarm/network/requests.go +++ b/swarm/network/requests.go @@ -16,122 +16,122 @@ package network -import ( - "fmt" +// import ( +// "fmt" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/swarm/storage" -) +// "github.com/ethereum/go-ethereum/log" +// "github.com/ethereum/go-ethereum/swarm/storage" +// ) -/* - Retrieve Request and store Request handling -*/ +// /* +// Retrieve Request and store Request handling +// */ -// Handler for storage/retrieval related protocol requests -type RequestHandler struct { - netStore *storage.NetStore -} +// // Handler for storage/retrieval related protocol requests +// type RequestHandler struct { +// netStore *storage.NetStore +// } -// NewEwquestHandler creates a new RequestHandler -// netStore to -func NewRequestHandler(netStore *storage.NetStore) *RequestHandler { - return &RequestHandler{ - netStore: netStore, // entrypoint internal - } -} +// // NewEwquestHandler creates a new RequestHandler +// // netStore to +// func NewRequestHandler(netStore *storage.NetStore) *RequestHandler { +// return &RequestHandler{ +// netStore: netStore, // entrypoint internal +// } +// } -/* -Retrieve request +// /* +// Retrieve request -MaxSize specifies the maximum size that the peer will accept. This is useful in -particular if we allow storage and delivery of multichunk payload representing -the entire or partial subtree unfolding from the requested root key. -So when only interested in limited part of a stream (infinite trees) or only -testing chunk availability etc etc, we can indicate it by limiting the size here. +// MaxSize specifies the maximum size that the peer will accept. This is useful in +// particular if we allow storage and delivery of multichunk payload representing +// the entire or partial subtree unfolding from the requested root key. +// So when only interested in limited part of a stream (infinite trees) or only +// testing chunk availability etc etc, we can indicate it by limiting the size here. -Request ID can be newly generated or kept from the request originator. +// Request ID can be newly generated or kept from the request originator. -*/ -type retrieveRequestMsg struct { - Key storage.Key // target Key address of chunk to be retrieved - Id uint64 // request id, request is a lookup if missing or zero - MaxSize uint64 // maximum size of delivery accepted - from *StreamerPeer // -} +// */ +// type retrieveRequestMsg struct { +// Key storage.Key // target Key address of chunk to be retrieved +// Id uint64 // request id, request is a lookup if missing or zero +// MaxSize uint64 // maximum size of delivery accepted +// from *StreamerPeer // +// } -func (self retrieveRequestMsg) String() string { - var from string - if self.from == nil { - from = "ourselves" - } else { - from = fmt.Sprintf("%x", self.from.Over()) - } - var target []byte - if len(self.Key) > 3 { - target = self.Key[:4] - } - return fmt.Sprintf("Requester: %v, Key: %x; ID: %v, MaxSize: %v", from, target, self.Id, self.MaxSize) -} +// func (self retrieveRequestMsg) String() string { +// var from string +// if self.from == nil { +// from = "ourselves" +// } else { +// from = fmt.Sprintf("%x", self.from.Over()) +// } +// var target []byte +// if len(self.Key) > 3 { +// target = self.Key[:4] +// } +// return fmt.Sprintf("Requester: %v, Key: %x; ID: %v, MaxSize: %v", from, target, self.Id, self.MaxSize) +// } // entrypoint for retrieve requests coming from the bzz wire protocol // checks swap balance - return if peer has no credit -func (self *StreamerPeer) handleRetrieveRequestMsg(msg interface{}) error { - req := msg.(*retrieveRequestMsg) - req.from = self - // TODO: - // swap - record credit for 1 request - // note that only charge actual reqsearches +// func (self *StreamerPeer) handleRetrieveRequestMsg(msg interface{}) error { +// req := msg.(*retrieveRequestMsg) +// req.from = self +// // TODO: +// // swap - record credit for 1 request +// // note that only charge actual reqsearches - // call storage.NetStore#Get which - // blocks until local retrieval finished - // launches cloud retrieval - chunk, _ := self.netStore.Get(req.Key) - rs := chunk.Req - if rs != nil { - rs = storage.NewRequestStatus(req.Key) - addRequester(rs, req) - chunk.Req = rs - } +// // call storage.NetStore#Get which +// // blocks until local retrieval finished +// // launches cloud retrieval +// chunk, _ := self.netStore.Get(req.Key) +// rs := chunk.Req +// if rs != nil { +// rs = storage.NewRequestStatus(req.Key) +// addRequester(rs, req) +// chunk.Req = rs +// } - // check if we can immediately deliver - if chunk.SData != nil { - if req.MaxSize == 0 || int64(req.MaxSize) >= chunk.Size { - err := self.Deliver(chunk, Top) - if err != nil { - log.Trace(fmt.Sprintf("%v - content found, delivery error: %v", req.Key.Log(), err)) - return nil - } - log.Trace(fmt.Sprintf("%v - content found, delivering...", req.Key.Log())) - } else { - log.Trace(fmt.Sprintf("%v - content found, not wanted", req.Key.Log())) - } - } else { - log.Trace(fmt.Sprintf("content not found locally, retrieve via bzz", req.Key.Log())) - } - return nil -} +// // check if we can immediately deliver +// if chunk.SData != nil { +// if req.MaxSize == 0 || int64(req.MaxSize) >= chunk.Size { +// err := self.Deliver(chunk, Top) +// if err != nil { +// log.Trace(fmt.Sprintf("%v - content found, delivery error: %v", req.Key.Log(), err)) +// return nil +// } +// log.Trace(fmt.Sprintf("%v - content found, delivering...", req.Key.Log())) +// } else { +// log.Trace(fmt.Sprintf("%v - content found, not wanted", req.Key.Log())) +// } +// } else { +// log.Trace(fmt.Sprintf("content not found locally, retrieve via bzz", req.Key.Log())) +// } +// return nil +// } -/* -adds a new peer to an existing open request -only add if less than requesterCount peers forwarded the same request id so far -note this is done irrespective of status (searching or found) -*/ -func addRequester(rs *storage.RequestStatus, req *retrieveRequestMsg) { - log.Trace(fmt.Sprintf("Depo.addRequester: key %v - add peer to req.Id %v", req.Key.Log(), req.Id)) - list := rs.Requesters[req.Id] - rs.Requesters[req.Id] = append(list, req) -} +// /* +// adds a new peer to an existing open request +// only add if less than requesterCount peers forwarded the same request id so far +// note this is done irrespective of status (searching or found) +// */ +// func addRequester(rs *storage.RequestStatus, req *retrieveRequestMsg) { +// log.Trace(fmt.Sprintf("Depo.addRequester: key %v - add peer to req.Id %v", req.Key.Log(), req.Id)) +// list := rs.Requesters[req.Id] +// rs.Requesters[req.Id] = append(list, req) +// } -func (self storeRequestMsg) String() string { - var from string - if self.from == nil { - from = "self" - } else { - from = fmt.Sprintf("%x", self.from.Over()) - } - end := len(self.SData) - if len(self.SData) > 10 { - end = 10 - } - return fmt.Sprintf("from: %v, ID: %v, SData %x", from, self.Id, self.SData[:end]) -} +// func (self storeRequestMsg) String() string { +// var from string +// if self.from == nil { +// from = "self" +// } else { +// from = fmt.Sprintf("%x", self.from.Over()) +// } +// end := len(self.SData) +// if len(self.SData) > 10 { +// end = 10 +// } +// return fmt.Sprintf("from: %v, ID: %v, SData %x", from, self.Id, self.SData[:end]) +// } diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go index 3c7314ceef..19f948b4fa 100644 --- a/swarm/network/streamer.go +++ b/swarm/network/streamer.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "sync" + "time" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/discover" @@ -128,13 +129,20 @@ type Streamer struct { outgoingLock sync.RWMutex outgoing map[Stream]func(*StreamerPeer) (OutgoingStreamer, error) incoming map[Stream]func(*StreamerPeer) (IncomingStreamer, error) + + dbAccess *DbAccess + overlay Overlay + receiveC chan *ChunkDeliveryMsg } // NewStreamer is Streamer constructor -func NewStreamer() *Streamer { +func NewStreamer(overlay Overlay, dbAccess *DbAccess) *Streamer { return &Streamer{ outgoing: make(map[Stream]func(*StreamerPeer) (OutgoingStreamer, error)), incoming: make(map[Stream]func(*StreamerPeer) (IncomingStreamer, error)), + dbAccess: dbAccess, + overlay: overlay, + receiveC: make(chan *ChunkDeliveryMsg, 10), } } @@ -183,15 +191,15 @@ func (self *Streamer) PeerInfo(id discover.NodeID) interface{} { } // OutgoingStreamer interface for outgoing peer Streamer -type OutgoingStreamerBackend interface { +type OutgoingStreamer interface { CurrentBatch() []byte - SetNextBatch(uint64, uint64) ([]byte, uint64, uint64, *HandoverProof, error) + SetNextBatch(uint64, uint64) (hashes []byte, from uint64, to uint64, proof *HandoverProof, err error) GetData([]byte) []byte Priority() int } // IncomingStreamer interface for incoming peer Streamer -type IncomingStreamerBackend interface { +type IncomingStreamer interface { NextBatch(uint64) (uint64, uint64) NeedData([]byte) func() Priority() int @@ -200,9 +208,10 @@ type IncomingStreamerBackend interface { // StreamerPeer is the Peer extention for the streaming protocol type StreamerPeer struct { Peer - streamer *Streamer - pq *pq.PriorityQueue - netStore storage.ChunkStore + streamer *Streamer + pq *pq.PriorityQueue + //netStore storage.ChunkStore + dbAccess *DbAccess outgoingLock sync.RWMutex incomingLock sync.RWMutex outgoing map[Stream]OutgoingStreamer @@ -210,15 +219,15 @@ type StreamerPeer struct { quit chan struct{} } -type IncomingStreamer struct { - priority uint8 - peer *StreamerPeer -} +// type IncomingStreamer struct { +// priority uint8 +// peer *StreamerPeer +// } -type OutgoingStreamer struct { - priority uint8 - peer *StreamerPeer -} +// type OutgoingStreamer struct { +// priority uint8 +// peer *StreamerPeer +// } // NewStreamerPeer is the constructor for StreamerPeer func NewStreamerPeer(p Peer, streamer *Streamer) *StreamerPeer { @@ -238,6 +247,142 @@ func NewStreamerPeer(p Peer, streamer *Streamer) *StreamerPeer { return self } +type RetrieveRequestMsg struct { + Key storage.Key +} + +// RetrieveRequestStreamer implements OutgoingStreamer +type RetrieveRequestStreamer struct { + deliveryC chan *storage.Chunk + batchC chan []byte + dbAccess *DbAccess + currentBatch []byte + currentLen uint64 +} + +func RegisterRequestStreamer(streamer *Streamer, dbAccess *DbAccess) { + streamer.RegisterOutgoingStreamer(retrieveRequestStream, func(_ *StreamerPeer) (OutgoingStreamer, error) { + return NewRetrieveRequestStreamer(dbAccess), nil + }) + streamer.RegisterIncomingStreamer(retrieveRequestStream, func(p *StreamerPeer) (IncomingStreamer, error) { + return NewIncomingSwarmSyncer(Top, nil, p, dbAccess, nil) + }) +} + +func NewRetrieveRequestStreamer(dbAccess *DbAccess) *RetrieveRequestStreamer { + s := &RetrieveRequestStreamer{ + deliveryC: make(chan *storage.Chunk), + batchC: make(chan []byte), + dbAccess: dbAccess, + } + go s.processDeliveries() + return s +} + +func (s *RetrieveRequestStreamer) processDeliveries() { + var hashes []byte + for { + select { + case delivery := <-s.deliveryC: + hashes = append(hashes, delivery.Key[:]...) + case s.batchC <- hashes: + hashes = nil + } + } +} + +func (s *RetrieveRequestStreamer) CurrentBatch() []byte { + return s.currentBatch +} + +func (s *RetrieveRequestStreamer) SetNextBatch(_, _ uint64) (hashes []byte, from uint64, to uint64, proof *HandoverProof, err error) { + hashes = <-s.batchC + s.currentBatch = hashes + from = s.currentLen + s.currentLen += uint64(len(hashes)) + to = s.currentLen + return +} + +func (s *RetrieveRequestStreamer) GetData(key []byte) []byte { + chunk, _ := s.dbAccess.get(storage.Key(key)) + return chunk.SData +} + +func (s *RetrieveRequestStreamer) Priority() int { + return Top +} + +const retrieveRequestStream = Stream("RETRIEVE_REQUEST") + +func (self *StreamerPeer) handleRetrieveRequestMsg(req *RetrieveRequestMsg) error { + chunk, created := self.dbAccess.getOrCreateRequest(req.Key) + s, err := self.getOutgoingStreamer(retrieveRequestStream) + if err != nil { + return err + } + streamer := s.(*RetrieveRequestStreamer) + if chunk.ReqC != nil { + if created { + if err := self.streamer.Retrieve(chunk); err != nil { + return err + } + } + go func() { + t := time.NewTicker(3 * time.Minute) + defer t.Stop() + + select { + case <-chunk.ReqC: + case <-self.quit: + return + case <-t.C: + return + } + + streamer.deliveryC <- chunk + }() + return nil + } + // TODO: call the retrieve function of the outgoing syncer + streamer.deliveryC <- chunk + return nil +} + +func (self *Streamer) Retrieve(chunk *storage.Chunk) error { + // TODO: using the overlay find the closes peer to send the retrieve + // request to. + // self.Overlay.EachConn(to, 256, func(op network.OverlayConn, po int, isproxbin bool) bool {}) + return nil +} + +func (self *StreamerPeer) handleChunkDeliveryMsg(req *ChunkDeliveryMsg) error { + chunk, err := self.dbAccess.get(req.Key) + if err != nil { + return err + } + + self.streamer.receiveC <- req + + log.Trace(fmt.Sprintf("delivery of %v from %v", chunk, self)) + return nil +} + +func (self *Streamer) processReceivedChunks() { + for { + select { + case req := <-self.receiveC: + chunk, err := self.dbAccess.get(req.Key) + if err != nil { + continue + } + chunk.SData = req.SData + self.dbAccess.put(chunk) + close(chunk.ReqC) + } + } +} + func (self *StreamerPeer) getOutgoingStreamer(s Stream) (OutgoingStreamer, error) { self.outgoingLock.RLock() defer self.outgoingLock.RUnlock() @@ -278,10 +423,6 @@ func (self *StreamerPeer) setIncomingStreamer(s Stream, i IncomingStreamer) erro return nil } -func (self *OutgoingStreamer) Subscribe(s Stream) OutgoingStreamerBackend { - -} - // Subscribe initiates the streamer func (self *StreamerPeer) Subscribe(s Stream, from, to uint64) error { f, err := self.streamer.GetIncomingStreamer(s) @@ -303,8 +444,7 @@ func (self *StreamerPeer) Subscribe(s Stream, from, to uint64) error { return nil } -func (self *StreamerPeer) handleSubscribeMsg(msg interface{}) error { - req := msg.(*SubscribeMsg) +func (self *StreamerPeer) handleSubscribeMsg(req *SubscribeMsg) error { f, err := self.streamer.GetOutgoingStreamer(req.Stream) if err != nil { return err @@ -322,8 +462,7 @@ func (self *StreamerPeer) handleSubscribeMsg(msg interface{}) error { // handleUnsyncedKeysMsg protocol msg handler calls the incoming streamer interface // Filter method -func (self *StreamerPeer) handleUnsyncedKeysMsg(msg interface{}) error { - req := msg.(*UnsyncedKeysMsg) +func (self *StreamerPeer) handleUnsyncedKeysMsg(req *UnsyncedKeysMsg) error { s, err := self.getIncomingStreamer(req.Stream) if err != nil { return err @@ -357,7 +496,7 @@ func (self *StreamerPeer) handleUnsyncedKeysMsg(msg interface{}) error { if from == to { return nil } - msg = &WantedKeysMsg{ + msg := &WantedKeysMsg{ Stream: req.Stream, Want: want.Bytes(), From: from, @@ -370,8 +509,7 @@ func (self *StreamerPeer) handleUnsyncedKeysMsg(msg interface{}) error { // handleWantedKeysMsg protocol msg handler // * sends the next batch of unsynced keys // * sends the actual data chunks as per WantedKeysMsg -func (self *StreamerPeer) handleWantedKeysMsg(msg interface{}) error { - req := msg.(*WantedKeysMsg) +func (self *StreamerPeer) handleWantedKeysMsg(req *WantedKeysMsg) error { s, err := self.getOutgoingStreamer(req.Stream) if err != nil { return err @@ -401,8 +539,7 @@ func (self *StreamerPeer) handleWantedKeysMsg(msg interface{}) error { return nil } -func (self *StreamerPeer) handleTakeoverProofMsg(msg interface{}) error { - req := msg.(*TakeoverProofMsg) +func (self *StreamerPeer) handleTakeoverProofMsg(req *TakeoverProofMsg) error { _, err := self.getOutgoingStreamer(req.Stream) if err != nil { return err @@ -411,21 +548,9 @@ func (self *StreamerPeer) handleTakeoverProofMsg(msg interface{}) error { return nil } -func (self *StreamerPeer) handleChunkDeliveryMsg(msg interface{}) error { - req := msg.(*chunkDeliveryMsg) - req.from = self - // TODO: chunk validation - chunk := storage.NewChunk(req.Key, nil) - chunk.SData = req.SData - chunk.Source = p - self.netStore.Put(chunk) - log.Trace(fmt.Sprintf("delivery of %v from %v", chunk, p)) - return nil -} - // Deliver sends a storeRequestMsg protocol message to the peer func (self *StreamerPeer) Deliver(chunk *storage.Chunk, priority int) error { - msg := &storeRequestMsg{ + msg := &ChunkDeliveryMsg{ Key: chunk.Key, SData: chunk.SData, } @@ -470,6 +595,10 @@ var StreamerSpec = &protocols.Spec{ func (s *Streamer) Run(p *bzzPeer) error { sp := NewStreamerPeer(p, s) // load saved intervals + sp.handleSubscribeMsg(&SubscribeMsg{ + Stream: retrieveRequestStream, + Priority: uint8(Top), + }) defer close(sp.quit) return sp.Run(sp.HandleMsg) } diff --git a/swarm/network/syncer.go b/swarm/network/syncer.go index fe212ff988..ed87e57870 100644 --- a/swarm/network/syncer.go +++ b/swarm/network/syncer.go @@ -55,6 +55,16 @@ func (self *DbAccess) iterator(from uint64, to uint64, po uint8, f func(storage. return self.db.SyncIterator(from, to, po, f) } +// to obtain the chunks from key or request db entry only +func (self *DbAccess) getOrCreateRequest(key storage.Key) (*storage.Chunk, bool) { + return self.loc.GetOrCreateRequest(key) +} + +// to obtain the chunks from key or request db entry only +func (self *DbAccess) put(chunk *storage.Chunk) { + self.loc.Put(chunk) +} + // OutgoingSwarmSyncer implements an OutgoingStreamer for history syncing on bins // offered streams: // * live request delivery with or without checkback @@ -133,7 +143,6 @@ func (self *OutgoingSwarmSyncer) SetNextBatch(from, to uint64) ([]byte, uint64, // IncomingSwarmSyncer type IncomingSwarmSyncer struct { - po uint8 priority int sessionAt uint64 nextC chan struct{} @@ -142,23 +151,19 @@ type IncomingSwarmSyncer struct { sessionReader storage.LazySectionReader retrieveC chan *storage.Chunk storeC chan *storage.Chunk - store storage.ChunkStore + dbAccess *DbAccess chunker storage.Chunker currentRoot storage.Key + requestFunc func(chunk *storage.Chunk) end, start uint64 } -type { - Subscribe() -} - // NewIncomingSwarmSyncer is a contructor for provable data exchange syncer -func NewIncomingSwarmSyncer(po uint8, priority int, intervals []uint64, p Peer, store storage.ChunkStore, chunker storage.Chunker) (*IncomingSwarmSyncer, error) { +func NewIncomingSwarmSyncer(priority int, intervals []uint64, p Peer, dbAccess *DbAccess, chunker storage.Chunker) (*IncomingSwarmSyncer, error) { self := &IncomingSwarmSyncer{ - po: po, priority: priority, intervals: intervals, - store: store, + dbAccess: dbAccess, chunker: chunker, } return self, nil @@ -194,12 +199,12 @@ func RegisterIncomingSyncers(streamer *Streamer, db *DbAccess) { for po := uint8(0); po < maxPO; po++ { stream := Stream(fmt.Sprintf("SYNC-%02d-live", po)) streamer.RegisterIncomingStreamer(stream, func(p *StreamerPeer) (IncomingStreamer, error) { - return NewIncomingSwarmSyncer(po, High, nil, p, nil, nil) + return NewIncomingSwarmSyncer(High, nil, p, nil, nil) }) stream = Stream(fmt.Sprintf("SYNC-%02d-history", po)) streamer.RegisterIncomingStreamer(stream, func(p *StreamerPeer) (IncomingStreamer, error) { //intervals := loadIntervals(p, po, false) - return NewIncomingSwarmSyncer(po, Mid, nil, p, nil, nil) + return NewIncomingSwarmSyncer(Mid, nil, p, nil, nil) }) // stream = fmt.Sprintf("SYNC-%02d-delete", po) // streamer.RegisterIncomingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) { @@ -210,13 +215,11 @@ func RegisterIncomingSyncers(streamer *Streamer, db *DbAccess) { } // NeedData -func (self *IncomingSwarmSyncer) NeedData(key []byte) func() { - chunk, err := self.store.Get(key) - if err == nil { - if chunk.SData == nil { - // send a request instead - return nil - } +func (self *IncomingSwarmSyncer) NeedData(key []byte) (wait func()) { + chunk, created := self.dbAccess.getOrCreateRequest(key) + // TODO: we may want to request from this peer anyway even if the request exists + if chunk.ReqC == nil || !created { + return nil } // create request and wait until the chunk data arrives and is stored return chunk.WaitToStore diff --git a/swarm/storage/dpa.go b/swarm/storage/dpa.go index 354ff32c15..7ef8a80a99 100644 --- a/swarm/storage/dpa.go +++ b/swarm/storage/dpa.go @@ -179,53 +179,44 @@ func (self *DPA) storeWorker() { // access by calling network is blocking with a timeout type dpaChunkStore struct { - n int - localStore ChunkStore - netStore ChunkStore + localStore *LocalStore + retrieve func(chunk *Chunk) error } -func NewDpaChunkStore(localStore, netStore ChunkStore) *dpaChunkStore { - return &dpaChunkStore{0, localStore, netStore} +func NewDpaChunkStore(localStore *LocalStore, retrieve func(chunk *Chunk) error) *dpaChunkStore { + return &dpaChunkStore{localStore, retrieve} } // Get is the entrypoint for local retrieve requests // waits for response or times out func (self *dpaChunkStore) Get(key Key) (chunk *Chunk, err error) { - chunk, err = self.netStore.Get(key) - if chunk.SData != nil { + var created bool + chunk, created = self.localStore.GetOrCreateRequest(key) + if chunk.ReqC == nil { log.Trace(fmt.Sprintf("DPA.Get: %v found locally, %d bytes", key.Log(), len(chunk.SData))) return } - // TODO: use self.timer time.Timer and reset with defer disableTimer - timer := time.After(searchTimeout) - select { - case <-timer: - log.Trace(fmt.Sprintf("DPA.Get: %v request time out ", key.Log())) - err = notFound - case <-chunk.Req.C: - log.Trace(fmt.Sprintf("DPA.Get: %v retrieved, %d bytes (%p)", key.Log(), len(chunk.SData), chunk)) + + if created { + if err := self.retrieve(chunk); err != nil { + return nil, err + } } - return + t := time.NewTicker(searchTimeout) + defer t.Stop() + + select { + case <-t.C: + log.Trace(fmt.Sprintf("DPA.Get: %v request time out ", key.Log())) + return nil, notFound + case <-chunk.ReqC: + } + return chunk, nil } // Put is the entrypoint for local store requests coming from storeLoop -func (self *dpaChunkStore) Put(entry *Chunk) { - chunk, err := self.localStore.Get(entry.Key) - if err != nil { - log.Trace(fmt.Sprintf("DPA.Put: %v new chunk. call netStore.Put", entry.Key.Log())) - chunk = entry - } else if chunk.SData == nil { - log.Trace(fmt.Sprintf("DPA.Put: %v request entry found", entry.Key.Log())) - chunk.SData = entry.SData - chunk.Size = entry.Size - } else { - log.Trace(fmt.Sprintf("DPA.Put: %v chunk already known", entry.Key.Log())) - return - } - // from this point on the storage logic is the same with network storage requests - log.Trace(fmt.Sprintf("DPA.Put %v: %v", self.n, chunk.Key.Log())) - self.n++ - self.netStore.Put(chunk) +func (self *dpaChunkStore) Put(chunk *Chunk) { + self.localStore.Put(chunk) } // Close chunk store diff --git a/swarm/storage/forwarder.go b/swarm/storage/forwarder.go deleted file mode 100644 index 0d1acfab57..0000000000 --- a/swarm/storage/forwarder.go +++ /dev/null @@ -1,16 +0,0 @@ -package storage - -// implements CloudStore -// noop placeholder for netstore functionality - -type Forwarder struct { -} - -func (self *Forwarder) Store(chunk *Chunk) { -} - -func (self *Forwarder) Retrieve(chunk *Chunk) { -} - -func (self *Forwarder) Deliver(chunk *Chunk) { -} diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index c70271a09a..843505c2cb 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -18,6 +18,9 @@ package storage import ( "encoding/binary" + "fmt" + + "github.com/ethereum/go-ethereum/log" ) // LocalStore is a combination of inmemory db over a disk persisted db @@ -66,6 +69,26 @@ func (self *LocalStore) Get(key Key) (chunk *Chunk, err error) { return } +// retrieve logic common for local and network chunk retrieval requests +func (self *LocalStore) GetOrCreateRequest(key Key) (chunk *Chunk, created bool) { + var err error + chunk, err = self.Get(key) + if err == nil { + if chunk.ReqC == nil { + log.Trace(fmt.Sprintf("LocalStore.GetOrRetrieve: %v found locally", key)) + } else { + log.Trace(fmt.Sprintf("LocalStore.GetOrRetrieve: %v hit on an existing request", key)) + // no need to launch again + } + return chunk, false + } + // no data and no request status + log.Trace(fmt.Sprintf("LocalStore.GetOrRetrieve: %v not found locally. open new request", key)) + chunk = NewChunk(key, make(chan bool)) + self.memStore.Put(chunk) + return chunk, true +} + // Close local store func (self *LocalStore) Close() { self.DbStore.Close() diff --git a/swarm/storage/memstore.go b/swarm/storage/memstore.go index fed1ae0094..e8e393baa9 100644 --- a/swarm/storage/memstore.go +++ b/swarm/storage/memstore.go @@ -168,8 +168,8 @@ func (s *MemStore) Put(entry *Chunk) { entry.Size = node.entry.Size entry.SData = node.entry.SData } - if entry.Req == nil { - entry.Req = node.entry.Req + if entry.ReqC == nil { + entry.ReqC = node.entry.ReqC } entry.C = node.entry.C node.entry = entry diff --git a/swarm/storage/netstore.go b/swarm/storage/netstore.go index 2afba75811..4a7caf7c2e 100644 --- a/swarm/storage/netstore.go +++ b/swarm/storage/netstore.go @@ -17,38 +17,43 @@ package storage import ( - "fmt" "path/filepath" "time" - - "github.com/ethereum/go-ethereum/log" ) -/* -NetStore is a cloud storage access abstaction layer for swarm -it contains the shared logic of network served chunk store/retrieval requests -both local (coming from DPA api) and remote (coming from peers via bzz protocol) -it implements the ChunkStore interface and embeds LocalStore +// import ( +// "fmt" +// "path/filepath" +// "time" -It is called by the bzz protocol instances via Depo (the store/retrieve request handler) -a protocol instance is running on each peer, so this is heavily parallelised. -NetStore falls back to a backend (CloudStorage interface) -implemented by bzz/network/forwarder. forwarder or IPFS or IPΞS -*/ -type NetStore struct { - hashfunc SwarmHasher - localStore *LocalStore - cloud CloudStore -} +// "github.com/ethereum/go-ethereum/log" +// ) -// backend engine for cloud store -// It can be aggregate dispatching to several parallel implementations: -// bzz/network/forwarder. forwarder or IPFS or IPΞS -type CloudStore interface { - Store(*Chunk) - Deliver(*Chunk) - Retrieve(*Chunk) -} +// /* +// NetStore is a cloud storage access abstaction layer for swarm +// it contains the shared logic of network served chunk store/retrieval requests +// both local (coming from DPA api) and remote (coming from peers via bzz protocol) +// it implements the ChunkStore interface and embeds LocalStore + +// It is called by the bzz protocol instances via Depo (the store/retrieve request handler) +// a protocol instance is running on each peer, so this is heavily parallelised. +// NetStore falls back to a backend (CloudStorage interface) +// implemented by bzz/network/forwarder. forwarder or IPFS or IPΞS +// */ +// type NetStore struct { +// hashfunc SwarmHasher +// localStore *LocalStore +// cloud CloudStore +// } + +// // backend engine for cloud store +// // It can be aggregate dispatching to several parallel implementations: +// // bzz/network/forwarder. forwarder or IPFS or IPΞS +// type CloudStore interface { +// Store(*Chunk) +// Deliver(*Chunk) +// Retrieve(*Chunk) +// } type StoreParams struct { ChunkDbPath string @@ -72,70 +77,56 @@ func (self *StoreParams) Init(path string) { self.ChunkDbPath = filepath.Join(path, "chunks") } -// netstore contructor, takes path argument that is used to initialise dbStore, -// the persistent (disk) storage component of LocalStore -// the second argument is the hive, the connection/logistics manager for the node -func NewNetStore(hash SwarmHasher, lstore *LocalStore, cloud CloudStore, params *StoreParams) *NetStore { - return &NetStore{ - hashfunc: hash, - localStore: lstore, - cloud: cloud, - } -} +// // netstore contructor, takes path argument that is used to initialise dbStore, +// // the persistent (disk) storage component of LocalStore +// // the second argument is the hive, the connection/logistics manager for the node +// func NewNetStore(hash SwarmHasher, lstore *LocalStore, cloud CloudStore, params *StoreParams) *NetStore { +// return &NetStore{ +// hashfunc: hash, +// localStore: lstore, +// cloud: cloud, +// } +// } -const ( - // maximum number of peers that a retrieved message is delivered to - requesterCount = 3 -) +// const ( +// // maximum number of peers that a retrieved message is delivered to +// requesterCount = 3 +// ) var ( // timeout interval before retrieval is timed out searchTimeout = 3 * time.Second ) -// store logic common to local and network chunk store requests -// ~ unsafe put in localdb no check if exists no extra copy no hash validation -// the chunk is forced to propagate (Cloud.Store) even if locally found! -// caller needs to make sure if that is wanted -func (self *NetStore) Put(entry *Chunk) { - self.localStore.Put(entry) +// // store logic common to local and network chunk store requests +// // ~ unsafe put in localdb no check if exists no extra copy no hash validation +// // the chunk is forced to propagate (Cloud.Store) even if locally found! +// // caller needs to make sure if that is wanted +// func (self *NetStore) Put(entry *Chunk) { +// self.localStore.Put(entry) - // handle deliveries - if entry.Req != nil { - log.Trace(fmt.Sprintf("NetStore.Put: localStore.Put %v hit existing request...delivering", entry.Key.Log())) - // closing C signals to other routines (local requests) - // that the chunk is has been retrieved - close(entry.Req.C) - // deliver the chunk to requesters upstream - go self.cloud.Deliver(entry) - } else { - log.Trace(fmt.Sprintf("NetStore.Put: localStore.Put %v stored locally", entry.Key.Log())) - // handle propagating store requests - // go self.cloud.Store(entry) - go self.cloud.Store(entry) - } -} +// // handle deliveries +// if entry.ReqC != nil { +// log.Trace(fmt.Sprintf("NetStore.Put: localStore.Put %v hit existing request...delivering", entry.Key.Log())) +// // closing C signals to other routines (local requests) +// // that the chunk is has been retrieved +// close(entry.ReqC) +// // deliver the chunk to requesters upstream +// go self.cloud.Deliver(entry) +// } else { +// log.Trace(fmt.Sprintf("NetStore.Put: localStore.Put %v stored locally", entry.Key.Log())) +// // handle propagating store requests +// // go self.cloud.Store(entry) +// go self.cloud.Store(entry) +// } +// } -// retrieve logic common for local and network chunk retrieval requests -func (self *NetStore) Get(key Key) (*Chunk, error) { - var err error - chunk, err := self.localStore.Get(key) - if err == nil { - if chunk.Req == nil { - log.Trace(fmt.Sprintf("NetStore.Get: %v found locally", key)) - } else { - log.Trace(fmt.Sprintf("NetStore.Get: %v hit on an existing request", key)) - // no need to launch again - } - return chunk, err - } - // no data and no request status - log.Trace(fmt.Sprintf("NetStore.Get: %v not found locally. open new request", key)) - chunk = NewChunk(key, NewRequestStatus(key)) - self.localStore.memStore.Put(chunk) - go self.cloud.Retrieve(chunk) - return chunk, nil -} +// // retrieve logic common for local and network chunk retrieval requests +// func (self *NetStore) Get(key Key) (*Chunk, error) { +// chunk, _ := self.localStore.GetOrCreateRequest(key) +// go self.cloud.Retrieve(chunk) +// return chunk, nil +// } -// Close netstore -func (self *NetStore) Close() {} +// // Close netstore +// func (self *NetStore) Close() {} diff --git a/swarm/storage/types.go b/swarm/storage/types.go index 833a1b4262..73a4f758d6 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -166,44 +166,23 @@ func (c KeyCollection) Swap(i, j int) { c[i], c[j] = c[j], c[i] } -// each chunk when first requested opens a record associated with the request -// next time a request for the same chunk arrives, this record is updated -// this request status keeps track of the request ID-s as well as the requesting -// peers and has a channel that is closed when the chunk is retrieved. Multiple -// local callers can wait on this channel (or combined with a timeout, block with a -// select). -type RequestStatus struct { - Key Key - Source Peer - C chan bool - Requesters map[uint64][]interface{} -} - -func NewRequestStatus(key Key) *RequestStatus { - return &RequestStatus{ - Key: key, - Requesters: make(map[uint64][]interface{}), - C: make(chan bool), - } -} - // Chunk also serves as a request object passed to ChunkStores // in case it is a retrieval request, Data is nil and Size is 0 // Note that Size is not the size of the data chunk, which is Data.Size() // but the size of the subtree encoded in the chunk // 0 if request, to be supplied by the dpa type Chunk struct { - Key Key // always - SData []byte // nil if request, to be supplied by dpa - Size int64 // size of the data covered by the subtree encoded in this chunk - Source Peer // peer - C chan bool // to signal data delivery by the dpa - Req *RequestStatus // request Status needed by netStore - dbStored chan bool // never remove a chunk from memStore before it is written to dbStore + Key Key // always + SData []byte // nil if request, to be supplied by dpa + Size int64 // size of the data covered by the subtree encoded in this chunk + //Source Peer // peer + C chan bool // to signal data delivery by the dpa + ReqC chan bool // to signal the request done + dbStored chan bool // never remove a chunk from memStore before it is written to dbStore } -func NewChunk(key Key, rs *RequestStatus) *Chunk { - return &Chunk{Key: key, Req: rs, dbStored: make(chan bool)} +func NewChunk(key Key, reqC chan bool) *Chunk { + return &Chunk{Key: key, ReqC: reqC, dbStored: make(chan bool)} } func (c *Chunk) WaitToStore() { diff --git a/swarm/swarm.go b/swarm/swarm.go index b2349c0fc4..11f5cf25f2 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -49,10 +49,11 @@ type Swarm struct { api *api.Api // high level api layer (fs/manifest) dns api.Resolver // DNS registrar //dbAccess *network.DbAccess // access to local chunk db iterator and storage counter - storage storage.ChunkStore // internal access to storage, common interface to cloud storage backends - dpa *storage.DPA // distributed preimage archive, the local API to the storage with document level storage/retrieval support + //storage storage.ChunkStore // internal access to storage, common interface to cloud storage backends + dpa *storage.DPA // distributed preimage archive, the local API to the storage with document level storage/retrieval support //depo network.StorageHandler // remote request handler, interface between bzz protocol and the storage - cloud storage.CloudStore // procurement, cloud storage backend (can multi-cloud) + streamer *network.Streamer + //cloud storage.CloudStore // procurement, cloud storage backend (can multi-cloud) bzz *network.Bzz // the logistic manager backend chequebook.Backend // simple blockchain Backend privateKey *ecdsa.PrivateKey @@ -114,8 +115,8 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e config.HiveParams.Discovery = true // setup cloud storage internal access layer - self.cloud = &storage.Forwarder{} - self.storage = storage.NewNetStore(hash, self.lstore, self.cloud, config.StoreParams) + //self.cloud = &storage.Forwarder{} + //self.storage = storage.NewNetStore(hash, self.lstore, self.cloud, config.StoreParams) log.Debug(fmt.Sprintf("-> swarm net store shared access layer to Swarm Chunk Store")) nodeid := discover.PubkeyID(crypto.ToECDSAPub(common.FromHex(config.PublicKey))) addr := network.NewAddrFromNodeID(nodeid) @@ -124,10 +125,16 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e UnderlayAddr: addr.UAddr, HiveParams: config.HiveParams, } - self.bzz = network.NewBzz(bzzconfig, to, nil) + + dbAccess := network.NewDbAccess(self.lstore) + self.streamer = network.NewStreamer(to, dbAccess) + network.RegisterOutgoingSyncers(self.streamer, dbAccess) + network.RegisterIncomingSyncers(self.streamer, dbAccess) + + self.bzz = network.NewBzz(bzzconfig, to, nil, self.streamer) // set up DPA, the cloud storage local access layer - dpaChunkStore := storage.NewDpaChunkStore(self.lstore, self.storage) + dpaChunkStore := storage.NewDpaChunkStore(self.lstore, self.streamer.Retrieve) log.Debug(fmt.Sprintf("-> Local Access to Swarm")) // Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage self.dpa = storage.NewDPA(dpaChunkStore, self.config.ChunkerParams) From 03655f7b7886b6a6cae2911b885440f4dd7ab315 Mon Sep 17 00:00:00 2001 From: zelig Date: Tue, 9 Jan 2018 15:05:21 +0100 Subject: [PATCH 017/210] swarm/storage, swarm/network: light mode request streamers --- swarm/network/lightnode.go | 191 +++++++++++++++++++++++++++++++ swarm/network/streamer.go | 206 +++++++++++++++++++++------------- swarm/network/syncer.go | 197 +++++++++++++++++++++----------- swarm/storage/chunker.go | 19 ++-- swarm/storage/chunker_test.go | 20 ++-- swarm/storage/dpa.go | 2 +- swarm/storage/types.go | 2 +- 7 files changed, 476 insertions(+), 161 deletions(-) create mode 100644 swarm/network/lightnode.go diff --git a/swarm/network/lightnode.go b/swarm/network/lightnode.go new file mode 100644 index 0000000000..ca1463b4d4 --- /dev/null +++ b/swarm/network/lightnode.go @@ -0,0 +1,191 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library.d +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package network + +import ( + "errors" + + "github.com/ethereum/go-ethereum/swarm/storage" +) + +// RemoteReader implements IncomingStreamer +type RemoteSectionReader struct { + db *DbAccess + start uint64 + end uint64 + hashes chan []byte + currentHashes []byte + currentData []byte + quit chan struct{} + root []byte +} + +// NewRemoteReader is the constructor for RemoteReader +func NewRemoteSectionReader(root []byte, db *DbAccess) *RemoteSectionReader { + return &RemoteSectionReader{ + db: db, + root: root, + hashes: make(chan []byte), + quit: make(chan struct{}), + } +} + +func (r *RemoteSectionReader) NeedData(key []byte) func() { + chunk, created := r.db.getOrCreateRequest(storage.Key(key)) + // TODO: we may want to request from this peer anyway even if the request exists + if chunk.ReqC == nil || !created { + return nil + } + return func() {} +} + +func (r *RemoteSectionReader) NextBatch(from uint64) (nextFrom uint64, nextTo uint64) { + return from, r.end +} + +func (r *RemoteSectionReader) BatchDone(s Stream, from uint64, hashes []byte, root []byte) func() (*TakeoverProof, error) { + r.hashes <- hashes + return nil +} + +func (r *RemoteSectionReader) Read(b []byte) (n int64, err error) { + l := int64(len(b)) + m := int64(len(r.currentData)) + if m > l { + m = l + } + copy(b, r.currentData[:m]) + if m == l { + r.currentData = r.currentData[m:] + return l, nil + } + var end bool + for i := 0; !end && i < len(r.currentHashes); i += HashSize { + hash := r.currentHashes[i : i+HashSize] + chunk, err := r.db.get(hash) + if err != nil { + return n, err + } + m := chunk.Size + if n+m > l { + m = l - n + end = true + } + copy(b[n:], chunk.SData[:m]) + n += int64(m) + } + + for { + select { + case <-r.quit: + return n, errors.New("aborted") + case hashes := <-r.hashes: + var i int + for ; !end && i < len(hashes); i += HashSize { + hash := hashes[i : i+HashSize] + chunk, err := r.db.get(hash) + if err != nil { + return n, err + } + m := chunk.Size + if n+m > l { + m = l - n + end = true + + } + copy(b[n:], chunk.SData[:m]) + n += m + } + hashes = hashes[i:] + } + } + return n, nil +} + +// RemoteSectionServer implements OutgoingStreamer +type RemoteSectionServer struct { + // quit chan struct{} + currentBatch []byte + root []byte + db *DbAccess + r *storage.LazyChunkReader +} + +// NewRemoteReader is the constructor for RemoteReader +func NewRemoteSectionServer(db *DbAccess, r *storage.LazyChunkReader) *RemoteSectionServer { + return &RemoteSectionServer{ + db: db, + r: r, + } +} + +// GetData retrieves the actual chunk from localstore +func (s *RemoteSectionServer) GetData(key []byte) []byte { + chunk, err := s.db.get(storage.Key(key)) + if err != nil { + return nil + } + return chunk.SData +} + +// GetBatch retrieves the next batch of hashes from the dbstore +func (s *RemoteSectionServer) SetNextBatch(from, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) { + if to > from+batchSize { + to = from + batchSize + } + batch := make([]byte, (to-from)*HashSize) + s.r.ReadAt(batch, int64(from)) + s.currentBatch = batch + return batch, from, to, nil, nil +} + +// RegisterRemoteSectionReader registers RemoteSectionReader on light downstream node +func RegisterRemoteSectionReader(s *Streamer, db *DbAccess) { + name := Stream("REMOTE_SECTION") + s.RegisterIncomingStreamer(name, func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { + return NewRemoteSectionReader(t, db), nil + }) +} + +// RegisterRemoteSectionServer registers RemoteSectionServer outgoing streamer on +// upstream light server node +func RegisterRemoteSectionServer(s *Streamer, db *DbAccess, rf func([]byte) *storage.LazyChunkReader) { + name := Stream("REMOTE_SECTION") + s.RegisterOutgoingStreamer(name, func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) { + r := rf(t) + return NewRemoteSectionServer(db, r), nil + }) +} + +// RegisterRemoteDownloader registers RemoteDownloader incoming streamer +// on downstream light node +func RegisterRemoteDownloader(s *Streamer, db *DbAccess) { + name := Stream("REMOTE_DOWNLOADER") + s.RegisterIncomingStreamer(name, func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { + return NewRemoteDownloader(t, db), nil + }) +} + +// RegisterRemoteDownloadServer registers RemoteDownloadServer outgoing streamer on +// upstream light server node +func RegisterRemoteDownloadServer(s *Streamer, db *DbAccess, rf func([]byte) *storage.LazyChunkReader) { + name := Stream("REMOTE_DOWNLOADER") + s.RegisterOutgoingStreamer(name, func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) { + r := rf(t) + return NewRemoteDownloadServer(db, r), nil + }) +} diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go index 19f948b4fa..bb6ace62b7 100644 --- a/swarm/network/streamer.go +++ b/swarm/network/streamer.go @@ -34,7 +34,7 @@ import ( const ( HashSize = 32 - Low int = iota + Low uint8 = iota Mid High Top @@ -80,6 +80,7 @@ func (self TakeoverProofMsg) String() string { // SubcribeMsg is the protocol msg for requesting a stream(section) type SubscribeMsg struct { Stream Stream + Key []byte From, To uint64 Priority uint8 // delivered on priority channel } @@ -88,6 +89,7 @@ type SubscribeMsg struct { // stream section type UnsyncedKeysMsg struct { Stream Stream // name of Stream + Key []byte // subtype or key From, To uint64 // peer and db-specific entry count Hashes []byte // stream of hashes (128) *HandoverProof // HandoverProof @@ -114,6 +116,7 @@ func (self UnsyncedKeysMsg) String() string { // offered in UnsyncedKeysMsg downstream peer actually wants sent over type WantedKeysMsg struct { Stream Stream // name of stream + Key []byte // subtype or key Want []byte // bitvector indicating which keys of the batch needed From, To uint64 // next interval offset - empty if not to be continued } @@ -127,8 +130,8 @@ func (self WantedKeysMsg) String() string { type Streamer struct { incomingLock sync.RWMutex outgoingLock sync.RWMutex - outgoing map[Stream]func(*StreamerPeer) (OutgoingStreamer, error) - incoming map[Stream]func(*StreamerPeer) (IncomingStreamer, error) + outgoing map[Stream]func(*StreamerPeer, []byte) (OutgoingStreamer, error) + incoming map[Stream]func(*StreamerPeer, []byte) (IncomingStreamer, error) dbAccess *DbAccess overlay Overlay @@ -138,8 +141,8 @@ type Streamer struct { // NewStreamer is Streamer constructor func NewStreamer(overlay Overlay, dbAccess *DbAccess) *Streamer { return &Streamer{ - outgoing: make(map[Stream]func(*StreamerPeer) (OutgoingStreamer, error)), - incoming: make(map[Stream]func(*StreamerPeer) (IncomingStreamer, error)), + outgoing: make(map[Stream]func(*StreamerPeer, []byte) (OutgoingStreamer, error)), + incoming: make(map[Stream]func(*StreamerPeer, []byte) (IncomingStreamer, error)), dbAccess: dbAccess, overlay: overlay, receiveC: make(chan *ChunkDeliveryMsg, 10), @@ -147,21 +150,21 @@ func NewStreamer(overlay Overlay, dbAccess *DbAccess) *Streamer { } // RegisterIncomingStreamer registers an incoming streamer constructor -func (self *Streamer) RegisterIncomingStreamer(stream Stream, f func(*StreamerPeer) (IncomingStreamer, error)) { +func (self *Streamer) RegisterIncomingStreamer(stream Stream, f func(*StreamerPeer, []byte) (IncomingStreamer, error)) { self.incomingLock.Lock() defer self.incomingLock.Unlock() self.incoming[stream] = f } // RegisterOutgoingStreamer registers an outgoing streamer constructor -func (self *Streamer) RegisterOutgoingStreamer(stream Stream, f func(*StreamerPeer) (OutgoingStreamer, error)) { +func (self *Streamer) RegisterOutgoingStreamer(stream Stream, f func(*StreamerPeer, []byte) (OutgoingStreamer, error)) { self.outgoingLock.Lock() defer self.outgoingLock.Unlock() self.outgoing[stream] = f } // GetIncomingStreamer accessor for incoming streamer constructors -func (self *Streamer) GetIncomingStreamer(stream Stream) (func(*StreamerPeer) (IncomingStreamer, error), error) { +func (self *Streamer) GetIncomingStreamer(stream Stream) (func(*StreamerPeer, []byte) (IncomingStreamer, error), error) { self.incomingLock.RLock() defer self.incomingLock.RUnlock() f := self.incoming[stream] @@ -172,7 +175,7 @@ func (self *Streamer) GetIncomingStreamer(stream Stream) (func(*StreamerPeer) (I } // GetOutgoingStreamer accessor for incoming streamer constructors -func (self *Streamer) GetOutgoingStreamer(stream Stream) (func(*StreamerPeer) (OutgoingStreamer, error), error) { +func (self *Streamer) GetOutgoingStreamer(stream Stream) (func(*StreamerPeer, []byte) (OutgoingStreamer, error), error) { self.outgoingLock.RLock() defer self.outgoingLock.RUnlock() f := self.outgoing[stream] @@ -190,19 +193,30 @@ func (self *Streamer) PeerInfo(id discover.NodeID) interface{} { return nil } +type outgoingStreamer struct { + OutgoingStreamer + priority uint8 + currentBatch []byte +} + // OutgoingStreamer interface for outgoing peer Streamer type OutgoingStreamer interface { - CurrentBatch() []byte SetNextBatch(uint64, uint64) (hashes []byte, from uint64, to uint64, proof *HandoverProof, err error) GetData([]byte) []byte - Priority() int +} + +type incomingStreamer struct { + IncomingStreamer + priority uint8 + quit chan struct{} + next chan struct{} } // IncomingStreamer interface for incoming peer Streamer type IncomingStreamer interface { NextBatch(uint64) (uint64, uint64) NeedData([]byte) func() - Priority() int + BatchDone(Stream, uint64, []byte, []byte) func() (*TakeoverProof, error) } // StreamerPeer is the Peer extention for the streaming protocol @@ -214,8 +228,8 @@ type StreamerPeer struct { dbAccess *DbAccess outgoingLock sync.RWMutex incomingLock sync.RWMutex - outgoing map[Stream]OutgoingStreamer - incoming map[Stream]IncomingStreamer + outgoing map[Stream]*outgoingStreamer + incoming map[Stream]*incomingStreamer quit chan struct{} } @@ -232,10 +246,10 @@ type StreamerPeer struct { // NewStreamerPeer is the constructor for StreamerPeer func NewStreamerPeer(p Peer, streamer *Streamer) *StreamerPeer { self := &StreamerPeer{ - pq: pq.New(PriorityQueue, PriorityQueueCap), + pq: pq.New(int(PriorityQueue), PriorityQueueCap), streamer: streamer, - outgoing: make(map[Stream]OutgoingStreamer), - incoming: make(map[Stream]IncomingStreamer), + outgoing: make(map[Stream]*outgoingStreamer), + incoming: make(map[Stream]*incomingStreamer), quit: make(chan struct{}), } ctx, cancel := context.WithCancel(context.Background()) @@ -247,38 +261,41 @@ func NewStreamerPeer(p Peer, streamer *Streamer) *StreamerPeer { return self } +// RetrieveRequestMsg is the protocol msg for chunk retrieve requests type RetrieveRequestMsg struct { Key storage.Key } // RetrieveRequestStreamer implements OutgoingStreamer type RetrieveRequestStreamer struct { - deliveryC chan *storage.Chunk - batchC chan []byte - dbAccess *DbAccess - currentBatch []byte - currentLen uint64 + deliveryC chan *storage.Chunk + batchC chan []byte + db *DbAccess + currentLen uint64 } -func RegisterRequestStreamer(streamer *Streamer, dbAccess *DbAccess) { - streamer.RegisterOutgoingStreamer(retrieveRequestStream, func(_ *StreamerPeer) (OutgoingStreamer, error) { - return NewRetrieveRequestStreamer(dbAccess), nil +// RegisterRequestStreamer registers outgoing and incoming streamers for request handling +func RegisterRequestStreamer(streamer *Streamer, db *DbAccess) { + streamer.RegisterOutgoingStreamer(retrieveRequestStream, func(_ *StreamerPeer, t []byte) (OutgoingStreamer, error) { + return NewRetrieveRequestStreamer(db), nil }) - streamer.RegisterIncomingStreamer(retrieveRequestStream, func(p *StreamerPeer) (IncomingStreamer, error) { - return NewIncomingSwarmSyncer(Top, nil, p, dbAccess, nil) + streamer.RegisterIncomingStreamer(retrieveRequestStream, func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { + return NewIncomingSwarmSyncer(nil, p, db, nil) }) } -func NewRetrieveRequestStreamer(dbAccess *DbAccess) *RetrieveRequestStreamer { +// NewRetrieveRequestStreamer is RetrieveRequestStreamer constructor +func NewRetrieveRequestStreamer(db *DbAccess) *RetrieveRequestStreamer { s := &RetrieveRequestStreamer{ deliveryC: make(chan *storage.Chunk), batchC: make(chan []byte), - dbAccess: dbAccess, + db: db, } go s.processDeliveries() return s } +// processDeliveries handles delivered chunk hashes func (s *RetrieveRequestStreamer) processDeliveries() { var hashes []byte for { @@ -291,28 +308,21 @@ func (s *RetrieveRequestStreamer) processDeliveries() { } } -func (s *RetrieveRequestStreamer) CurrentBatch() []byte { - return s.currentBatch -} - +// SetNextBatch func (s *RetrieveRequestStreamer) SetNextBatch(_, _ uint64) (hashes []byte, from uint64, to uint64, proof *HandoverProof, err error) { hashes = <-s.batchC - s.currentBatch = hashes from = s.currentLen s.currentLen += uint64(len(hashes)) to = s.currentLen return } +// GetData retrives chunk data from db store func (s *RetrieveRequestStreamer) GetData(key []byte) []byte { - chunk, _ := s.dbAccess.get(storage.Key(key)) + chunk, _ := s.db.get(storage.Key(key)) return chunk.SData } -func (s *RetrieveRequestStreamer) Priority() int { - return Top -} - const retrieveRequestStream = Stream("RETRIEVE_REQUEST") func (self *StreamerPeer) handleRetrieveRequestMsg(req *RetrieveRequestMsg) error { @@ -321,7 +331,7 @@ func (self *StreamerPeer) handleRetrieveRequestMsg(req *RetrieveRequestMsg) erro if err != nil { return err } - streamer := s.(*RetrieveRequestStreamer) + streamer := s.OutgoingStreamer.(*RetrieveRequestStreamer) if chunk.ReqC != nil { if created { if err := self.streamer.Retrieve(chunk); err != nil { @@ -349,10 +359,16 @@ func (self *StreamerPeer) handleRetrieveRequestMsg(req *RetrieveRequestMsg) erro return nil } +// Retrieve sends a chunk retrieve request to func (self *Streamer) Retrieve(chunk *storage.Chunk) error { - // TODO: using the overlay find the closes peer to send the retrieve - // request to. - // self.Overlay.EachConn(to, 256, func(op network.OverlayConn, po int, isproxbin bool) bool {}) + self.overlay.EachConn(chunk.Key[:], 255, func(p OverlayConn, po int, nn bool) bool { + sp := p.(*StreamerPeer) + // TODO: skip light nodes that do not accept retrieve requests + sp.SendPriority(&RetrieveRequestMsg{ + Key: chunk.Key[:], + }, Top) + return false + }) return nil } @@ -383,7 +399,7 @@ func (self *Streamer) processReceivedChunks() { } } -func (self *StreamerPeer) getOutgoingStreamer(s Stream) (OutgoingStreamer, error) { +func (self *StreamerPeer) getOutgoingStreamer(s Stream) (*outgoingStreamer, error) { self.outgoingLock.RLock() defer self.outgoingLock.RUnlock() streamer := self.outgoing[s] @@ -393,7 +409,7 @@ func (self *StreamerPeer) getOutgoingStreamer(s Stream) (OutgoingStreamer, error return streamer, nil } -func (self *StreamerPeer) getIncomingStreamer(s Stream) (IncomingStreamer, error) { +func (self *StreamerPeer) getIncomingStreamer(s Stream) (*incomingStreamer, error) { self.incomingLock.RLock() defer self.incomingLock.RUnlock() streamer := self.incoming[s] @@ -403,44 +419,59 @@ func (self *StreamerPeer) getIncomingStreamer(s Stream) (IncomingStreamer, error return streamer, nil } -func (self *StreamerPeer) setOutgoingStreamer(s Stream, o OutgoingStreamer) error { +func (self *StreamerPeer) setOutgoingStreamer(s Stream, o OutgoingStreamer, priority uint8) (*outgoingStreamer, error) { self.outgoingLock.Lock() defer self.outgoingLock.Unlock() if self.outgoing[s] != nil { - return fmt.Errorf("stream %v already registered", s) + return nil, fmt.Errorf("stream %v already registered", s) } - self.outgoing[s] = o - return nil + os := &outgoingStreamer{ + OutgoingStreamer: o, + priority: priority, + } + self.outgoing[s] = os + return os, nil } -func (self *StreamerPeer) setIncomingStreamer(s Stream, i IncomingStreamer) error { +func (self *StreamerPeer) setIncomingStreamer(s Stream, i IncomingStreamer, priority uint8) error { self.incomingLock.Lock() defer self.incomingLock.Unlock() if self.incoming[s] != nil { return fmt.Errorf("stream %v already registered", s) } - self.incoming[s] = i + next := make(chan struct{}, 1) + self.incoming[s] = &incomingStreamer{ + IncomingStreamer: i, + priority: priority, + next: next, + } + next <- struct{}{} // this is to allow wantedKeysMsg before first batch arrives return nil } // Subscribe initiates the streamer -func (self *StreamerPeer) Subscribe(s Stream, from, to uint64) error { +func (self *StreamerPeer) Subscribe(s Stream, t []byte, from, to uint64, priority uint8) error { f, err := self.streamer.GetIncomingStreamer(s) if err != nil { return err } - is, err := f(self) + is, err := f(self, t) if err != nil { return err } - self.setIncomingStreamer(s, is) + err = self.setIncomingStreamer(s, is, priority) + if err != nil { + return err + } + msg := &SubscribeMsg{ Stream: s, + Key: t, From: from, To: to, - Priority: uint8(is.Priority()), + Priority: priority, } - self.SendPriority(msg, is.Priority()) + self.SendPriority(msg, priority) return nil } @@ -449,14 +480,16 @@ func (self *StreamerPeer) handleSubscribeMsg(req *SubscribeMsg) error { if err != nil { return err } - s, err := f(self) + s, err := f(self, req.Key) if err != nil { return err } - if err := self.setOutgoingStreamer(req.Stream, s); err != nil { + key := string(req.Stream) + string(req.Key) + os, err := self.setOutgoingStreamer(Stream(key), s, req.Priority) + if err != nil { return nil } - self.SendUnsyncedKeys(s, req.From, req.To, int(req.Priority)) + go self.SendUnsyncedKeys(os, req.From, req.To) return nil } @@ -485,11 +518,17 @@ func (self *StreamerPeer) handleUnsyncedKeysMsg(req *UnsyncedKeysMsg) error { }(wait) } } - // go func() { - // wg.Wait() - // msg := s.TakeoverProof(req.Stream, req.From, req.Hashes, req.Root) - // self.Send(msg, s.Priority()) - // }() + go func() { + wg.Wait() + if tf := s.BatchDone(req.Stream, req.From, hashes, req.Root); tf != nil { + tp, err := tf() + if err != nil { + return + } + self.SendPriority(tp, s.priority) + } + s.next <- struct{}{} + }() // only send wantedKeysMsg if all missing chunks of the previous batch arrived // except from, to := s.NextBatch(req.To) @@ -502,7 +541,14 @@ func (self *StreamerPeer) handleUnsyncedKeysMsg(req *UnsyncedKeysMsg) error { From: from, To: to, } - self.SendPriority(msg, s.Priority()) + go func() { + select { + case <-s.next: + case <-s.quit: + return + } + self.SendPriority(msg, s.priority) + }() return nil } @@ -514,9 +560,9 @@ func (self *StreamerPeer) handleWantedKeysMsg(req *WantedKeysMsg) error { if err != nil { return err } - hashes := s.CurrentBatch() + hashes := s.currentBatch // launch in go routine since GetBatch blocks until new hashes arrive - go self.SendUnsyncedKeys(s, req.From, req.To, s.Priority()) + go self.SendUnsyncedKeys(s, req.From, req.To) l := len(hashes) / HashSize want, err := bv.NewFromBytes(req.Want, l) if err != nil { @@ -531,7 +577,7 @@ func (self *StreamerPeer) handleWantedKeysMsg(req *WantedKeysMsg) error { } chunk := storage.NewChunk(hash, nil) chunk.SData = data - if err := self.Deliver(chunk, s.Priority()); err != nil { + if err := self.Deliver(chunk, s.priority); err != nil { return err } } @@ -549,32 +595,33 @@ func (self *StreamerPeer) handleTakeoverProofMsg(req *TakeoverProofMsg) error { } // Deliver sends a storeRequestMsg protocol message to the peer -func (self *StreamerPeer) Deliver(chunk *storage.Chunk, priority int) error { +func (self *StreamerPeer) Deliver(chunk *storage.Chunk, priority uint8) error { msg := &ChunkDeliveryMsg{ Key: chunk.Key, SData: chunk.SData, } - return self.pq.Push(nil, msg, priority) + return self.pq.Push(nil, msg, int(priority)) } // Deliver sends a storeRequestMsg protocol message to the peer -func (self *StreamerPeer) SendPriority(msg interface{}, priority int) error { - return self.pq.Push(nil, msg, priority) +func (self *StreamerPeer) SendPriority(msg interface{}, priority uint8) error { + return self.pq.Push(nil, msg, int(priority)) } // UnsyncedKeys sends UnsyncedKeysMsg protocol msg -func (self *StreamerPeer) SendUnsyncedKeys(s OutgoingStreamer, f, t uint64, priority int) error { +func (self *StreamerPeer) SendUnsyncedKeys(s *outgoingStreamer, f, t uint64) error { hashes, from, to, proof, err := s.SetNextBatch(f, t) if err != nil { return err } + s.currentBatch = hashes msg := &UnsyncedKeysMsg{ HandoverProof: proof, Hashes: hashes, From: from, To: to, } - return self.SendPriority(msg, s.Priority()) + return self.SendPriority(msg, s.priority) } // StreamerSpec is the spec of the streamer protocol. @@ -595,10 +642,13 @@ var StreamerSpec = &protocols.Spec{ func (s *Streamer) Run(p *bzzPeer) error { sp := NewStreamerPeer(p, s) // load saved intervals - sp.handleSubscribeMsg(&SubscribeMsg{ - Stream: retrieveRequestStream, - Priority: uint8(Top), - }) + // autosubscribe to request handler to serve request only for non-light nodes + // sp.handleSubscribeMsg(&SubscribeMsg{ + // Stream: retrieveRequestStream, + // Priority: uint8(Top), + // }) + // subscribe to request handling ; only with non-light nodes + sp.Subscribe(retrieveRequestStream, nil, 0, 0, Top) defer close(sp.quit) return sp.Run(sp.HandleMsg) } diff --git a/swarm/network/syncer.go b/swarm/network/syncer.go index ed87e57870..d0da14f78d 100644 --- a/swarm/network/syncer.go +++ b/swarm/network/syncer.go @@ -70,19 +70,24 @@ func (self *DbAccess) put(chunk *storage.Chunk) { // * live request delivery with or without checkback // * (live/non-live historical) chunk syncing per proximity bin type OutgoingSwarmSyncer struct { - po uint8 - db *DbAccess - sessionAt uint64 - currentBatch []byte - priority int + po uint8 + db *DbAccess + sessionAt uint64 + start uint64 } // NewOutgoingSwarmSyncer is contructor for OutgoingSwarmSyncer -func NewOutgoingSwarmSyncer(po uint8, db *DbAccess) (*OutgoingSwarmSyncer, error) { +func NewOutgoingSwarmSyncer(live bool, po uint8, db *DbAccess) (*OutgoingSwarmSyncer, error) { + sessionAt := db.currentBucketStorageIndex(po) + var start uint64 + if live { + start = sessionAt + } self := &OutgoingSwarmSyncer{ po: po, db: db, - sessionAt: db.currentBucketStorageIndex(po), + sessionAt: sessionAt, + start: start, } return self, nil } @@ -90,20 +95,22 @@ func NewOutgoingSwarmSyncer(po uint8, db *DbAccess) (*OutgoingSwarmSyncer, error const maxPO = 32 func RegisterOutgoingSyncers(streamer *Streamer, db *DbAccess) { - for po := uint8(0); po < maxPO; po++ { - stream := Stream(fmt.Sprintf("SYNC-%02d-live", po)) - streamer.RegisterOutgoingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) { - return NewOutgoingSwarmSyncer(po, db) - }) - stream = Stream(fmt.Sprintf("SYNC-%02d-history", po)) - streamer.RegisterOutgoingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) { - return NewOutgoingSwarmSyncer(po, db) - }) - // stream = Stream(fmt.Sprintf("SYNC-%02d-delete", po)) - // streamer.RegisterOutgoingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) { - // return NewOutgoingProvableSwarmSyncer(po, db) - // }) - } + stream := Stream("SYNC") + streamer.RegisterOutgoingStreamer(stream, func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) { + syncType, po := parseSyncLabel(t) + switch syncType { + case "LIVE": + return NewOutgoingSwarmSyncer(true, po, db) + case "HISTORY": + return NewOutgoingSwarmSyncer(false, po, db) + default: + return nil, errors.New("invalid sync type") + } + }) + // stream = Stream(fmt.Sprintf("SYNC-%02d-delete", po)) + // streamer.RegisterOutgoingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) { + // return NewOutgoingProvableSwarmSyncer(po, db) + // }) } // GetSection retrieves the actual chunk from localstore @@ -115,18 +122,13 @@ func (self *OutgoingSwarmSyncer) GetData(key []byte) []byte { return chunk.SData } -func (self *OutgoingSwarmSyncer) CurrentBatch() []byte { - return self.currentBatch -} - -func (self *OutgoingSwarmSyncer) Priority() int { - return self.priority -} - // GetBatch retrieves the next batch of hashes from the dbstore func (self *OutgoingSwarmSyncer) SetNextBatch(from, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) { var batch []byte i := 0 + if from == 0 { + from = self.start + } err := self.db.iterator(from, to, self.po, func(key storage.Key, idx uint64) bool { batch = append(batch, key[:]...) i++ @@ -136,17 +138,15 @@ func (self *OutgoingSwarmSyncer) SetNextBatch(from, to uint64) ([]byte, uint64, if err != nil { return nil, 0, 0, nil, err } - self.currentBatch = batch log.Debug("Swarm batch", "po", self.po, "len", i, "from", from, "to", to) return batch, from, to, nil, nil } // IncomingSwarmSyncer type IncomingSwarmSyncer struct { - priority int sessionAt uint64 nextC chan struct{} - intervals []uint64 + intervals *Intervals sessionRoot storage.Key sessionReader storage.LazySectionReader retrieveC chan *storage.Chunk @@ -159,9 +159,8 @@ type IncomingSwarmSyncer struct { } // NewIncomingSwarmSyncer is a contructor for provable data exchange syncer -func NewIncomingSwarmSyncer(priority int, intervals []uint64, p Peer, dbAccess *DbAccess, chunker storage.Chunker) (*IncomingSwarmSyncer, error) { +func NewIncomingSwarmSyncer(intervals *Intervals, p Peer, dbAccess *DbAccess, chunker storage.Chunker) (*IncomingSwarmSyncer, error) { self := &IncomingSwarmSyncer{ - priority: priority, intervals: intervals, dbAccess: dbAccess, chunker: chunker, @@ -169,10 +168,6 @@ func NewIncomingSwarmSyncer(priority int, intervals []uint64, p Peer, dbAccess * return self, nil } -func (s *IncomingSwarmSyncer) Priority() int { - return s.priority -} - // // NewIncomingProvableSwarmSyncer is a contructor for provable data exchange syncer // func NewIncomingProvableSwarmSyncer(po int, priority int, index uint64, sessionAt uint64, intervals []uint64, sessionRoot storage.Key, chunker *storage.PyramidChunker, store storage.ChunkStore, p Peer) *IncomingSwarmSyncer { // retrieveC := make(storage.Chunk, chunksCap) @@ -195,30 +190,91 @@ func (s *IncomingSwarmSyncer) Priority() int { // return self // } -func RegisterIncomingSyncers(streamer *Streamer, db *DbAccess) { - for po := uint8(0); po < maxPO; po++ { - stream := Stream(fmt.Sprintf("SYNC-%02d-live", po)) - streamer.RegisterIncomingStreamer(stream, func(p *StreamerPeer) (IncomingStreamer, error) { - return NewIncomingSwarmSyncer(High, nil, p, nil, nil) - }) - stream = Stream(fmt.Sprintf("SYNC-%02d-history", po)) - streamer.RegisterIncomingStreamer(stream, func(p *StreamerPeer) (IncomingStreamer, error) { - //intervals := loadIntervals(p, po, false) - return NewIncomingSwarmSyncer(Mid, nil, p, nil, nil) - }) - // stream = fmt.Sprintf("SYNC-%02d-delete", po) - // streamer.RegisterIncomingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) { - // intervals := loadIntervals(p, po, true) - // return NewIncomingSwarmSyncer(po, Mid, sessionAt, intervals, p) - // }) +type Syncer struct { + intervals map[string][]uint64 +} + +func NewSyncer() *Syncer { + return &Syncer{} +} + +type Intervals struct { + syncer *Syncer + key []byte +} + +func (s *Intervals) load() error { + return s.syncer.load(s.key) +} + +func (s *Intervals) save() error { + return s.syncer.save(s.key) +} + +func (s *Intervals) get() []uint64 { + return s.syncer.get(s.key) +} + +func (s *Intervals) set(v []uint64) { + s.syncer.set(s.key, v) +} + +func (s *Syncer) NewIntervals(key []byte) *Intervals { + return &Intervals{ + syncer: s, + key: key, } } +func newSyncLabel(typ string, po uint8) []byte { + t := []byte(typ) + t = append(t, byte(po)) + return t +} + +func parseSyncLabel(t []byte) (string, uint8) { + l := len(t) - 1 + return sstring(t[:l]), uint8(t[l]) +} + +// StartSyncing is called on the StreamerPeer to start the syncing process +// the idea is that it is called only after kademlia is close to healthy +func StartSyncing(s *StreamerPeer, po uint8, nn bool) { + lastPO := po + if nn { + lastPO = maxPO + } + for i := po; i <= lastPO; i++ { + s.Subscribe(Stream("SYNC"), newSyncLabel("LIVE", po), 0, 0, High) + s.Subscribe(Stream("SYNC"), newSyncLabel("HISTORY", po), 0, 0, Mid) + } +} + +func RegisterIncomingSyncers(streamer *Streamer, syncer *Syncer, db *DbAccess) { + stream := Stream("SYNC") + streamer.RegisterIncomingStreamer(stream, func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { + syncType, po := parseSyncLabel(t) + switch syncType { + case "LIVE": + return NewIncomingSwarmSyncer(nil, p, nil, nil) + case "HISTORY": + intervals := syncer.NewIntervals(t) + return NewIncomingSwarmSyncer(intervals, p, nil, nil) + } + return nil, fmt.Errorf("unknown sync type %q", syncType) + }) + // stream = fmt.Sprintf("SYNC-%02d-delete", po) + // streamer.RegisterIncomingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) { + // intervals := loadIntervals(p, po, true) + // return NewIncomingSwarmSyncer(po, Mid, sessionAt, intervals, p) + // }) +} + // NeedData func (self *IncomingSwarmSyncer) NeedData(key []byte) (wait func()) { - chunk, created := self.dbAccess.getOrCreateRequest(key) + chunk, _ := self.dbAccess.getOrCreateRequest(key) // TODO: we may want to request from this peer anyway even if the request exists - if chunk.ReqC == nil || !created { + if chunk.ReqC == nil { return nil } // create request and wait until the chunk data arrives and is stored @@ -227,28 +283,37 @@ func (self *IncomingSwarmSyncer) NeedData(key []byte) (wait func()) { // NextBatch adjusts the indexes by inspecting the intervals func (self *IncomingSwarmSyncer) NextBatch(from uint64) (nextFrom uint64, nextTo uint64) { - if self.intervals[0] >= self.sessionAt { // live syncing + intervals := self.intervals.get() + if intervals[0] >= self.sessionAt { // live syncing nextFrom = from - self.intervals[1] = from + intervals[1] = from } else if from >= self.sessionAt { // history sync complete - self.intervals = nil - } else if len(self.intervals) > 2 && from >= self.intervals[2] { // filled a gap in the intervals - self.intervals = append(self.intervals[:1], self.intervals[3:]...) - nextFrom = self.intervals[1] - if len(self.intervals) > 2 { - nextTo = self.intervals[2] + intervals = nil + } else if len(intervals) > 2 && from >= intervals[2] { // filled a gap in the intervals + intervals = append(intervals[:1], intervals[3:]...) + nextFrom = intervals[1] + if len(intervals) > 2 { + nextTo = intervals[2] } else { nextTo = self.sessionAt } } else { nextFrom = from - self.intervals[1] = from + intervals[1] = from nextTo = self.sessionAt } + self.intervals.set(intervals) return nextFrom, nextTo } -// +// BatchDone +func (self *IncomingSwarmSyncer) BatchDone(s Stream, from uint64, hashes []byte, root []byte) func() (*TakeoverProof, error) { + if self.chunker != nil { + return func() (*TakeoverProof, error) { return self.TakeoverProof(s, from, hashes, root) } + } + return nil +} + func (self *IncomingSwarmSyncer) TakeoverProof(s Stream, from uint64, hashes []byte, root storage.Key) (*TakeoverProof, error) { // for provable syncer currentRoot is non-zero length if self.chunker != nil { diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index be142f227b..918c0fc45b 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -302,16 +302,18 @@ type LazyChunkReader struct { chunkSize int64 // inherit from chunker branches int64 // inherit from chunker hashSize int64 // inherit from chunker + depth int } // implements the Joiner interface -func (self *TreeChunker) Join(key Key, chunkC chan *Chunk) LazySectionReader { +func (self *TreeChunker) Join(key Key, chunkC chan *Chunk, depth int) LazySectionReader { return &LazyChunkReader{ key: key, chunkC: chunkC, chunkSize: self.chunkSize, branches: self.branches, hashSize: self.hashSize, + depth: depth, } } @@ -358,8 +360,13 @@ func (self *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) { depth++ } wg := sync.WaitGroup{} + length := int64(len(b)) + for d := 0; d < self.depth; d++ { + off *= self.chunkSize + length *= self.chunkSize + } wg.Add(1) - go self.join(b, off, off+int64(len(b)), depth, treeSize/self.branches, self.chunk, &wg, errC, quitC) + go self.join(b, off, off+length, depth, treeSize/self.branches, self.chunk, &wg, errC, quitC) go func() { wg.Wait() close(errC) @@ -379,18 +386,14 @@ func (self *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) { func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeSize int64, chunk *Chunk, parentWg *sync.WaitGroup, errC chan error, quitC chan bool) { defer parentWg.Done() - // return NewDPA(&LocalStore{}) - - // chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) - // find appropriate block level - for chunk.Size < treeSize && depth > 0 { + for chunk.Size < treeSize && depth > self.depth { treeSize /= self.branches depth-- } // leaf chunk found - if depth == 0 { + if depth == self.depth { extra := 8 + eoff - int64(len(chunk.SData)) if extra > 0 { eoff -= extra diff --git a/swarm/storage/chunker_test.go b/swarm/storage/chunker_test.go index 5cb1125bf5..a0f82245d3 100644 --- a/swarm/storage/chunker_test.go +++ b/swarm/storage/chunker_test.go @@ -163,7 +163,7 @@ func (self *chunkerTester) Join(chunker Chunker, key Key, c int, chunkC chan *Ch } }() - reader := chunker.Join(key, chunkC) + reader := chunker.Join(key, chunkC, 0) return reader } @@ -489,7 +489,8 @@ func BenchmarkJoin_4(t *testing.B) { benchmarkJoin(10000, t) } func BenchmarkJoin_5(t *testing.B) { benchmarkJoin(100000, t) } func BenchmarkJoin_6(t *testing.B) { benchmarkJoin(1000000, t) } func BenchmarkJoin_7(t *testing.B) { benchmarkJoin(10000000, t) } -func BenchmarkJoin_8(t *testing.B) { benchmarkJoin(100000000, t) } + +// func BenchmarkJoin_8(t *testing.B) { benchmarkJoin(100000000, t) } func BenchmarkSplitTreeSHA3_2(t *testing.B) { benchmarkSplitTreeSHA3(100, t) } func BenchmarkSplitTreeSHA3_2h(t *testing.B) { benchmarkSplitTreeSHA3(500, t) } @@ -500,7 +501,8 @@ func BenchmarkSplitTreeSHA3_4h(t *testing.B) { benchmarkSplitTreeSHA3(50000, t) func BenchmarkSplitTreeSHA3_5(t *testing.B) { benchmarkSplitTreeSHA3(100000, t) } func BenchmarkSplitTreeSHA3_6(t *testing.B) { benchmarkSplitTreeSHA3(1000000, t) } func BenchmarkSplitTreeSHA3_7(t *testing.B) { benchmarkSplitTreeSHA3(10000000, t) } -func BenchmarkSplitTreeSHA3_8(t *testing.B) { benchmarkSplitTreeSHA3(100000000, t) } + +// func BenchmarkSplitTreeSHA3_8(t *testing.B) { benchmarkSplitTreeSHA3(100000000, t) } func BenchmarkSplitTreeBMT_2(t *testing.B) { benchmarkSplitTreeBMT(100, t) } func BenchmarkSplitTreeBMT_2h(t *testing.B) { benchmarkSplitTreeBMT(500, t) } @@ -511,7 +513,8 @@ func BenchmarkSplitTreeBMT_4h(t *testing.B) { benchmarkSplitTreeBMT(50000, t) } func BenchmarkSplitTreeBMT_5(t *testing.B) { benchmarkSplitTreeBMT(100000, t) } func BenchmarkSplitTreeBMT_6(t *testing.B) { benchmarkSplitTreeBMT(1000000, t) } func BenchmarkSplitTreeBMT_7(t *testing.B) { benchmarkSplitTreeBMT(10000000, t) } -func BenchmarkSplitTreeBMT_8(t *testing.B) { benchmarkSplitTreeBMT(100000000, t) } + +// func BenchmarkSplitTreeBMT_8(t *testing.B) { benchmarkSplitTreeBMT(100000000, t) } func BenchmarkSplitPyramidSHA3_2(t *testing.B) { benchmarkSplitPyramidSHA3(100, t) } func BenchmarkSplitPyramidSHA3_2h(t *testing.B) { benchmarkSplitPyramidSHA3(500, t) } @@ -522,7 +525,8 @@ func BenchmarkSplitPyramidSHA3_4h(t *testing.B) { benchmarkSplitPyramidSHA3(5000 func BenchmarkSplitPyramidSHA3_5(t *testing.B) { benchmarkSplitPyramidSHA3(100000, t) } func BenchmarkSplitPyramidSHA3_6(t *testing.B) { benchmarkSplitPyramidSHA3(1000000, t) } func BenchmarkSplitPyramidSHA3_7(t *testing.B) { benchmarkSplitPyramidSHA3(10000000, t) } -func BenchmarkSplitPyramidSHA3_8(t *testing.B) { benchmarkSplitPyramidSHA3(100000000, t) } + +// func BenchmarkSplitPyramidSHA3_8(t *testing.B) { benchmarkSplitPyramidSHA3(100000000, t) } func BenchmarkSplitPyramidBMT_2(t *testing.B) { benchmarkSplitPyramidBMT(100, t) } func BenchmarkSplitPyramidBMT_2h(t *testing.B) { benchmarkSplitPyramidBMT(500, t) } @@ -533,7 +537,8 @@ func BenchmarkSplitPyramidBMT_4h(t *testing.B) { benchmarkSplitPyramidBMT(50000, func BenchmarkSplitPyramidBMT_5(t *testing.B) { benchmarkSplitPyramidBMT(100000, t) } func BenchmarkSplitPyramidBMT_6(t *testing.B) { benchmarkSplitPyramidBMT(1000000, t) } func BenchmarkSplitPyramidBMT_7(t *testing.B) { benchmarkSplitPyramidBMT(10000000, t) } -func BenchmarkSplitPyramidBMT_8(t *testing.B) { benchmarkSplitPyramidBMT(100000000, t) } + +// func BenchmarkSplitPyramidBMT_8(t *testing.B) { benchmarkSplitPyramidBMT(100000000, t) } func BenchmarkAppendPyramid_2(t *testing.B) { benchmarkAppendPyramid(100, 1000, t) } func BenchmarkAppendPyramid_2h(t *testing.B) { benchmarkAppendPyramid(500, 1000, t) } @@ -543,7 +548,8 @@ func BenchmarkAppendPyramid_4h(t *testing.B) { benchmarkAppendPyramid(50000, 100 func BenchmarkAppendPyramid_5(t *testing.B) { benchmarkAppendPyramid(1000000, 1000, t) } func BenchmarkAppendPyramid_6(t *testing.B) { benchmarkAppendPyramid(1000000, 1000, t) } func BenchmarkAppendPyramid_7(t *testing.B) { benchmarkAppendPyramid(10000000, 1000, t) } -func BenchmarkAppendPyramid_8(t *testing.B) { benchmarkAppendPyramid(100000000, 1000, t) } + +// func BenchmarkAppendPyramid_8(t *testing.B) { benchmarkAppendPyramid(100000000, 1000, t) } // go test -timeout 20m -cpu 4 -bench=./swarm/storage -run no // If you dont add the timeout argument above .. the benchmark will timeout and dump diff --git a/swarm/storage/dpa.go b/swarm/storage/dpa.go index 7ef8a80a99..5f0a95470e 100644 --- a/swarm/storage/dpa.go +++ b/swarm/storage/dpa.go @@ -92,7 +92,7 @@ func NewDPA(store ChunkStore, params *ChunkerParams) *DPA { // Chunk retrieval blocks on netStore requests with a timeout so reader will // report error if retrieval of chunks within requested range time out. func (self *DPA) Retrieve(key Key) LazySectionReader { - return self.Chunker.Join(key, self.retrieveC) + return self.Chunker.Join(key, self.retrieveC, 0) } // Public API. Main entry point for document storage directly. Used by the diff --git a/swarm/storage/types.go b/swarm/storage/types.go index 73a4f758d6..956b8ddd8b 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -273,7 +273,7 @@ type Joiner interface { The chunks are not meant to be validated by the chunker when joining. This is because it is left to the DPA to decide which sources are trusted. */ - Join(key Key, chunkC chan *Chunk) LazySectionReader + Join(key Key, chunkC chan *Chunk, depth int) LazySectionReader } type Chunker interface { From a21d3399fc21fd713fc106c7be0671545a0e1cd5 Mon Sep 17 00:00:00 2001 From: zelig Date: Wed, 10 Jan 2018 19:35:55 +0100 Subject: [PATCH 018/210] swarm/network: refactor and modify streamer code --- swarm/network/README.md | 154 ++++++++++++++++--------------- swarm/network/lightnode.go | 36 ++++---- swarm/network/requests.go | 183 ++++++++++++++++--------------------- swarm/network/streamer.go | 177 +++++++++++++++-------------------- swarm/network/syncer.go | 93 +++---------------- 5 files changed, 257 insertions(+), 386 deletions(-) diff --git a/swarm/network/README.md b/swarm/network/README.md index 335ff42e61..ad429b38be 100644 --- a/swarm/network/README.md +++ b/swarm/network/README.md @@ -1,30 +1,67 @@ -# Requirements +## Streaming + +Streaming is a new protocol of the swarm bzz bundle of protocols. +This protocol provides the basic logic for chunk-based data flow. +It implements simple retrieve requests and delivery using priority queue. +A data exchange stream is a directional flow of chunks between peers. +The source of datachunks is the upstream, the receiver is called the +downstream peer. Each streaming protocol defines an outgoing streamer +and an incoming streamer, the former installing on the upstream, +the latter on the downstream peer. + +Subscribe on StreamerPeer launches an incoming streamer that sends +a subscribe msg upstream. The streamer on the upstream peer +handles the subscribe msg by installing the relevant outgoing streamer +. The modules now engage in a process of upstream sending a sequence of hashes of +chunks downstream (OfferedHashesMsg). The downstream peer evaluates which hashes are needed +and get it delivered by sending back a msg (WantedHashesMsg). + +Historical syncing is supported - currently not the right abstraction -- +state kept across sessions by saving a series of intervals after their last +batch actually arrived. + +Live streaming is also supported, by starting session from the first item +after the subscription. + +Provable data exchange. In case a stream represents a swarm document's data layer +or higher level chunks, streaming up to a certain index is always provable. It saves on +sending intermediate chunks. + +Using the streamer logic, various stream types are easy to implement: + +* light node requests: + * url lookup with offset + * document download + * document upload +* syncing + * live session syncing + * historical syncing +* simple retrieve requests and deliveries +* mutable resource updates streams +* receipting for finger pointing + +## Syncing + +Syncing is the process that makes sure storer nodes end up storing all and only the chunks that are requested from them. + +### Requirements - eventual consistency: so each chunk historical should be syncable - since the same chunk can and will arrive from many peers, (network traffic should be -optimised (only one transfer of data per chunk) +optimised, only one transfer of data per chunk) - explicit request deliveries should be prioritised higher than recent chunks received during the ongoing session which in turn should be higher than historical chunks. - insured chunks should get receipted for finger pointing litigation, the receipts storage should be organised efficiently, upstream peer should also be able to find these receipts for a deleted chunk easily to refute their challenge. - syncing should be resilient to cut connections, metadata should be persisted that -keep track of syncing state across sessions. +keep track of syncing state across sessions, historical syncing state should survive restart - extra data structures to support syncing should be kept at minimum - syncing is organized separately for chunk types (resource update v content chunk) +- various types of streams should have common logic abstracted - -When two peers connect, the bidirectional protocol is a result of two identical -syncing protocols mirrored. So take one direction and call the two parties -upstream and downstream peer. - -when a new chunk is stored, its hash is appended to a boundless stream maintained for -each kademlia bin. This state is always permanently recorded periodically possibly -using mutable resource update scheme. - -At any point in time (with n chunks in total ) the swarm hash of this hash stream state is -well defined. Upstream peer pushes so that the reader reads sequential hashes and -periodically calculates the swarm hash of their stream. +Syncing is now entirely mediated by the localstore, ie., no processes or memory leaks due to network contention. +When a new chunk is stored, its chunk hash is index by proximity bin peers syncronise by getting the chunks closer to the downstream peer than to the upstream one. Consequently peers just sync all stored items for the kad bin the receiving peer falls into. @@ -32,24 +69,14 @@ The special case of nearest neighbour sets is handled by the downstream peer indicating they want to sync all kademlia bins with proximity equal to or higher than their depth. -When peers connect upstream peer sends the latest sync state (item index/cursor length) -for each relevant bin downstream peer is interested in. This sync state represents the initial state of a sync connection session. -Conversely downstream peers maintain the last state (swarm hash with length) which -the ranges of covered offsets. +Retrieval is dictated by downstream peers simply using a special streamer protocol. -Retrieval is dictated by downstream peers simply using the the chunker joiner to read certain offsets. - -Syncing chunks created during the session by the upstream peer is called session Syncing +Syncing chunks created during the session by the upstream peer is called live session syncing while syncing of earlier chunks is historical syncing. -Historical syncing is simply carried out by iteratively requesting ranges of hash offsets. -For simplicity we assume that the minimum unit requested is a chunk. If every 128 chunks -is considered syncable one can use data chunk index instead of offset in byte length. -Note that data chunk here is a sequence of hashes above one ground level of stream of content. - Once the relevant chunk is retrieved, downstream peer looks up all hash segments in its localstore -and sends to the upstream peer a message with a 128-long bitvector (uint16) to indicate +and sends to the upstream peer a message with a a bitvector to indicate missing chunks (e.g., for chunk `k`, hash with chunk internal index which case ) new items. In turn upstream peer sends the relevant chunk data alongside their index. @@ -64,20 +91,35 @@ Session syncing involves downstream peer to request a new state on a bin from up using the new state, the range (of chunks) between the previous state and the new one are retrieved and chunks are requested identical to the historical case. After receiving all the missing chunks from the new hashes, downstream peer will request a new range. If this happens before upstream peer updates a new state, -we say that session syncing is live or the two peers are in sync. +we say that session syncing is live or the two peers are in sync. In general the time interval passed since downstream peer request up to the current session cursor is a good indication of a permanent (probably increasing) lag. -If there is no historical backlog, downstream peer is said to be fully synced with the upstream. +If there is no historical backlog, and downstream peer has an acceptable 'last synced' tag, then it is said to be fully synced with the upstream peer. If a peer is fully synced with all its storer peers, it can advertise itself as globally fully synced. -For healthy operation, however, it is expected that the session is regularly in sync. If this is -not the case, that indicates that traffic during the session is continuously more than the peer can cope with and -downstream peer is effectively accumulating a historical backlog. - The downstream peer persists the record of the last synced offset. When the two peers disconnect and reconnect syncing can start from there. This situation however can also happen while historical syncing is not yet complete. Effectively this means that the peer needs to persist a record of an arbitrary array of offset ranges covered. +### Delivery requests + +once the appropriate ranges of the hashstream are retrieved and buffered, downstream peer just scans the hashes, looks them up in localstore, if not found, create a request entry. +The range is referenced by the chunk index. Alongside the name (indicating the stream, e.g., content chunks for bin 6) and the range +downstream peer sends a 128 long bitvector indicating which chunks are needed. +Newly created requests are satisfied bound together in a waitgroup which when done, will promptt sending the next one. +to be able to do check and storage concurrently, we keep a buffer of one, we start with two batches of hashes. +If there is nothing to give, upstream peers SetNextBatch is blocking. Subscription ends with an unsubscribe. which removes the syncer from the map. + +Canceling requests (for instance the late chunks of an erasure batch) should be a chan closed +on the request + +Simple request is also a subscribe +different streaming protocols are different p2p protocols with same message types. +the constructor is the Run function itself. which takes a streamerpeer as argument + + +### provable streams + The swarm hash over the hash stream has many advantages. It implements a provable data transfer and provide efficient storage for receipts in the form of inclusion proofs useable for finger pointing litigation. When challenged on a missing chunk, upstream peer will provide an inclusion proof of a chunk hash against the state of the @@ -91,19 +133,16 @@ As part of the deletion protocol then, hashes of insured chunks to be removed ar Downstream peer on the other hand needs to make sure that they can only be finger pointed about a chunk they did receive and store. For this the check of a state should be exhaustive. If historical syncing finishes on one state, all hashes before are covered, no surprises. In other words historical syncing this process is self verifying. With session syncing however, it is not enough to check going back covering the range from old offset to new. Continuity (i.e., that the new state is extension of the old) needs to be verified: after downstream peer reads the range into a buffer, it appends the buffer the last known state at the last known offset and verifies the resulting hash matches -the latest state. The same goes with intervals of historical syncing where the state used to verify the preceding interval is different from the one used to cover the current range. In these cases too, verification by append is needed for complete security for downstream peer. - +the latest state. Past intervals of historical syncing are checked via the the session root. Upstream peer signs the states, downstream peers can use as handover proofs. Downstream peers sign off on a state together with an initial offset. -latter needed for reasonable sized not ever growing syncer -possible is eachn chunk needs to be reentered if they remain insured Once historical syncing is complete and the session does not lag, downstream peer only preserves the latest upstream state and store the signed version. Upstream peer needs to keep the latest takeover states: each deleted chunk's hash should be covered by takeover proof of at least one peer. If historical syncing is complete, upstream peer typically will store only the latest takeover proof from downstream peer. Crucially, the structure is totally independent of the number of peers in the bin, so it scales extremely well. -implementation +## implementation The simplest protocol just involves upstream peer to prefix the key with the kademlia proximity order (say 0-15 or 0-31) and simply iterate on index per bin when syncing with a peer. @@ -111,40 +150,3 @@ and simply iterate on index per bin when syncing with a peer. priority queues are used for sending chunks so that user triggered requests should be responded to first, session syncing second, and historical with lower priority. The request on chunks remains implemented as a dataless entry in the memory store. The lifecycle of this object should be more carefully thought through, ie., when it fails to retrieve it should be removed. - - - -Model 1 -The main appeal in this model is that downstream driven syncing falls back to the exact same retrieval mechanism as the one used when downloading a file. If the chunks of the hash stream (datachunks as well as intermediate chunks of the swarm tree above it) are themselves distributed by upstream peer in the normal way, then requesting them from swarm is viable. -This requires no extra implementation. However, it is unlikely this is feasible for live syncing since the chunks' delay to arrive at their destination has a lag exactly due to session syncronisation. -Note that if upstream peer handles chunks of the hash stream as normal chunks, there are issues. One is that some of these chunks will fall in the same bin as the one building leading to a situation where the hash stream grows even though there are no external chunks received. If upstream peer wishes to use finger pointing proofs, it has to either store these chunks themselves or insure them. - - -Model 2 -In this model, when retrieving the hash stream from the state, requests are targeted to the upstream peer. -The simplest way to generate the right requests from a sync state is to have a -peer specific dpa chunkstore for chunker join. This can be used by all chunk types and all bins. -All it does is when picking retrieve tasks off the chunk channel, it marks the chunk with a reference to the -upstream peer so that when netstore does not find it, the request is sent to the upstream peer (only). -(Normally the request would be routed based on its address). -We need to introduce a special field on the chunk to indicate that the data should be requested from a particular peer. -Upstream peer could still distribute these chunks used in the hash stream in swarm as usual, e.g., long non-synced early -history. -This model does not suffer from the availability lag of the first one, correctly puts the burden on upstream peer to preserve -chunks of the hash stream either in swarm or not. - -Model 3 -in another alternative upstream peer sends only the data level (the hash stream interval) not all the intermediate chunks. This saves on traffic and downstream peer can calculate the state by append to verify against the state (root hash). -This mode of operation is anyhow feasible in cases where the same peer having the top request will be expected to have all the children chunks -of an intermediate one. This is the case for syncing and hash stream or when light swarm clients channel all their requests to a proxy node (public database lookup or unencrypted content). - -This model would require no peer specific dpa and would involve the chunker split only to get the right hand side for the append for verification. - -Delivery requests - -once the appropriate ranges of the hashstream are retrieved and buffered, downstream peer just scans the hashes, looks them up in localstore, if not found, create a request entry with specific reference to the upstream peer as source. -The range is referenced by the chunk index. Alongside the name (indicating the stream, e.g., content chunks for bin 6) and the range -downstream peer sends a 128 long bitvector indicating which chunks are needed. -Newly created requests are satisfied bound together in a waitgroup which when done, will prompt sending the next one. -to be able to do check and storage concurrently, we keep a buffer of one, we start with two chunks. -For session syncing too, if it has not arrived sby the time the next chunk. diff --git a/swarm/network/lightnode.go b/swarm/network/lightnode.go index ca1463b4d4..b351e87dcf 100644 --- a/swarm/network/lightnode.go +++ b/swarm/network/lightnode.go @@ -50,14 +50,15 @@ func (r *RemoteSectionReader) NeedData(key []byte) func() { if chunk.ReqC == nil || !created { return nil } - return func() {} + return func() { + select { + case <-chunk.ReqC: + case <-r.quit: + } + } } -func (r *RemoteSectionReader) NextBatch(from uint64) (nextFrom uint64, nextTo uint64) { - return from, r.end -} - -func (r *RemoteSectionReader) BatchDone(s Stream, from uint64, hashes []byte, root []byte) func() (*TakeoverProof, error) { +func (r *RemoteSectionReader) BatchDone(s string, from uint64, hashes []byte, root []byte) func() (*TakeoverProof, error) { r.hashes <- hashes return nil } @@ -113,16 +114,14 @@ func (r *RemoteSectionReader) Read(b []byte) (n int64, err error) { hashes = hashes[i:] } } - return n, nil } // RemoteSectionServer implements OutgoingStreamer type RemoteSectionServer struct { // quit chan struct{} - currentBatch []byte - root []byte - db *DbAccess - r *storage.LazyChunkReader + root []byte + db *DbAccess + r *storage.LazyChunkReader } // NewRemoteReader is the constructor for RemoteReader @@ -149,14 +148,12 @@ func (s *RemoteSectionServer) SetNextBatch(from, to uint64) ([]byte, uint64, uin } batch := make([]byte, (to-from)*HashSize) s.r.ReadAt(batch, int64(from)) - s.currentBatch = batch return batch, from, to, nil, nil } // RegisterRemoteSectionReader registers RemoteSectionReader on light downstream node func RegisterRemoteSectionReader(s *Streamer, db *DbAccess) { - name := Stream("REMOTE_SECTION") - s.RegisterIncomingStreamer(name, func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { + s.RegisterIncomingStreamer("REMOTE_SECTION", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { return NewRemoteSectionReader(t, db), nil }) } @@ -164,8 +161,7 @@ func RegisterRemoteSectionReader(s *Streamer, db *DbAccess) { // RegisterRemoteSectionServer registers RemoteSectionServer outgoing streamer on // upstream light server node func RegisterRemoteSectionServer(s *Streamer, db *DbAccess, rf func([]byte) *storage.LazyChunkReader) { - name := Stream("REMOTE_SECTION") - s.RegisterOutgoingStreamer(name, func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) { + s.RegisterOutgoingStreamer("REMOTE_SECTION", func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) { r := rf(t) return NewRemoteSectionServer(db, r), nil }) @@ -174,8 +170,7 @@ func RegisterRemoteSectionServer(s *Streamer, db *DbAccess, rf func([]byte) *sto // RegisterRemoteDownloader registers RemoteDownloader incoming streamer // on downstream light node func RegisterRemoteDownloader(s *Streamer, db *DbAccess) { - name := Stream("REMOTE_DOWNLOADER") - s.RegisterIncomingStreamer(name, func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { + s.RegisterIncomingStreamer("REMOTE_DOWNLOADER", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { return NewRemoteDownloader(t, db), nil }) } @@ -183,9 +178,10 @@ func RegisterRemoteDownloader(s *Streamer, db *DbAccess) { // RegisterRemoteDownloadServer registers RemoteDownloadServer outgoing streamer on // upstream light server node func RegisterRemoteDownloadServer(s *Streamer, db *DbAccess, rf func([]byte) *storage.LazyChunkReader) { - name := Stream("REMOTE_DOWNLOADER") - s.RegisterOutgoingStreamer(name, func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) { + s.RegisterOutgoingStreamer("REMOTE_DOWNLOADER", func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) { r := rf(t) return NewRemoteDownloadServer(db, r), nil }) } + +func NewRemoteDownloader() diff --git a/swarm/network/requests.go b/swarm/network/requests.go index f5cea0ea44..e3dd2846a1 100644 --- a/swarm/network/requests.go +++ b/swarm/network/requests.go @@ -16,122 +16,93 @@ package network -// import ( -// "fmt" +import "github.com/ethereum/go-ethereum/swarm/storage" -// "github.com/ethereum/go-ethereum/log" -// "github.com/ethereum/go-ethereum/swarm/storage" -// ) +const retrieveRequestStream = "RETRIEVE_REQUEST" -// /* -// Retrieve Request and store Request handling -// */ +// Intervals is a stream specific history of downloaded intervals +// for historical streams +type Intervals struct { + streamer *Streamer + key string +} -// // Handler for storage/retrieval related protocol requests -// type RequestHandler struct { -// netStore *storage.NetStore -// } +func (s *Intervals) load() error { + return s.streamer.load(s.key) +} -// // NewEwquestHandler creates a new RequestHandler -// // netStore to -// func NewRequestHandler(netStore *storage.NetStore) *RequestHandler { -// return &RequestHandler{ -// netStore: netStore, // entrypoint internal -// } -// } +func (s *Intervals) save() error { + return s.streamer.save(s.key) +} -// /* -// Retrieve request +func (s *Intervals) get() []uint64 { + return s.streamer.get(s.key) +} -// MaxSize specifies the maximum size that the peer will accept. This is useful in -// particular if we allow storage and delivery of multichunk payload representing -// the entire or partial subtree unfolding from the requested root key. -// So when only interested in limited part of a stream (infinite trees) or only -// testing chunk availability etc etc, we can indicate it by limiting the size here. +func (s *Intervals) set(v []uint64) { + s.streamer.set(s.key, v) +} -// Request ID can be newly generated or kept from the request originator. +func NewIntervals(key string, s *Streamer) *Intervals { + return &Intervals{ + streamer: s, + key: key, + } +} -// */ -// type retrieveRequestMsg struct { -// Key storage.Key // target Key address of chunk to be retrieved -// Id uint64 // request id, request is a lookup if missing or zero -// MaxSize uint64 // maximum size of delivery accepted -// from *StreamerPeer // -// } +// RetrieveRequestStreamer implements OutgoingStreamer +type RetrieveRequestStreamer struct { + deliveryC chan *storage.Chunk + batchC chan []byte + db *DbAccess + currentLen uint64 +} -// func (self retrieveRequestMsg) String() string { -// var from string -// if self.from == nil { -// from = "ourselves" -// } else { -// from = fmt.Sprintf("%x", self.from.Over()) -// } -// var target []byte -// if len(self.Key) > 3 { -// target = self.Key[:4] -// } -// return fmt.Sprintf("Requester: %v, Key: %x; ID: %v, MaxSize: %v", from, target, self.Id, self.MaxSize) -// } +// RegisterRequestStreamer registers outgoing and incoming streamers for request handling +func RegisterRequestStreamer(streamer *Streamer, db *DbAccess) { + streamer.RegisterOutgoingStreamer(retrieveRequestStream, func(_ *StreamerPeer, t []byte) (OutgoingStreamer, error) { + return NewRetrieveRequestStreamer(db), nil + }) + streamer.RegisterIncomingStreamer(retrieveRequestStream, func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { + return NewIncomingSwarmSyncer(p, db, nil) + }) +} -// entrypoint for retrieve requests coming from the bzz wire protocol -// checks swap balance - return if peer has no credit -// func (self *StreamerPeer) handleRetrieveRequestMsg(msg interface{}) error { -// req := msg.(*retrieveRequestMsg) -// req.from = self -// // TODO: -// // swap - record credit for 1 request -// // note that only charge actual reqsearches +// NewRetrieveRequestStreamer is RetrieveRequestStreamer constructor +func NewRetrieveRequestStreamer(db *DbAccess) *RetrieveRequestStreamer { + s := &RetrieveRequestStreamer{ + deliveryC: make(chan *storage.Chunk), + batchC: make(chan []byte), + db: db, + } + go s.processDeliveries() + return s +} -// // call storage.NetStore#Get which -// // blocks until local retrieval finished -// // launches cloud retrieval -// chunk, _ := self.netStore.Get(req.Key) -// rs := chunk.Req -// if rs != nil { -// rs = storage.NewRequestStatus(req.Key) -// addRequester(rs, req) -// chunk.Req = rs -// } +// processDeliveries handles delivered chunk hashes +func (s *RetrieveRequestStreamer) processDeliveries() { + var hashes []byte + for { + select { + case delivery := <-s.deliveryC: + hashes = append(hashes, delivery.Key[:]...) + case s.batchC <- hashes: + hashes = nil + } + } +} -// // check if we can immediately deliver -// if chunk.SData != nil { -// if req.MaxSize == 0 || int64(req.MaxSize) >= chunk.Size { -// err := self.Deliver(chunk, Top) -// if err != nil { -// log.Trace(fmt.Sprintf("%v - content found, delivery error: %v", req.Key.Log(), err)) -// return nil -// } -// log.Trace(fmt.Sprintf("%v - content found, delivering...", req.Key.Log())) -// } else { -// log.Trace(fmt.Sprintf("%v - content found, not wanted", req.Key.Log())) -// } -// } else { -// log.Trace(fmt.Sprintf("content not found locally, retrieve via bzz", req.Key.Log())) -// } -// return nil -// } +// SetNextBatch +func (s *RetrieveRequestStreamer) SetNextBatch(_, _ uint64) (hashes []byte, from uint64, to uint64, proof *HandoverProof, err error) { + hashes = <-s.batchC + from = s.currentLen + s.currentLen += uint64(len(hashes)) + to = s.currentLen + return +} -// /* -// adds a new peer to an existing open request -// only add if less than requesterCount peers forwarded the same request id so far -// note this is done irrespective of status (searching or found) -// */ -// func addRequester(rs *storage.RequestStatus, req *retrieveRequestMsg) { -// log.Trace(fmt.Sprintf("Depo.addRequester: key %v - add peer to req.Id %v", req.Key.Log(), req.Id)) -// list := rs.Requesters[req.Id] -// rs.Requesters[req.Id] = append(list, req) -// } - -// func (self storeRequestMsg) String() string { -// var from string -// if self.from == nil { -// from = "self" -// } else { -// from = fmt.Sprintf("%x", self.from.Over()) -// } -// end := len(self.SData) -// if len(self.SData) > 10 { -// end = 10 -// } -// return fmt.Sprintf("from: %v, ID: %v, SData %x", from, self.Id, self.SData[:end]) -// } +// GetData retrives chunk data from db store +func (s *RetrieveRequestStreamer) GetData(key []byte) []byte { + chunk, _ := s.db.get(storage.Key(key)) + return chunk.SData +} diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go index bb6ace62b7..b5651c4b87 100644 --- a/swarm/network/streamer.go +++ b/swarm/network/streamer.go @@ -42,12 +42,9 @@ const ( PriorityQueueCap = 3 // queue capacity ) -// Stream is string descriptor of the stream -type Stream string - // Handover represents a statement that the upstream peer hands over the stream section type Handover struct { - Stream Stream // name of stream + Stream string // name of stream Start, End uint64 // index of hashes Root []byte // Root hash for indexed segment inclusion proofs } @@ -79,7 +76,7 @@ func (self TakeoverProofMsg) String() string { // SubcribeMsg is the protocol msg for requesting a stream(section) type SubscribeMsg struct { - Stream Stream + Stream string Key []byte From, To uint64 Priority uint8 // delivered on priority channel @@ -88,7 +85,7 @@ type SubscribeMsg struct { // UnsyncedKeysMsg is the protocol msg for offering to hand over a // stream section type UnsyncedKeysMsg struct { - Stream Stream // name of Stream + Stream string // name of Stream Key []byte // subtype or key From, To uint64 // peer and db-specific entry count Hashes []byte // stream of hashes (128) @@ -115,7 +112,7 @@ func (self UnsyncedKeysMsg) String() string { // WantedKeysMsg is the protocol msg data for signaling which hashes // offered in UnsyncedKeysMsg downstream peer actually wants sent over type WantedKeysMsg struct { - Stream Stream // name of stream + Stream string // name of stream Key []byte // subtype or key Want []byte // bitvector indicating which keys of the batch needed From, To uint64 // next interval offset - empty if not to be continued @@ -130,8 +127,8 @@ func (self WantedKeysMsg) String() string { type Streamer struct { incomingLock sync.RWMutex outgoingLock sync.RWMutex - outgoing map[Stream]func(*StreamerPeer, []byte) (OutgoingStreamer, error) - incoming map[Stream]func(*StreamerPeer, []byte) (IncomingStreamer, error) + outgoing map[string]func(*StreamerPeer, []byte) (OutgoingStreamer, error) + incoming map[string]func(*StreamerPeer, []byte) (IncomingStreamer, error) dbAccess *DbAccess overlay Overlay @@ -141,8 +138,8 @@ type Streamer struct { // NewStreamer is Streamer constructor func NewStreamer(overlay Overlay, dbAccess *DbAccess) *Streamer { return &Streamer{ - outgoing: make(map[Stream]func(*StreamerPeer, []byte) (OutgoingStreamer, error)), - incoming: make(map[Stream]func(*StreamerPeer, []byte) (IncomingStreamer, error)), + outgoing: make(map[string]func(*StreamerPeer, []byte) (OutgoingStreamer, error)), + incoming: make(map[string]func(*StreamerPeer, []byte) (IncomingStreamer, error)), dbAccess: dbAccess, overlay: overlay, receiveC: make(chan *ChunkDeliveryMsg, 10), @@ -150,21 +147,21 @@ func NewStreamer(overlay Overlay, dbAccess *DbAccess) *Streamer { } // RegisterIncomingStreamer registers an incoming streamer constructor -func (self *Streamer) RegisterIncomingStreamer(stream Stream, f func(*StreamerPeer, []byte) (IncomingStreamer, error)) { +func (self *Streamer) RegisterIncomingStreamer(stream string, f func(*StreamerPeer, []byte) (IncomingStreamer, error)) { self.incomingLock.Lock() defer self.incomingLock.Unlock() self.incoming[stream] = f } // RegisterOutgoingStreamer registers an outgoing streamer constructor -func (self *Streamer) RegisterOutgoingStreamer(stream Stream, f func(*StreamerPeer, []byte) (OutgoingStreamer, error)) { +func (self *Streamer) RegisterOutgoingStreamer(stream string, f func(*StreamerPeer, []byte) (OutgoingStreamer, error)) { self.outgoingLock.Lock() defer self.outgoingLock.Unlock() self.outgoing[stream] = f } // GetIncomingStreamer accessor for incoming streamer constructors -func (self *Streamer) GetIncomingStreamer(stream Stream) (func(*StreamerPeer, []byte) (IncomingStreamer, error), error) { +func (self *Streamer) GetIncomingStreamer(stream string) (func(*StreamerPeer, []byte) (IncomingStreamer, error), error) { self.incomingLock.RLock() defer self.incomingLock.RUnlock() f := self.incoming[stream] @@ -175,7 +172,7 @@ func (self *Streamer) GetIncomingStreamer(stream Stream) (func(*StreamerPeer, [] } // GetOutgoingStreamer accessor for incoming streamer constructors -func (self *Streamer) GetOutgoingStreamer(stream Stream) (func(*StreamerPeer, []byte) (OutgoingStreamer, error), error) { +func (self *Streamer) GetOutgoingStreamer(stream string) (func(*StreamerPeer, []byte) (OutgoingStreamer, error), error) { self.outgoingLock.RLock() defer self.outgoingLock.RUnlock() f := self.outgoing[stream] @@ -207,16 +204,18 @@ type OutgoingStreamer interface { type incomingStreamer struct { IncomingStreamer - priority uint8 - quit chan struct{} - next chan struct{} + priority uint8 + intervals *Intervals + sessionAt uint64 + live bool + quit chan struct{} + next chan struct{} } // IncomingStreamer interface for incoming peer Streamer type IncomingStreamer interface { - NextBatch(uint64) (uint64, uint64) NeedData([]byte) func() - BatchDone(Stream, uint64, []byte, []byte) func() (*TakeoverProof, error) + BatchDone(string, uint64, []byte, []byte) func() (*TakeoverProof, error) } // StreamerPeer is the Peer extention for the streaming protocol @@ -228,28 +227,18 @@ type StreamerPeer struct { dbAccess *DbAccess outgoingLock sync.RWMutex incomingLock sync.RWMutex - outgoing map[Stream]*outgoingStreamer - incoming map[Stream]*incomingStreamer + outgoing map[string]*outgoingStreamer + incoming map[string]*incomingStreamer quit chan struct{} } -// type IncomingStreamer struct { -// priority uint8 -// peer *StreamerPeer -// } - -// type OutgoingStreamer struct { -// priority uint8 -// peer *StreamerPeer -// } - // NewStreamerPeer is the constructor for StreamerPeer func NewStreamerPeer(p Peer, streamer *Streamer) *StreamerPeer { self := &StreamerPeer{ pq: pq.New(int(PriorityQueue), PriorityQueueCap), streamer: streamer, - outgoing: make(map[Stream]*outgoingStreamer), - incoming: make(map[Stream]*incomingStreamer), + outgoing: make(map[string]*outgoingStreamer), + incoming: make(map[string]*incomingStreamer), quit: make(chan struct{}), } ctx, cancel := context.WithCancel(context.Background()) @@ -266,65 +255,6 @@ type RetrieveRequestMsg struct { Key storage.Key } -// RetrieveRequestStreamer implements OutgoingStreamer -type RetrieveRequestStreamer struct { - deliveryC chan *storage.Chunk - batchC chan []byte - db *DbAccess - currentLen uint64 -} - -// RegisterRequestStreamer registers outgoing and incoming streamers for request handling -func RegisterRequestStreamer(streamer *Streamer, db *DbAccess) { - streamer.RegisterOutgoingStreamer(retrieveRequestStream, func(_ *StreamerPeer, t []byte) (OutgoingStreamer, error) { - return NewRetrieveRequestStreamer(db), nil - }) - streamer.RegisterIncomingStreamer(retrieveRequestStream, func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { - return NewIncomingSwarmSyncer(nil, p, db, nil) - }) -} - -// NewRetrieveRequestStreamer is RetrieveRequestStreamer constructor -func NewRetrieveRequestStreamer(db *DbAccess) *RetrieveRequestStreamer { - s := &RetrieveRequestStreamer{ - deliveryC: make(chan *storage.Chunk), - batchC: make(chan []byte), - db: db, - } - go s.processDeliveries() - return s -} - -// processDeliveries handles delivered chunk hashes -func (s *RetrieveRequestStreamer) processDeliveries() { - var hashes []byte - for { - select { - case delivery := <-s.deliveryC: - hashes = append(hashes, delivery.Key[:]...) - case s.batchC <- hashes: - hashes = nil - } - } -} - -// SetNextBatch -func (s *RetrieveRequestStreamer) SetNextBatch(_, _ uint64) (hashes []byte, from uint64, to uint64, proof *HandoverProof, err error) { - hashes = <-s.batchC - from = s.currentLen - s.currentLen += uint64(len(hashes)) - to = s.currentLen - return -} - -// GetData retrives chunk data from db store -func (s *RetrieveRequestStreamer) GetData(key []byte) []byte { - chunk, _ := s.db.get(storage.Key(key)) - return chunk.SData -} - -const retrieveRequestStream = Stream("RETRIEVE_REQUEST") - func (self *StreamerPeer) handleRetrieveRequestMsg(req *RetrieveRequestMsg) error { chunk, created := self.dbAccess.getOrCreateRequest(req.Key) s, err := self.getOutgoingStreamer(retrieveRequestStream) @@ -399,7 +329,7 @@ func (self *Streamer) processReceivedChunks() { } } -func (self *StreamerPeer) getOutgoingStreamer(s Stream) (*outgoingStreamer, error) { +func (self *StreamerPeer) getOutgoingStreamer(s string) (*outgoingStreamer, error) { self.outgoingLock.RLock() defer self.outgoingLock.RUnlock() streamer := self.outgoing[s] @@ -409,7 +339,7 @@ func (self *StreamerPeer) getOutgoingStreamer(s Stream) (*outgoingStreamer, erro return streamer, nil } -func (self *StreamerPeer) getIncomingStreamer(s Stream) (*incomingStreamer, error) { +func (self *StreamerPeer) getIncomingStreamer(s string) (*incomingStreamer, error) { self.incomingLock.RLock() defer self.incomingLock.RUnlock() streamer := self.incoming[s] @@ -419,7 +349,7 @@ func (self *StreamerPeer) getIncomingStreamer(s Stream) (*incomingStreamer, erro return streamer, nil } -func (self *StreamerPeer) setOutgoingStreamer(s Stream, o OutgoingStreamer, priority uint8) (*outgoingStreamer, error) { +func (self *StreamerPeer) setOutgoingStreamer(s string, o OutgoingStreamer, priority uint8) (*outgoingStreamer, error) { self.outgoingLock.Lock() defer self.outgoingLock.Unlock() if self.outgoing[s] != nil { @@ -433,15 +363,22 @@ func (self *StreamerPeer) setOutgoingStreamer(s Stream, o OutgoingStreamer, prio return os, nil } -func (self *StreamerPeer) setIncomingStreamer(s Stream, i IncomingStreamer, priority uint8) error { +func (self *StreamerPeer) setIncomingStreamer(s string, i IncomingStreamer, priority uint8, live bool) error { self.incomingLock.Lock() defer self.incomingLock.Unlock() if self.incoming[s] != nil { return fmt.Errorf("stream %v already registered", s) } next := make(chan struct{}, 1) + var intervals *Intervals + if !live { + key := s + self.ID().String() + intervals = NewIntervals(key, self.streamer) + } self.incoming[s] = &incomingStreamer{ IncomingStreamer: i, + intervals: intervals, + live: live, priority: priority, next: next, } @@ -449,8 +386,37 @@ func (self *StreamerPeer) setIncomingStreamer(s Stream, i IncomingStreamer, prio return nil } +// NextBatch adjusts the indexes by inspecting the intervals +func (self *incomingStreamer) nextBatch(from uint64) (nextFrom uint64, nextTo uint64) { + intervals := self.intervals.get() + if self.live { + if len(intervals) == 0 { + intervals = []uint64{self.sessionAt, from} + } else { + intervals[1] = from + } + nextFrom = from + } else if from >= self.sessionAt { // history sync complete + intervals = nil + } else if len(intervals) > 2 && from >= intervals[2] { // filled a gap in the intervals + intervals = append(intervals[:1], intervals[3:]...) + nextFrom = intervals[1] + if len(intervals) > 2 { + nextTo = intervals[2] + } else { + nextTo = self.sessionAt + } + } else { + nextFrom = from + intervals[1] = from + nextTo = self.sessionAt + } + self.intervals.set(intervals) + return nextFrom, nextTo +} + // Subscribe initiates the streamer -func (self *StreamerPeer) Subscribe(s Stream, t []byte, from, to uint64, priority uint8) error { +func (self *StreamerPeer) Subscribe(s string, t []byte, from, to uint64, priority uint8, live bool) error { f, err := self.streamer.GetIncomingStreamer(s) if err != nil { return err @@ -459,7 +425,7 @@ func (self *StreamerPeer) Subscribe(s Stream, t []byte, from, to uint64, priorit if err != nil { return err } - err = self.setIncomingStreamer(s, is, priority) + err = self.setIncomingStreamer(s, is, priority, live) if err != nil { return err } @@ -484,8 +450,8 @@ func (self *StreamerPeer) handleSubscribeMsg(req *SubscribeMsg) error { if err != nil { return err } - key := string(req.Stream) + string(req.Key) - os, err := self.setOutgoingStreamer(Stream(key), s, req.Priority) + key := req.Stream + string(req.Key) + os, err := self.setOutgoingStreamer(key, s, req.Priority) if err != nil { return nil } @@ -531,7 +497,10 @@ func (self *StreamerPeer) handleUnsyncedKeysMsg(req *UnsyncedKeysMsg) error { }() // only send wantedKeysMsg if all missing chunks of the previous batch arrived // except - from, to := s.NextBatch(req.To) + if s.live { + s.sessionAt = req.From + } + from, to := s.nextBatch(req.To) if from == to { return nil } @@ -644,11 +613,11 @@ func (s *Streamer) Run(p *bzzPeer) error { // load saved intervals // autosubscribe to request handler to serve request only for non-light nodes // sp.handleSubscribeMsg(&SubscribeMsg{ - // Stream: retrieveRequestStream, + // Stream: retrieveRequeststring, // Priority: uint8(Top), // }) // subscribe to request handling ; only with non-light nodes - sp.Subscribe(retrieveRequestStream, nil, 0, 0, Top) + sp.Subscribe(retrieveRequestStream, nil, 0, 0, Top, true) defer close(sp.quit) return sp.Run(sp.HandleMsg) } diff --git a/swarm/network/syncer.go b/swarm/network/syncer.go index d0da14f78d..6a57b19181 100644 --- a/swarm/network/syncer.go +++ b/swarm/network/syncer.go @@ -95,8 +95,7 @@ func NewOutgoingSwarmSyncer(live bool, po uint8, db *DbAccess) (*OutgoingSwarmSy const maxPO = 32 func RegisterOutgoingSyncers(streamer *Streamer, db *DbAccess) { - stream := Stream("SYNC") - streamer.RegisterOutgoingStreamer(stream, func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) { + streamer.RegisterOutgoingStreamer("SYNC", func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) { syncType, po := parseSyncLabel(t) switch syncType { case "LIVE": @@ -107,7 +106,6 @@ func RegisterOutgoingSyncers(streamer *Streamer, db *DbAccess) { return nil, errors.New("invalid sync type") } }) - // stream = Stream(fmt.Sprintf("SYNC-%02d-delete", po)) // streamer.RegisterOutgoingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) { // return NewOutgoingProvableSwarmSyncer(po, db) // }) @@ -146,7 +144,6 @@ func (self *OutgoingSwarmSyncer) SetNextBatch(from, to uint64) ([]byte, uint64, type IncomingSwarmSyncer struct { sessionAt uint64 nextC chan struct{} - intervals *Intervals sessionRoot storage.Key sessionReader storage.LazySectionReader retrieveC chan *storage.Chunk @@ -159,11 +156,10 @@ type IncomingSwarmSyncer struct { } // NewIncomingSwarmSyncer is a contructor for provable data exchange syncer -func NewIncomingSwarmSyncer(intervals *Intervals, p Peer, dbAccess *DbAccess, chunker storage.Chunker) (*IncomingSwarmSyncer, error) { +func NewIncomingSwarmSyncer(p Peer, dbAccess *DbAccess, chunker storage.Chunker) (*IncomingSwarmSyncer, error) { self := &IncomingSwarmSyncer{ - intervals: intervals, - dbAccess: dbAccess, - chunker: chunker, + dbAccess: dbAccess, + chunker: chunker, } return self, nil } @@ -190,42 +186,6 @@ func NewIncomingSwarmSyncer(intervals *Intervals, p Peer, dbAccess *DbAccess, ch // return self // } -type Syncer struct { - intervals map[string][]uint64 -} - -func NewSyncer() *Syncer { - return &Syncer{} -} - -type Intervals struct { - syncer *Syncer - key []byte -} - -func (s *Intervals) load() error { - return s.syncer.load(s.key) -} - -func (s *Intervals) save() error { - return s.syncer.save(s.key) -} - -func (s *Intervals) get() []uint64 { - return s.syncer.get(s.key) -} - -func (s *Intervals) set(v []uint64) { - s.syncer.set(s.key, v) -} - -func (s *Syncer) NewIntervals(key []byte) *Intervals { - return &Intervals{ - syncer: s, - key: key, - } -} - func newSyncLabel(typ string, po uint8) []byte { t := []byte(typ) t = append(t, byte(po)) @@ -234,7 +194,7 @@ func newSyncLabel(typ string, po uint8) []byte { func parseSyncLabel(t []byte) (string, uint8) { l := len(t) - 1 - return sstring(t[:l]), uint8(t[l]) + return string(t[:l]), uint8(t[l]) } // StartSyncing is called on the StreamerPeer to start the syncing process @@ -245,21 +205,19 @@ func StartSyncing(s *StreamerPeer, po uint8, nn bool) { lastPO = maxPO } for i := po; i <= lastPO; i++ { - s.Subscribe(Stream("SYNC"), newSyncLabel("LIVE", po), 0, 0, High) - s.Subscribe(Stream("SYNC"), newSyncLabel("HISTORY", po), 0, 0, Mid) + s.Subscribe("SYNC", newSyncLabel("LIVE", po), 0, 0, High, true) + s.Subscribe("SYNC", newSyncLabel("HISTORY", po), 0, 0, Mid, false) } } -func RegisterIncomingSyncers(streamer *Streamer, syncer *Syncer, db *DbAccess) { - stream := Stream("SYNC") - streamer.RegisterIncomingStreamer(stream, func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { +func RegisterIncomingSyncers(streamer *Streamer, db *DbAccess) { + streamer.RegisterIncomingStreamer("SYNC", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { syncType, po := parseSyncLabel(t) switch syncType { case "LIVE": - return NewIncomingSwarmSyncer(nil, p, nil, nil) + return NewIncomingSwarmSyncer(p, nil, nil) case "HISTORY": - intervals := syncer.NewIntervals(t) - return NewIncomingSwarmSyncer(intervals, p, nil, nil) + return NewIncomingSwarmSyncer(p, nil, nil) } return nil, fmt.Errorf("unknown sync type %q", syncType) }) @@ -281,40 +239,15 @@ func (self *IncomingSwarmSyncer) NeedData(key []byte) (wait func()) { return chunk.WaitToStore } -// NextBatch adjusts the indexes by inspecting the intervals -func (self *IncomingSwarmSyncer) NextBatch(from uint64) (nextFrom uint64, nextTo uint64) { - intervals := self.intervals.get() - if intervals[0] >= self.sessionAt { // live syncing - nextFrom = from - intervals[1] = from - } else if from >= self.sessionAt { // history sync complete - intervals = nil - } else if len(intervals) > 2 && from >= intervals[2] { // filled a gap in the intervals - intervals = append(intervals[:1], intervals[3:]...) - nextFrom = intervals[1] - if len(intervals) > 2 { - nextTo = intervals[2] - } else { - nextTo = self.sessionAt - } - } else { - nextFrom = from - intervals[1] = from - nextTo = self.sessionAt - } - self.intervals.set(intervals) - return nextFrom, nextTo -} - // BatchDone -func (self *IncomingSwarmSyncer) BatchDone(s Stream, from uint64, hashes []byte, root []byte) func() (*TakeoverProof, error) { +func (self *IncomingSwarmSyncer) BatchDone(s string, from uint64, hashes []byte, root []byte) func() (*TakeoverProof, error) { if self.chunker != nil { return func() (*TakeoverProof, error) { return self.TakeoverProof(s, from, hashes, root) } } return nil } -func (self *IncomingSwarmSyncer) TakeoverProof(s Stream, from uint64, hashes []byte, root storage.Key) (*TakeoverProof, error) { +func (self *IncomingSwarmSyncer) TakeoverProof(s string, from uint64, hashes []byte, root storage.Key) (*TakeoverProof, error) { // for provable syncer currentRoot is non-zero length if self.chunker != nil { if from > self.sessionAt { // for live syncing currentRoot is always updated From e091310e950114321711089099b5e3d4c5c6f562 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Thu, 11 Jan 2018 12:19:00 +0100 Subject: [PATCH 019/210] Add error checking to hive test --- swarm/network/hive_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/swarm/network/hive_test.go b/swarm/network/hive_test.go index 8e49e9029d..37aebd363e 100644 --- a/swarm/network/hive_test.go +++ b/swarm/network/hive_test.go @@ -43,7 +43,7 @@ func TestRegisterAndConnect(t *testing.T) { pp.Start(s.Server) defer pp.Stop() // retrieve and broadcast - s.TestExchanges(p2ptest.Exchange{ + err := s.TestExchanges(p2ptest.Exchange{ Label: "getPeersMsg message", Expects: []p2ptest.Expect{ p2ptest.Expect{ @@ -53,4 +53,8 @@ func TestRegisterAndConnect(t *testing.T) { }, }, }) + + if err != nil { + t.Fatal(err) + } } From 4d4a67a9cbac7a1db7f82a5ff031468fb91cff15 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Thu, 11 Jan 2018 13:28:02 +0100 Subject: [PATCH 020/210] swarm/storage: Rebase resource update --- swarm/storage/resource.go | 57 ++-------------------------------- swarm/storage/resource_test.go | 44 ++++++++++++-------------- 2 files changed, 22 insertions(+), 79 deletions(-) diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 48bdc4f4aa..28b94fd803 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -4,7 +4,6 @@ import ( "crypto/ecdsa" "encoding/binary" "fmt" - "path/filepath" "strconv" "sync" "time" @@ -109,19 +108,9 @@ type ResourceHandler struct { } // Create or open resource update chunk store -func NewResourceHandler(privKey *ecdsa.PrivateKey, datadir string, cloudStore CloudStore, ethapi *rpc.Client) (*ResourceHandler, error) { - path := filepath.Join(datadir, "resource") - dbStore, err := NewDbStore(datadir, nil, singletonSwarmDbCapacity, 0) - if err != nil { - return nil, err - } - localStore := &LocalStore{ - memStore: NewMemStore(dbStore, singletonSwarmDbCapacity), - DbStore: dbStore, - } - hasher := MakeHashFunc("SHA3") +func NewResourceHandler(privKey *ecdsa.PrivateKey, hasher SwarmHasher, chunkStore ChunkStore, ethapi *rpc.Client) (*ResourceHandler, error) { return &ResourceHandler{ - ChunkStore: newResourceChunkStore(path, hasher, localStore, cloudStore), + ChunkStore: chunkStore, ethapi: ethapi, resources: make(map[string]*resource), hasher: hasher(), @@ -554,48 +543,6 @@ func (self *ResourceHandler) verifyContent(chunkdata []byte) error { return nil } -type resourceChunkStore struct { - localStore ChunkStore - netStore ChunkStore -} - -func newResourceChunkStore(path string, hasher SwarmHasher, localStore *LocalStore, cloudStore CloudStore) *resourceChunkStore { - return &resourceChunkStore{ - localStore: localStore, - netStore: NewNetStore(hasher, localStore, cloudStore, NewDefaultStoreParams()), - } -} - -func (r *resourceChunkStore) Get(key Key) (*Chunk, error) { - chunk, err := r.netStore.Get(key) - if err != nil { - return nil, err - } - // if the chunk has to be remotely retrieved, we define a timeout of how long to wait for it before failing. - // sadly due to the nature of swarm, the error will never be conclusive as to whether it was a network issue - // that caused the failure or that the chunk doesn't exist. - if chunk.Req == nil { - return chunk, nil - } - t := time.NewTimer(time.Second * 1) - select { - case <-t.C: - return nil, fmt.Errorf("timeout") - case <-chunk.C: - log.Trace("Received resource update chunk", "peer", chunk.Req.Source) - } - return chunk, nil -} - -func (r *resourceChunkStore) Put(chunk *Chunk) { - r.netStore.Put(chunk) -} - -func (r *resourceChunkStore) Close() { - r.netStore.Close() - r.localStore.Close() -} - func getNextBlock(start uint64, current uint64, frequency uint64) uint64 { blockdiff := current - start periods := (blockdiff / frequency) + 1 diff --git a/swarm/storage/resource_test.go b/swarm/storage/resource_test.go index db5a2d3ca1..cf9f300ad0 100644 --- a/swarm/storage/resource_test.go +++ b/swarm/storage/resource_test.go @@ -17,7 +17,6 @@ import ( "github.com/ethereum/go-ethereum/contracts/ens" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" ) @@ -26,10 +25,6 @@ var ( cleanF func() ) -func init() { - log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) -} - type FakeRPC struct { blockcount *uint64 } @@ -109,7 +104,7 @@ func TestResourceHandler(t *testing.T) { // check that the new resource is stored correctly namehash := ens.EnsNode(resourcevalidname) - chunk, err := rh.ChunkStore.(*resourceChunkStore).localStore.(*LocalStore).memStore.Get(Key(namehash[:])) + chunk, err := rh.ChunkStore.Get(Key(namehash[:])) if err != nil { teardownTest(t, err) } else if len(chunk.SData) < 16 { @@ -159,7 +154,10 @@ func TestResourceHandler(t *testing.T) { // it will match on second iteration startblocknumber + (resourcefrequency * 3) blockCount = startblocknumber + (resourcefrequency * 4) - rh2, err := NewResourceHandler(privkey, datadir, &testCloudStore{}, rh.ethapi) + rh2, err := newTestResourceHandler(datadir, privkey, rh.ethapi) + if err != nil { + teardownTest(t, err) + } _, err = rh2.LookupLatest(resourcename, true) if err != nil { teardownTest(t, err) @@ -273,7 +271,8 @@ func setupTest() (rh *ResourceHandler, privkey *ecdsa.PrivateKey, datadir string return } - rh, err = NewResourceHandler(privkey, datadir, &testCloudStore{}, rpcclient) + rh, err = newTestResourceHandler(datadir, privkey, rpcclient) + teardown = func(t *testing.T, err error) { cleanF() if err != nil { @@ -284,21 +283,18 @@ func setupTest() (rh *ResourceHandler, privkey *ecdsa.PrivateKey, datadir string return } -//func teardownTest(t *testing.T, errstr string) { -// cleanF() -// if errstr != "" { -// t.Fatal(errstr) -// } -//} +func newTestResourceHandler(datadir string, privkey *ecdsa.PrivateKey, rpcclient *rpc.Client) (*ResourceHandler, error) { + path := filepath.Join(datadir, "resource") + basekey := make([]byte, 32) + hasher := MakeHashFunc("SHA3") + dbStore, err := NewDbStore(path, hasher, singletonSwarmDbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) }) + if err != nil { + return nil, err + } + localStore := &LocalStore{ + memStore: NewMemStore(dbStore, singletonSwarmDbCapacity), + DbStore: dbStore, + } -type testCloudStore struct { -} - -func (c *testCloudStore) Store(*Chunk) { -} - -func (c *testCloudStore) Deliver(*Chunk) { -} - -func (c *testCloudStore) Retrieve(*Chunk) { + return NewResourceHandler(privkey, hasher, localStore, rpcclient) } From 00c7ab917847b8b12b6542b8600a57aaccd83b57 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Fri, 12 Jan 2018 09:38:43 +0100 Subject: [PATCH 021/210] Use discover.PubkeyID to generate node id in simulation --- p2p/simulations/adapters/types.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/p2p/simulations/adapters/types.go b/p2p/simulations/adapters/types.go index 5b4b47fe2f..3f76b19843 100644 --- a/p2p/simulations/adapters/types.go +++ b/p2p/simulations/adapters/types.go @@ -163,9 +163,8 @@ func RandomNodeConfig() *NodeConfig { if err != nil { panic("unable to generate key") } - var id discover.NodeID - pubkey := crypto.FromECDSAPub(&key.PublicKey) - copy(id[:], pubkey[1:]) + + id := discover.PubkeyID(&key.PublicKey) return &NodeConfig{ ID: id, PrivateKey: key, From c8f51a2e6dd62732c9327c0969c46dda966739e7 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Fri, 12 Jan 2018 09:39:26 +0100 Subject: [PATCH 022/210] Temporarily comment out unfinished code --- swarm/network/lightnode.go | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/swarm/network/lightnode.go b/swarm/network/lightnode.go index b351e87dcf..8ee7f5b22e 100644 --- a/swarm/network/lightnode.go +++ b/swarm/network/lightnode.go @@ -169,19 +169,17 @@ func RegisterRemoteSectionServer(s *Streamer, db *DbAccess, rf func([]byte) *sto // RegisterRemoteDownloader registers RemoteDownloader incoming streamer // on downstream light node -func RegisterRemoteDownloader(s *Streamer, db *DbAccess) { - s.RegisterIncomingStreamer("REMOTE_DOWNLOADER", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { - return NewRemoteDownloader(t, db), nil - }) -} - -// RegisterRemoteDownloadServer registers RemoteDownloadServer outgoing streamer on -// upstream light server node -func RegisterRemoteDownloadServer(s *Streamer, db *DbAccess, rf func([]byte) *storage.LazyChunkReader) { - s.RegisterOutgoingStreamer("REMOTE_DOWNLOADER", func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) { - r := rf(t) - return NewRemoteDownloadServer(db, r), nil - }) -} - -func NewRemoteDownloader() +// func RegisterRemoteDownloader(s *Streamer, db *DbAccess) { +// s.RegisterIncomingStreamer("REMOTE_DOWNLOADER", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { +// return NewRemoteDownloader(t, db), nil +// }) +// } +// +// // RegisterRemoteDownloadServer registers RemoteDownloadServer outgoing streamer on +// // upstream light server node +// func RegisterRemoteDownloadServer(s *Streamer, db *DbAccess, rf func([]byte) *storage.LazyChunkReader) { +// s.RegisterOutgoingStreamer("REMOTE_DOWNLOADER", func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) { +// r := rf(t) +// return NewRemoteDownloadServer(db, r), nil +// }) +// } From 01fd455d6b684bdaf0c134274b35fd1a81f6352b Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Fri, 12 Jan 2018 09:44:29 +0100 Subject: [PATCH 023/210] Temporarily remove/comment Intervals code Later we will have to decide what to do with it, but first let's make it compile --- swarm/network/requests.go | 44 +++++++++++++++++++-------------------- swarm/network/streamer.go | 22 ++++++++++---------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/swarm/network/requests.go b/swarm/network/requests.go index e3dd2846a1..ffb9855266 100644 --- a/swarm/network/requests.go +++ b/swarm/network/requests.go @@ -27,28 +27,28 @@ type Intervals struct { key string } -func (s *Intervals) load() error { - return s.streamer.load(s.key) -} - -func (s *Intervals) save() error { - return s.streamer.save(s.key) -} - -func (s *Intervals) get() []uint64 { - return s.streamer.get(s.key) -} - -func (s *Intervals) set(v []uint64) { - s.streamer.set(s.key, v) -} - -func NewIntervals(key string, s *Streamer) *Intervals { - return &Intervals{ - streamer: s, - key: key, - } -} +// func (s *Intervals) load() error { +// return s.streamer.load(s.key) +// } +// +// func (s *Intervals) save() error { +// return s.streamer.save(s.key) +// } +// +// func (s *Intervals) get() []uint64 { +// return s.streamer.get(s.key) +// } +// +// func (s *Intervals) set(v []uint64) { +// s.streamer.set(s.key, v) +// } +// +// func NewIntervals(key string, s *Streamer) *Intervals { +// return &Intervals{ +// streamer: s, +// key: key, +// } +// } // RetrieveRequestStreamer implements OutgoingStreamer type RetrieveRequestStreamer struct { diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go index b5651c4b87..7043145b55 100644 --- a/swarm/network/streamer.go +++ b/swarm/network/streamer.go @@ -370,17 +370,17 @@ func (self *StreamerPeer) setIncomingStreamer(s string, i IncomingStreamer, prio return fmt.Errorf("stream %v already registered", s) } next := make(chan struct{}, 1) - var intervals *Intervals - if !live { - key := s + self.ID().String() - intervals = NewIntervals(key, self.streamer) - } + // var intervals *Intervals + // if !live { + // key := s + self.ID().String() + // intervals = NewIntervals(key, self.streamer) + // } self.incoming[s] = &incomingStreamer{ IncomingStreamer: i, - intervals: intervals, - live: live, - priority: priority, - next: next, + // intervals: intervals, + live: live, + priority: priority, + next: next, } next <- struct{}{} // this is to allow wantedKeysMsg before first batch arrives return nil @@ -388,7 +388,7 @@ func (self *StreamerPeer) setIncomingStreamer(s string, i IncomingStreamer, prio // NextBatch adjusts the indexes by inspecting the intervals func (self *incomingStreamer) nextBatch(from uint64) (nextFrom uint64, nextTo uint64) { - intervals := self.intervals.get() + var intervals []uint64 if self.live { if len(intervals) == 0 { intervals = []uint64{self.sessionAt, from} @@ -411,7 +411,7 @@ func (self *incomingStreamer) nextBatch(from uint64) (nextFrom uint64, nextTo ui intervals[1] = from nextTo = self.sessionAt } - self.intervals.set(intervals) + // self.intervals.set(intervals) return nextFrom, nextTo } From debf3f69326a487181fd0fa04398ff0ecd633596 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Fri, 12 Jan 2018 09:50:23 +0100 Subject: [PATCH 024/210] Move Subscribe function from StreamerPeer to Streamer We manage a map in Streamer from the peer nodeids to the StreamerPeer instances. Subscribe is on StreamerPeer and receives a peer nodeId --- swarm/network/streamer.go | 35 +++++++++++++++++++++++++++++------ swarm/network/syncer.go | 10 ++++++---- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go index 7043145b55..4354a5eaf2 100644 --- a/swarm/network/streamer.go +++ b/swarm/network/streamer.go @@ -127,12 +127,14 @@ func (self WantedKeysMsg) String() string { type Streamer struct { incomingLock sync.RWMutex outgoingLock sync.RWMutex + peersLock sync.RWMutex outgoing map[string]func(*StreamerPeer, []byte) (OutgoingStreamer, error) incoming map[string]func(*StreamerPeer, []byte) (IncomingStreamer, error) dbAccess *DbAccess overlay Overlay receiveC chan *ChunkDeliveryMsg + peers map[discover.NodeID]*StreamerPeer } // NewStreamer is Streamer constructor @@ -143,6 +145,7 @@ func NewStreamer(overlay Overlay, dbAccess *DbAccess) *Streamer { dbAccess: dbAccess, overlay: overlay, receiveC: make(chan *ChunkDeliveryMsg, 10), + peers: make(map[discover.NodeID]*StreamerPeer), } } @@ -235,6 +238,7 @@ type StreamerPeer struct { // NewStreamerPeer is the constructor for StreamerPeer func NewStreamerPeer(p Peer, streamer *Streamer) *StreamerPeer { self := &StreamerPeer{ + Peer: p, pq: pq.New(int(PriorityQueue), PriorityQueueCap), streamer: streamer, outgoing: make(map[string]*outgoingStreamer), @@ -416,16 +420,24 @@ func (self *incomingStreamer) nextBatch(from uint64) (nextFrom uint64, nextTo ui } // Subscribe initiates the streamer -func (self *StreamerPeer) Subscribe(s string, t []byte, from, to uint64, priority uint8, live bool) error { - f, err := self.streamer.GetIncomingStreamer(s) +func (self *Streamer) Subscribe(peerId discover.NodeID, s string, t []byte, from, to uint64, priority uint8, live bool) error { + f, err := self.GetIncomingStreamer(s) if err != nil { return err } - is, err := f(self, t) + + self.peersLock.RLock() + peer := self.peers[peerId] + self.peersLock.RUnlock() + if peer == nil { + return fmt.Errorf("peer not found %v", peerId) + } + + is, err := f(peer, t) if err != nil { return err } - err = self.setIncomingStreamer(s, is, priority, live) + err = peer.setIncomingStreamer(s, is, priority, live) if err != nil { return err } @@ -437,7 +449,7 @@ func (self *StreamerPeer) Subscribe(s string, t []byte, from, to uint64, priorit To: to, Priority: priority, } - self.SendPriority(msg, priority) + peer.SendPriority(msg, priority) return nil } @@ -617,7 +629,18 @@ func (s *Streamer) Run(p *bzzPeer) error { // Priority: uint8(Top), // }) // subscribe to request handling ; only with non-light nodes - sp.Subscribe(retrieveRequestStream, nil, 0, 0, Top, true) + + s.peersLock.Lock() + s.peers[sp.ID()] = sp + s.peersLock.Unlock() + + defer func() { + s.peersLock.Lock() + delete(s.peers, sp.ID()) + s.peersLock.Unlock() + }() + + s.Subscribe(sp.ID(), retrieveRequestStream, nil, 0, 0, Top, true) defer close(sp.quit) return sp.Run(sp.HandleMsg) } diff --git a/swarm/network/syncer.go b/swarm/network/syncer.go index 6a57b19181..c9fe1d3555 100644 --- a/swarm/network/syncer.go +++ b/swarm/network/syncer.go @@ -23,6 +23,7 @@ import ( "io" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/swarm/storage" ) @@ -199,20 +200,21 @@ func parseSyncLabel(t []byte) (string, uint8) { // StartSyncing is called on the StreamerPeer to start the syncing process // the idea is that it is called only after kademlia is close to healthy -func StartSyncing(s *StreamerPeer, po uint8, nn bool) { +func StartSyncing(s *Streamer, peerId discover.NodeID, po uint8, nn bool) { lastPO := po if nn { lastPO = maxPO } + for i := po; i <= lastPO; i++ { - s.Subscribe("SYNC", newSyncLabel("LIVE", po), 0, 0, High, true) - s.Subscribe("SYNC", newSyncLabel("HISTORY", po), 0, 0, Mid, false) + s.Subscribe(peerId, "SYNC", newSyncLabel("LIVE", po), 0, 0, High, true) + s.Subscribe(peerId, "SYNC", newSyncLabel("HISTORY", po), 0, 0, Mid, false) } } func RegisterIncomingSyncers(streamer *Streamer, db *DbAccess) { streamer.RegisterIncomingStreamer("SYNC", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { - syncType, po := parseSyncLabel(t) + syncType, _ := parseSyncLabel(t) switch syncType { case "LIVE": return NewIncomingSwarmSyncer(p, nil, nil) From c0c45e3825bb3bd064c2ff68a3405ed043de8c4a Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Fri, 12 Jan 2018 09:50:53 +0100 Subject: [PATCH 025/210] Add new localStore constructor for tests --- swarm/storage/localstore.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index 843505c2cb..4fcddbf735 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -42,6 +42,20 @@ func NewLocalStore(hash SwarmHasher, params *StoreParams, basekey []byte) (*Loca }, nil } +func NewTestLocalStore(path string) (*LocalStore, error) { + basekey := make([]byte, 32) + hasher := MakeHashFunc("SHA3") + dbStore, err := NewDbStore(path, hasher, singletonSwarmDbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) }) + if err != nil { + return nil, err + } + localStore := &LocalStore{ + memStore: NewMemStore(dbStore, singletonSwarmDbCapacity), + DbStore: dbStore, + } + return localStore, nil +} + // LocalStore is itself a chunk store // unsafe, in that the data is not integrity checked func (self *LocalStore) Put(chunk *Chunk) { From 3836dd0da5b4d6a4000c13c9bb2eae9a8da5dbf1 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Fri, 12 Jan 2018 09:53:34 +0100 Subject: [PATCH 026/210] Start unit testing for Streamer --- swarm/network/streamer_test.go | 134 +++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 swarm/network/streamer_test.go diff --git a/swarm/network/streamer_test.go b/swarm/network/streamer_test.go new file mode 100644 index 0000000000..ba9800232e --- /dev/null +++ b/swarm/network/streamer_test.go @@ -0,0 +1,134 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package network + +import ( + "io/ioutil" + "os" + "testing" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/protocols" + p2ptest "github.com/ethereum/go-ethereum/p2p/testing" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +func init() { + log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlWarn, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) +} + +// TODO: extract newStreamer +func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Streamer, func(), error) { + // setup + addr := RandomAddr() // tested peers peer address + to := NewKademlia(addr.OAddr, NewKadParams()) + + // temp datadir + datadir, err := ioutil.TempDir("", "streamer") + if err != nil { + return nil, nil, func() {}, err + } + teardown := func() { + os.RemoveAll(datadir) + } + + localStore, err := storage.NewTestLocalStore(datadir) + if err != nil { + return nil, nil, teardown, err + } + + dbAccess := NewDbAccess(localStore) + streamer := NewStreamer(to, dbAccess) + + run := func(p *p2p.Peer, rw p2p.MsgReadWriter) error { + bzzPeer := &bzzPeer{ + Peer: protocols.NewPeer(p, rw, StreamerSpec), + localAddr: addr, + BzzAddr: NewAddrFromNodeID(p.ID()), + } + return streamer.Run(bzzPeer) + } + + protocolTester := p2ptest.NewProtocolTester(t, NewNodeIDFromAddr(addr), 1, run) + return protocolTester, streamer, teardown, nil +} + +// TODO +// func newStreamer() (*Streamer, error) { +// +// } + +func TestStreamerSubscribe(t *testing.T) { + tester, streamer, teardown, err := newStreamerTester(t) + defer teardown() + if err != nil { + t.Fatal(err) + } + + err = streamer.Subscribe(tester.IDs[0], "foo", nil, 0, 0, Top, true) + if err == nil || err.Error() != "stream foo not registered" { + t.Fatalf("Expected error %v, got %v", "stream foo not registered", err) + } +} + +type testIncomingStreamer struct { + t []byte +} + +func (self *testIncomingStreamer) NeedData([]byte) func() { + return nil +} + +func (self *testIncomingStreamer) BatchDone(string, uint64, []byte, []byte) func() (*TakeoverProof, error) { + return nil +} + +func TestStreamerRegisterIncoming(t *testing.T) { + // TODO: we only need streamer + tester, streamer, teardown, err := newStreamerTester(t) + defer teardown() + if err != nil { + t.Fatal(err) + } + + streamer.RegisterIncomingStreamer("foo", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { + return &testIncomingStreamer{ + t: t, + }, nil + }) + + tick := time.NewTicker(10 * time.Millisecond) + timeout := time.NewTimer(1 * time.Second) +WAIT: + for { + select { + case <-tick.C: + if len(streamer.peers) > 0 { + break WAIT + } + case <-timeout.C: + t.Fatal("timeout") + } + } + + err = streamer.Subscribe(tester.IDs[0], "foo", nil, 0, 0, Top, true) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } +} From 83b6cc42808e2ef844e17dd72c45608794760e66 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Fri, 12 Jan 2018 11:14:48 +0100 Subject: [PATCH 027/210] Extract thread safe functions to manipulate Streamer.peers map --- swarm/network/streamer.go | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go index 4354a5eaf2..04cf3b5f18 100644 --- a/swarm/network/streamer.go +++ b/swarm/network/streamer.go @@ -306,6 +306,30 @@ func (self *Streamer) Retrieve(chunk *storage.Chunk) error { return nil } +func (self *Streamer) getPeer(peerId discover.NodeID) *StreamerPeer { + if self.peers == nil { + return nil + } + self.peersLock.RLock() + defer self.peersLock.RUnlock() + return self.peers[peerId] +} + +func (self *Streamer) setPeer(peer *StreamerPeer) { + if self.peers == nil { + self.peers = make(map[discover.NodeID]*StreamerPeer) + } + self.peersLock.Lock() + self.peers[peer.ID()] = peer + self.peersLock.Unlock() +} + +func (self *Streamer) deletePeer(peer *StreamerPeer) { + self.peersLock.Lock() + delete(self.peers, peer.ID()) + self.peersLock.Unlock() +} + func (self *StreamerPeer) handleChunkDeliveryMsg(req *ChunkDeliveryMsg) error { chunk, err := self.dbAccess.get(req.Key) if err != nil { @@ -426,9 +450,7 @@ func (self *Streamer) Subscribe(peerId discover.NodeID, s string, t []byte, from return err } - self.peersLock.RLock() - peer := self.peers[peerId] - self.peersLock.RUnlock() + peer := self.getPeer(peerId) if peer == nil { return fmt.Errorf("peer not found %v", peerId) } @@ -630,15 +652,9 @@ func (s *Streamer) Run(p *bzzPeer) error { // }) // subscribe to request handling ; only with non-light nodes - s.peersLock.Lock() - s.peers[sp.ID()] = sp - s.peersLock.Unlock() + s.setPeer(sp) - defer func() { - s.peersLock.Lock() - delete(s.peers, sp.ID()) - s.peersLock.Unlock() - }() + defer s.deletePeer(sp) s.Subscribe(sp.ID(), retrieveRequestStream, nil, 0, 0, Top, true) defer close(sp.quit) From 852a6d669be506f3aeca036df2149c5e9010448b Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Fri, 12 Jan 2018 11:29:33 +0100 Subject: [PATCH 028/210] Extract waiting loop for peers into a function --- swarm/network/streamer_test.go | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/swarm/network/streamer_test.go b/swarm/network/streamer_test.go index ba9800232e..93ae2551d0 100644 --- a/swarm/network/streamer_test.go +++ b/swarm/network/streamer_test.go @@ -17,6 +17,7 @@ package network import ( + "errors" "io/ioutil" "os" "testing" @@ -113,18 +114,9 @@ func TestStreamerRegisterIncoming(t *testing.T) { }, nil }) - tick := time.NewTicker(10 * time.Millisecond) - timeout := time.NewTimer(1 * time.Second) -WAIT: - for { - select { - case <-tick.C: - if len(streamer.peers) > 0 { - break WAIT - } - case <-timeout.C: - t.Fatal("timeout") - } + err = waitForPeers(streamer, 1*time.Second) + if err != nil { + t.Fatal("timeout: peer is not created") } err = streamer.Subscribe(tester.IDs[0], "foo", nil, 0, 0, Top, true) @@ -132,3 +124,18 @@ WAIT: t.Fatalf("Expected no error, got %v", err) } } + +func waitForPeers(streamer *Streamer, timeout time.Duration) error { + ticker := time.NewTicker(10 * time.Millisecond) + timeoutTimer := time.NewTimer(timeout) + for { + select { + case <-ticker.C: + if len(streamer.peers) > 0 { + return nil + } + case <-timeoutTimer.C: + return errors.New("timeout") + } + } +} From 23861e0fa85e67276922dbff76089558ecfa4bda Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Fri, 12 Jan 2018 14:01:21 +0100 Subject: [PATCH 029/210] Rename messages --- swarm/network/streamer.go | 56 +++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go index 04cf3b5f18..af9730a971 100644 --- a/swarm/network/streamer.go +++ b/swarm/network/streamer.go @@ -82,9 +82,9 @@ type SubscribeMsg struct { Priority uint8 // delivered on priority channel } -// UnsyncedKeysMsg is the protocol msg for offering to hand over a +// OfferedHashesMsg is the protocol msg for offering to hand over a // stream section -type UnsyncedKeysMsg struct { +type OfferedHashesMsg struct { Stream string // name of Stream Key []byte // subtype or key From, To uint64 // peer and db-specific entry count @@ -104,22 +104,22 @@ type ChunkDeliveryMsg struct { from Peer // [not serialised] protocol registers the requester } -// String pretty prints UnsyncedKeysMsg -func (self UnsyncedKeysMsg) String() string { +// String pretty prints OfferedHashesMsg +func (self OfferedHashesMsg) String() string { return fmt.Sprintf("Stream '%v' [%v-%v] (%v)", self.Stream, self.From, self.To, len(self.Hashes)/HashSize) } -// WantedKeysMsg is the protocol msg data for signaling which hashes -// offered in UnsyncedKeysMsg downstream peer actually wants sent over -type WantedKeysMsg struct { +// WantedHashesMsg is the protocol msg data for signaling which hashes +// offered in OfferedHashesMsg downstream peer actually wants sent over +type WantedHashesMsg struct { Stream string // name of stream Key []byte // subtype or key Want []byte // bitvector indicating which keys of the batch needed From, To uint64 // next interval offset - empty if not to be continued } -// String pretty prints WantedKeysMsg -func (self WantedKeysMsg) String() string { +// String pretty prints WantedHashesMsg +func (self WantedHashesMsg) String() string { return fmt.Sprintf("Stream '%v', Want: %x, Next: [%v-%v]", self.Stream, self.Want, self.From, self.To) } @@ -410,7 +410,7 @@ func (self *StreamerPeer) setIncomingStreamer(s string, i IncomingStreamer, prio priority: priority, next: next, } - next <- struct{}{} // this is to allow wantedKeysMsg before first batch arrives + next <- struct{}{} // this is to allow wantedHashesMsg before first batch arrives return nil } @@ -489,13 +489,13 @@ func (self *StreamerPeer) handleSubscribeMsg(req *SubscribeMsg) error { if err != nil { return nil } - go self.SendUnsyncedKeys(os, req.From, req.To) + go self.SendOfferedHashes(os, req.From, req.To) return nil } -// handleUnsyncedKeysMsg protocol msg handler calls the incoming streamer interface +// handleOfferedHashesMsg protocol msg handler calls the incoming streamer interface // Filter method -func (self *StreamerPeer) handleUnsyncedKeysMsg(req *UnsyncedKeysMsg) error { +func (self *StreamerPeer) handleOfferedHashesMsg(req *OfferedHashesMsg) error { s, err := self.getIncomingStreamer(req.Stream) if err != nil { return err @@ -529,7 +529,7 @@ func (self *StreamerPeer) handleUnsyncedKeysMsg(req *UnsyncedKeysMsg) error { } s.next <- struct{}{} }() - // only send wantedKeysMsg if all missing chunks of the previous batch arrived + // only send wantedHashesMsg if all missing chunks of the previous batch arrived // except if s.live { s.sessionAt = req.From @@ -538,7 +538,7 @@ func (self *StreamerPeer) handleUnsyncedKeysMsg(req *UnsyncedKeysMsg) error { if from == to { return nil } - msg := &WantedKeysMsg{ + msg := &WantedHashesMsg{ Stream: req.Stream, Want: want.Bytes(), From: from, @@ -555,17 +555,17 @@ func (self *StreamerPeer) handleUnsyncedKeysMsg(req *UnsyncedKeysMsg) error { return nil } -// handleWantedKeysMsg protocol msg handler +// handleWantedHashesMsg protocol msg handler // * sends the next batch of unsynced keys -// * sends the actual data chunks as per WantedKeysMsg -func (self *StreamerPeer) handleWantedKeysMsg(req *WantedKeysMsg) error { +// * sends the actual data chunks as per WantedHashesMsg +func (self *StreamerPeer) handleWantedHashesMsg(req *WantedHashesMsg) error { s, err := self.getOutgoingStreamer(req.Stream) if err != nil { return err } hashes := s.currentBatch // launch in go routine since GetBatch blocks until new hashes arrive - go self.SendUnsyncedKeys(s, req.From, req.To) + go self.SendOfferedHashes(s, req.From, req.To) l := len(hashes) / HashSize want, err := bv.NewFromBytes(req.Want, l) if err != nil { @@ -611,14 +611,14 @@ func (self *StreamerPeer) SendPriority(msg interface{}, priority uint8) error { return self.pq.Push(nil, msg, int(priority)) } -// UnsyncedKeys sends UnsyncedKeysMsg protocol msg -func (self *StreamerPeer) SendUnsyncedKeys(s *outgoingStreamer, f, t uint64) error { +// OfferedHashes sends OfferedHashesMsg protocol msg +func (self *StreamerPeer) SendOfferedHashes(s *outgoingStreamer, f, t uint64) error { hashes, from, to, proof, err := s.SetNextBatch(f, t) if err != nil { return err } s.currentBatch = hashes - msg := &UnsyncedKeysMsg{ + msg := &OfferedHashesMsg{ HandoverProof: proof, Hashes: hashes, From: from, @@ -634,8 +634,8 @@ var StreamerSpec = &protocols.Spec{ MaxMsgSize: 10 * 1024 * 1024, Messages: []interface{}{ HandshakeMsg{}, - UnsyncedKeysMsg{}, - WantedKeysMsg{}, + OfferedHashesMsg{}, + WantedHashesMsg{}, TakeoverProofMsg{}, SubscribeMsg{}, }, @@ -668,14 +668,14 @@ func (self *StreamerPeer) HandleMsg(msg interface{}) error { case *SubscribeMsg: return self.handleSubscribeMsg(msg) - case *UnsyncedKeysMsg: - return self.handleUnsyncedKeysMsg(msg) + case *OfferedHashesMsg: + return self.handleOfferedHashesMsg(msg) case *TakeoverProofMsg: return self.handleTakeoverProofMsg(msg) - case *WantedKeysMsg: - return self.handleWantedKeysMsg(msg) + case *WantedHashesMsg: + return self.handleWantedHashesMsg(msg) case *ChunkDeliveryMsg: return self.handleChunkDeliveryMsg(msg) From 8c22fb87bf708213694db724d1647eccd2872341 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Fri, 12 Jan 2018 14:02:04 +0100 Subject: [PATCH 030/210] swarm/network: Added test to registering outgoing streamer --- swarm/network/streamer_test.go | 92 +++++++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) diff --git a/swarm/network/streamer_test.go b/swarm/network/streamer_test.go index 93ae2551d0..ac0f3de754 100644 --- a/swarm/network/streamer_test.go +++ b/swarm/network/streamer_test.go @@ -92,6 +92,10 @@ type testIncomingStreamer struct { t []byte } +type testOutgoingStreamer struct { + t []byte +} + func (self *testIncomingStreamer) NeedData([]byte) func() { return nil } @@ -100,6 +104,14 @@ func (self *testIncomingStreamer) BatchDone(string, uint64, []byte, []byte) func return nil } +func (self *testOutgoingStreamer) SetNextBatch(from uint64, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) { + return make([]byte, HashSize), from + 1, to + 1, nil, nil +} + +func (self *testOutgoingStreamer) GetData([]byte) []byte { + return nil +} + func TestStreamerRegisterIncoming(t *testing.T) { // TODO: we only need streamer tester, streamer, teardown, err := newStreamerTester(t) @@ -119,10 +131,88 @@ func TestStreamerRegisterIncoming(t *testing.T) { t.Fatal("timeout: peer is not created") } - err = streamer.Subscribe(tester.IDs[0], "foo", nil, 0, 0, Top, true) + peerId := tester.IDs[0] + + err = streamer.Subscribe(peerId, "foo", []byte{}, 5, 8, Top, true) if err != nil { t.Fatalf("Expected no error, got %v", err) } + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "Subscribe message", + Expects: []p2ptest.Expect{ + p2ptest.Expect{ + Code: 4, + Msg: &SubscribeMsg{ + Stream: "foo", + Key: []byte{}, + From: 5, + To: 8, + Priority: Top, + }, + Peer: peerId, + }, + }, + }) + + if err != nil { + t.Fatal(err) + } +} + +func TestStreamerRegisterOutgoing(t *testing.T) { + // TODO: we only need streamer + tester, streamer, teardown, err := newStreamerTester(t) + defer teardown() + if err != nil { + t.Fatal(err) + } + + streamer.RegisterOutgoingStreamer("foo", func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) { + return &testOutgoingStreamer{ + t: t, + }, nil + }) + + err = waitForPeers(streamer, 1*time.Second) + if err != nil { + t.Fatal("timeout: peer is not created") + } + + peerId := tester.IDs[0] + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "Subscribe message", + Triggers: []p2ptest.Trigger{ + p2ptest.Trigger{ + Code: 4, + Msg: &SubscribeMsg{ + Stream: "foo", + Key: []byte{}, + From: 5, + To: 8, + Priority: Top, + }, + Peer: peerId, + }, + }, + Expects: []p2ptest.Expect{ + p2ptest.Expect{ + Code: 1, + Msg: &OfferedHashesMsg{ + HandoverProof: nil, + Hashes: make([]byte, HashSize), + From: 6, + To: 9, + }, + Peer: peerId, + }, + }, + }) + + if err != nil { + t.Fatal(err) + } } func waitForPeers(streamer *Streamer, timeout time.Duration) error { From 2efb994c19ed16d6e6d23df69561c766deb9103f Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Fri, 12 Jan 2018 20:00:44 +0100 Subject: [PATCH 031/210] swarm/network: Move registering streamers to streamer constructor --- swarm/network/requests.go | 10 ---------- swarm/network/streamer.go | 7 +++++++ 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/swarm/network/requests.go b/swarm/network/requests.go index ffb9855266..bea5ff09de 100644 --- a/swarm/network/requests.go +++ b/swarm/network/requests.go @@ -58,16 +58,6 @@ type RetrieveRequestStreamer struct { currentLen uint64 } -// RegisterRequestStreamer registers outgoing and incoming streamers for request handling -func RegisterRequestStreamer(streamer *Streamer, db *DbAccess) { - streamer.RegisterOutgoingStreamer(retrieveRequestStream, func(_ *StreamerPeer, t []byte) (OutgoingStreamer, error) { - return NewRetrieveRequestStreamer(db), nil - }) - streamer.RegisterIncomingStreamer(retrieveRequestStream, func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { - return NewIncomingSwarmSyncer(p, db, nil) - }) -} - // NewRetrieveRequestStreamer is RetrieveRequestStreamer constructor func NewRetrieveRequestStreamer(db *DbAccess) *RetrieveRequestStreamer { s := &RetrieveRequestStreamer{ diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go index af9730a971..9c6abc8bed 100644 --- a/swarm/network/streamer.go +++ b/swarm/network/streamer.go @@ -147,6 +147,13 @@ func NewStreamer(overlay Overlay, dbAccess *DbAccess) *Streamer { receiveC: make(chan *ChunkDeliveryMsg, 10), peers: make(map[discover.NodeID]*StreamerPeer), } + streamer.RegisterOutgoingStreamer(retrieveRequestStream, func(_ *StreamerPeer, t []byte) (OutgoingStreamer, error) { + return NewRetrieveRequestStreamer(dbAccess), nil + }) + streamer.RegisterIncomingStreamer(retrieveRequestStream, func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { + return NewIncomingSwarmSyncer(p, dbAccess, nil) + }) + return streamer } // RegisterIncomingStreamer registers an incoming streamer constructor From 1c7805bde450bb32579b0bdd67f54a7af91b988e Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Fri, 12 Jan 2018 20:04:37 +0100 Subject: [PATCH 032/210] swarm/network: Wait in processDeliveries until hash is ready --- swarm/network/requests.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/swarm/network/requests.go b/swarm/network/requests.go index bea5ff09de..ce68afd35d 100644 --- a/swarm/network/requests.go +++ b/swarm/network/requests.go @@ -72,11 +72,13 @@ func NewRetrieveRequestStreamer(db *DbAccess) *RetrieveRequestStreamer { // processDeliveries handles delivered chunk hashes func (s *RetrieveRequestStreamer) processDeliveries() { var hashes []byte + var batchC chan []byte for { select { case delivery := <-s.deliveryC: hashes = append(hashes, delivery.Key[:]...) - case s.batchC <- hashes: + batchC = s.batchC + case batchC <- hashes: hashes = nil } } From 3956470fce0857d91d96933ef93901c0909dd00c Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Fri, 12 Jan 2018 20:05:16 +0100 Subject: [PATCH 033/210] swarm/network: Move hashSize to end of const block --- swarm/network/streamer.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go index 9c6abc8bed..de259ba8fb 100644 --- a/swarm/network/streamer.go +++ b/swarm/network/streamer.go @@ -32,14 +32,13 @@ import ( ) const ( - HashSize = 32 - Low uint8 = iota Mid High Top PriorityQueue // number of queues PriorityQueueCap = 3 // queue capacity + HashSize = 32 ) // Handover represents a statement that the upstream peer hands over the stream section From 1d7d6c9049cf4d0c9a7dd02494cae2188984dc99 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Fri, 12 Jan 2018 20:06:41 +0100 Subject: [PATCH 034/210] swarm/network: Allow nil HandoverProof in OfferedHashesMsg --- swarm/network/streamer.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go index de259ba8fb..54b33b3fa6 100644 --- a/swarm/network/streamer.go +++ b/swarm/network/streamer.go @@ -84,11 +84,11 @@ type SubscribeMsg struct { // OfferedHashesMsg is the protocol msg for offering to hand over a // stream section type OfferedHashesMsg struct { - Stream string // name of Stream - Key []byte // subtype or key - From, To uint64 // peer and db-specific entry count - Hashes []byte // stream of hashes (128) - *HandoverProof // HandoverProof + Stream string // name of Stream + Key []byte // subtype or key + From, To uint64 // peer and db-specific entry count + Hashes []byte // stream of hashes (128) + *HandoverProof `rlp:"nil"` // HandoverProof } /* From bab67a45bea84ffed8171adf24947a382c4fa6d7 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Fri, 12 Jan 2018 20:09:27 +0100 Subject: [PATCH 035/210] swarm/network: No need to store dbAccess on StreamerPeer --- swarm/network/streamer.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go index 54b33b3fa6..72801d5a08 100644 --- a/swarm/network/streamer.go +++ b/swarm/network/streamer.go @@ -138,7 +138,7 @@ type Streamer struct { // NewStreamer is Streamer constructor func NewStreamer(overlay Overlay, dbAccess *DbAccess) *Streamer { - return &Streamer{ + streamer := &Streamer{ outgoing: make(map[string]func(*StreamerPeer, []byte) (OutgoingStreamer, error)), incoming: make(map[string]func(*StreamerPeer, []byte) (IncomingStreamer, error)), dbAccess: dbAccess, @@ -233,7 +233,6 @@ type StreamerPeer struct { streamer *Streamer pq *pq.PriorityQueue //netStore storage.ChunkStore - dbAccess *DbAccess outgoingLock sync.RWMutex incomingLock sync.RWMutex outgoing map[string]*outgoingStreamer @@ -266,7 +265,7 @@ type RetrieveRequestMsg struct { } func (self *StreamerPeer) handleRetrieveRequestMsg(req *RetrieveRequestMsg) error { - chunk, created := self.dbAccess.getOrCreateRequest(req.Key) + chunk, created := self.streamer.dbAccess.getOrCreateRequest(req.Key) s, err := self.getOutgoingStreamer(retrieveRequestStream) if err != nil { return err @@ -337,7 +336,7 @@ func (self *Streamer) deletePeer(peer *StreamerPeer) { } func (self *StreamerPeer) handleChunkDeliveryMsg(req *ChunkDeliveryMsg) error { - chunk, err := self.dbAccess.get(req.Key) + chunk, err := self.streamer.dbAccess.get(req.Key) if err != nil { return err } From 2f703edd1223fde6f4c893db1d4359d92e444845 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Fri, 12 Jan 2018 20:10:38 +0100 Subject: [PATCH 036/210] swarm/network: Add missing RetrieveRequestMsg in spec in and handler --- swarm/network/streamer.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go index 72801d5a08..1e78a49adf 100644 --- a/swarm/network/streamer.go +++ b/swarm/network/streamer.go @@ -643,6 +643,7 @@ var StreamerSpec = &protocols.Spec{ WantedHashesMsg{}, TakeoverProofMsg{}, SubscribeMsg{}, + RetrieveRequestMsg{}, }, } @@ -685,6 +686,9 @@ func (self *StreamerPeer) HandleMsg(msg interface{}) error { case *ChunkDeliveryMsg: return self.handleChunkDeliveryMsg(msg) + case *RetrieveRequestMsg: + return self.handleRetrieveRequestMsg(msg) + default: return fmt.Errorf("unknown message type: %T", msg) } From 646d46614c2e487b62e9ad4d90460afdc8eab6db Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Fri, 12 Jan 2018 20:13:30 +0100 Subject: [PATCH 037/210] swarm/network: Fix loop variable --- swarm/network/streamer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go index 1e78a49adf..40fb038aa8 100644 --- a/swarm/network/streamer.go +++ b/swarm/network/streamer.go @@ -511,10 +511,10 @@ func (self *StreamerPeer) handleOfferedHashesMsg(req *OfferedHashesMsg) error { return err } wg := sync.WaitGroup{} - for i := 0; i < len(hashes)/HashSize; i += HashSize { + for i := 0; i < len(hashes); i += HashSize { hash := hashes[i : i+HashSize] if wait := s.NeedData(hash); wait != nil { - want.Set(i, true) + want.Set(i/HashSize, true) wg.Add(1) // create request and wait until the chunk data arrives and is stored go func(w func()) { From b2f0655c57da2e956e259e038a647d246ce91566 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Fri, 12 Jan 2018 20:15:00 +0100 Subject: [PATCH 038/210] swarm/network: Fix comments --- swarm/network/streamer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go index 40fb038aa8..c32039098e 100644 --- a/swarm/network/streamer.go +++ b/swarm/network/streamer.go @@ -415,7 +415,7 @@ func (self *StreamerPeer) setIncomingStreamer(s string, i IncomingStreamer, prio priority: priority, next: next, } - next <- struct{}{} // this is to allow wantedHashesMsg before first batch arrives + next <- struct{}{} // this is to allow wantedKeysMsg before first batch arrives return nil } @@ -534,7 +534,7 @@ func (self *StreamerPeer) handleOfferedHashesMsg(req *OfferedHashesMsg) error { } s.next <- struct{}{} }() - // only send wantedHashesMsg if all missing chunks of the previous batch arrived + // only send wantedKeysMsg if all missing chunks of the previous batch arrived // except if s.live { s.sessionAt = req.From From 791cac2403bb11a73a1a0d9c34df7849323bd1f6 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Fri, 12 Jan 2018 20:24:18 +0100 Subject: [PATCH 039/210] swarm/network: Streamer.Retrieve allows peer list to skip --- swarm/network/streamer.go | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go index c32039098e..7911161452 100644 --- a/swarm/network/streamer.go +++ b/swarm/network/streamer.go @@ -273,8 +273,8 @@ func (self *StreamerPeer) handleRetrieveRequestMsg(req *RetrieveRequestMsg) erro streamer := s.OutgoingStreamer.(*RetrieveRequestStreamer) if chunk.ReqC != nil { if created { - if err := self.streamer.Retrieve(chunk); err != nil { - return err + if err := self.streamer.Retrieve(chunk, self.ID()); err != nil { + return nil } } go func() { @@ -299,16 +299,27 @@ func (self *StreamerPeer) handleRetrieveRequestMsg(req *RetrieveRequestMsg) erro } // Retrieve sends a chunk retrieve request to -func (self *Streamer) Retrieve(chunk *storage.Chunk) error { +func (self *Streamer) Retrieve(chunk *storage.Chunk, peersToSkip ...discover.NodeID) error { + var success bool self.overlay.EachConn(chunk.Key[:], 255, func(p OverlayConn, po int, nn bool) bool { - sp := p.(*StreamerPeer) + spId := p.(Peer).ID() + for _, p := range peersToSkip { + if p == spId { + return true + } + } + sp := self.getPeer(spId) // TODO: skip light nodes that do not accept retrieve requests sp.SendPriority(&RetrieveRequestMsg{ Key: chunk.Key[:], }, Top) + success = true return false }) - return nil + if success { + return nil + } + return errors.New("no peer found") } func (self *Streamer) getPeer(peerId discover.NodeID) *StreamerPeer { @@ -391,6 +402,7 @@ func (self *StreamerPeer) setOutgoingStreamer(s string, o OutgoingStreamer, prio os := &outgoingStreamer{ OutgoingStreamer: o, priority: priority, + stream: s, } self.outgoing[s] = os return os, nil From dc134e0219814e4dcf13879753ef692c4e79c0d8 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Fri, 12 Jan 2018 20:24:45 +0100 Subject: [PATCH 040/210] swarm/network: Store stream in outgoingStreamer --- swarm/network/streamer.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go index 7911161452..365222ab17 100644 --- a/swarm/network/streamer.go +++ b/swarm/network/streamer.go @@ -203,6 +203,7 @@ type outgoingStreamer struct { OutgoingStreamer priority uint8 currentBatch []byte + stream string } // OutgoingStreamer interface for outgoing peer Streamer @@ -640,6 +641,9 @@ func (self *StreamerPeer) SendOfferedHashes(s *outgoingStreamer, f, t uint64) er Hashes: hashes, From: from, To: to, + Stream: s.stream, + // TODO: use real key here + Key: []byte{}, } return self.SendPriority(msg, s.priority) } From 3af844d37cc6b531c0e81b8e656026dc3b88c149 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Fri, 12 Jan 2018 20:25:12 +0100 Subject: [PATCH 041/210] swarm/network: Added a bunch of (failing) streamer unit tests --- swarm/network/streamer_test.go | 363 ++++++++++++++++++++++++++++++++- 1 file changed, 353 insertions(+), 10 deletions(-) diff --git a/swarm/network/streamer_test.go b/swarm/network/streamer_test.go index ac0f3de754..6a75a163fc 100644 --- a/swarm/network/streamer_test.go +++ b/swarm/network/streamer_test.go @@ -17,12 +17,14 @@ package network import ( + "bytes" "errors" "io/ioutil" "os" "testing" "time" + sha3 "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/protocols" @@ -35,7 +37,7 @@ func init() { } // TODO: extract newStreamer -func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Streamer, func(), error) { +func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Streamer, *storage.LocalStore, func(), error) { // setup addr := RandomAddr() // tested peers peer address to := NewKademlia(addr.OAddr, NewKadParams()) @@ -43,7 +45,7 @@ func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Streamer, func() // temp datadir datadir, err := ioutil.TempDir("", "streamer") if err != nil { - return nil, nil, func() {}, err + return nil, nil, nil, func() {}, err } teardown := func() { os.RemoveAll(datadir) @@ -51,7 +53,7 @@ func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Streamer, func() localStore, err := storage.NewTestLocalStore(datadir) if err != nil { - return nil, nil, teardown, err + return nil, nil, nil, teardown, err } dbAccess := NewDbAccess(localStore) @@ -63,11 +65,12 @@ func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Streamer, func() localAddr: addr, BzzAddr: NewAddrFromNodeID(p.ID()), } + to.On(bzzPeer) return streamer.Run(bzzPeer) } protocolTester := p2ptest.NewProtocolTester(t, NewNodeIDFromAddr(addr), 1, run) - return protocolTester, streamer, teardown, nil + return protocolTester, streamer, localStore, teardown, nil } // TODO @@ -76,7 +79,7 @@ func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Streamer, func() // } func TestStreamerSubscribe(t *testing.T) { - tester, streamer, teardown, err := newStreamerTester(t) + tester, streamer, _, teardown, err := newStreamerTester(t) defer teardown() if err != nil { t.Fatal(err) @@ -88,6 +91,18 @@ func TestStreamerSubscribe(t *testing.T) { } } +var ( + hash0 = sha3.Sum256([]byte{0}) + hash1 = sha3.Sum256([]byte{1}) + hash2 = sha3.Sum256([]byte{2}) + hashesTmp = append(hash0[:], hash1[:]...) + hashes = append(hashesTmp, hash2[:]...) + receivedHashes map[string][]byte = make(map[string][]byte) + wait0 = make(chan bool) + wait2 = make(chan bool) + batchDone = make(chan bool) +) + type testIncomingStreamer struct { t []byte } @@ -96,11 +111,22 @@ type testOutgoingStreamer struct { t []byte } -func (self *testIncomingStreamer) NeedData([]byte) func() { +func (self *testIncomingStreamer) NeedData(hash []byte) func() { + receivedHashes[string(hash)] = hash + if bytes.Equal(hash, hash0[:]) { + return func() { + <-wait0 + } + } else if bytes.Equal(hash, hash2[:]) { + return func() { + <-wait2 + } + } return nil } func (self *testIncomingStreamer) BatchDone(string, uint64, []byte, []byte) func() (*TakeoverProof, error) { + close(batchDone) return nil } @@ -112,9 +138,9 @@ func (self *testOutgoingStreamer) GetData([]byte) []byte { return nil } -func TestStreamerRegisterIncoming(t *testing.T) { +func TestStreamerDownstreamSubscribeMsgExchange(t *testing.T) { // TODO: we only need streamer - tester, streamer, teardown, err := newStreamerTester(t) + tester, streamer, _, teardown, err := newStreamerTester(t) defer teardown() if err != nil { t.Fatal(err) @@ -160,9 +186,9 @@ func TestStreamerRegisterIncoming(t *testing.T) { } } -func TestStreamerRegisterOutgoing(t *testing.T) { +func TestStreamerUpstreamSubscribeMsgExchange(t *testing.T) { // TODO: we only need streamer - tester, streamer, teardown, err := newStreamerTester(t) + tester, streamer, _, teardown, err := newStreamerTester(t) defer teardown() if err != nil { t.Fatal(err) @@ -210,6 +236,323 @@ func TestStreamerRegisterOutgoing(t *testing.T) { }, }) + if err != nil { + t.Fatal(err) + } + +} + +func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) { + // TODO: we only need streamer + tester, streamer, _, teardown, err := newStreamerTester(t) + defer teardown() + if err != nil { + t.Fatal(err) + } + + streamer.RegisterIncomingStreamer("foo", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { + return &testIncomingStreamer{ + t: t, + }, nil + }) + + err = waitForPeers(streamer, 1*time.Second) + if err != nil { + t.Fatal("timeout: peer is not created") + } + + peerId := tester.IDs[0] + + err = streamer.Subscribe(peerId, "foo", []byte{}, 5, 8, Top, true) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "Subscribe message", + Expects: []p2ptest.Expect{ + p2ptest.Expect{ + Code: 4, + Msg: &SubscribeMsg{ + Stream: "foo", + Key: []byte{}, + From: 5, + To: 8, + Priority: Top, + }, + Peer: peerId, + }, + }, + }, + p2ptest.Exchange{ + Label: "WantedHashes message", + Triggers: []p2ptest.Trigger{ + p2ptest.Trigger{ + Code: 1, + Msg: &OfferedHashesMsg{ + HandoverProof: &HandoverProof{ + Handover: &Handover{}, + }, + Hashes: hashes, + From: 5, + To: 8, + Stream: "foo", + }, + Peer: peerId, + }, + }, + Expects: []p2ptest.Expect{ + p2ptest.Expect{ + Code: 2, + Msg: &WantedHashesMsg{ + Stream: "foo", + Want: []byte{5}, + From: 8, + To: 0, + }, + Peer: peerId, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + + if len(receivedHashes) != 3 { + t.Fatalf("Expected number of received hashes %v, got %v", 3, len(receivedHashes)) + } + + close(wait0) + + timeout := time.NewTimer(100 * time.Millisecond) + defer timeout.Stop() + + select { + case <-batchDone: + t.Fatal("batch done early") + case <-timeout.C: + } + + close(wait2) + + timeout2 := time.NewTimer(10000 * time.Millisecond) + defer timeout2.Stop() + + select { + case <-batchDone: + case <-timeout2.C: + t.Fatal("timeout waiting batchdone call") + } + +} + +func TestRetrieveRequest(t *testing.T) { + // TODO: we only need streamer + tester, streamer, _, teardown, err := newStreamerTester(t) + defer teardown() + if err != nil { + t.Fatal(err) + } + + err = waitForPeers(streamer, 1*time.Second) + if err != nil { + t.Fatal("timeout: peer is not created") + } + + peerId := tester.IDs[0] + + chunk := storage.NewChunk(storage.Key(hash0[:]), nil) + + streamer.Retrieve(chunk) + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "RetrieveRequestMsg", + Expects: []p2ptest.Expect{ + p2ptest.Expect{ + Code: 5, + Msg: &RetrieveRequestMsg{ + Key: chunk.Key[:], + }, + Peer: peerId, + }, + }, + }) + + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } +} + +func TestUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { + // TODO: we only need streamer + tester, streamer, _, teardown, err := newStreamerTester(t) + defer teardown() + if err != nil { + t.Fatal(err) + } + + // streamer.RegisterOutgoingStreamer("foo", func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) { + // return &testOutgoingStreamer{ + // t: t, + // }, nil + // }) + + err = waitForPeers(streamer, 1*time.Second) + if err != nil { + t.Fatal("timeout: peer is not created") + } + + peerId := tester.IDs[0] + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "SubscribeMsg", + Expects: []p2ptest.Expect{ + p2ptest.Expect{ + Code: 4, + Msg: &SubscribeMsg{ + Stream: retrieveRequestStream, + Key: nil, + From: 0, + To: 0, + Priority: Top, + }, + Peer: peerId, + }, + }, + }) + + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + chunk := storage.NewChunk(storage.Key(hash0[:]), nil) + + peer := streamer.getPeer(peerId) + + peer.handleSubscribeMsg(&SubscribeMsg{ + Stream: retrieveRequestStream, + Key: nil, + From: 0, + To: 0, + Priority: Top, + }) + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "RetrieveRequestMsg", + Triggers: []p2ptest.Trigger{ + p2ptest.Trigger{ + Code: 5, + Msg: &RetrieveRequestMsg{ + Key: chunk.Key[:], + }, + Peer: peerId, + }, + }, + Expects: []p2ptest.Expect{ + p2ptest.Expect{ + Code: 1, + Msg: &OfferedHashesMsg{ + HandoverProof: nil, + Hashes: nil, + From: 0, + To: 0, + }, + Peer: peerId, + }, + }, + }) + + expectedError := "exchange 0: 'RetrieveRequestMsg' timed out" + if err == nil || err.Error() != expectedError { + t.Fatalf("Expected error %v, got %v", expectedError, err) + } +} + +func TestUpstreamRetrieveRequestMsgExchange(t *testing.T) { + // TODO: we only need streamer + tester, streamer, localStore, teardown, err := newStreamerTester(t) + defer teardown() + if err != nil { + t.Fatal(err) + } + + // streamer.RegisterOutgoingStreamer("foo", func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) { + // return &testOutgoingStreamer{ + // t: t, + // }, nil + // }) + + err = waitForPeers(streamer, 1*time.Second) + if err != nil { + t.Fatal("timeout: peer is not created") + } + + peerId := tester.IDs[0] + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "SubscribeMsg", + Expects: []p2ptest.Expect{ + p2ptest.Expect{ + Code: 4, + Msg: &SubscribeMsg{ + Stream: retrieveRequestStream, + Key: nil, + From: 0, + To: 0, + Priority: Top, + }, + Peer: peerId, + }, + }, + }) + + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + chunk := storage.NewChunk(storage.Key(hash0[:]), nil) + + peer := streamer.getPeer(peerId) + + peer.handleSubscribeMsg(&SubscribeMsg{ + Stream: retrieveRequestStream, + Key: nil, + From: 0, + To: 0, + Priority: Top, + }) + + chunk.SData = hash0[:] + localStore.Put(chunk) + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "RetrieveRequestMsg", + Triggers: []p2ptest.Trigger{ + p2ptest.Trigger{ + Code: 5, + Msg: &RetrieveRequestMsg{ + Key: chunk.Key[:], + }, + Peer: peerId, + }, + }, + Expects: []p2ptest.Expect{ + p2ptest.Expect{ + Code: 1, + Msg: &OfferedHashesMsg{ + HandoverProof: nil, + Hashes: chunk.Key[:], + From: 0, + // TODO: why is this 32??? + To: 32, + Key: []byte{}, + Stream: retrieveRequestStream, + }, + Peer: peerId, + }, + }, + }) + if err != nil { t.Fatal(err) } From 20d4a867fd81c9d0ea8cb1a2800ff77511cb85d8 Mon Sep 17 00:00:00 2001 From: zelig Date: Sat, 13 Jan 2018 10:51:46 +0100 Subject: [PATCH 042/210] swarm/network: all unit tests pass for streamer * separate retrieve request tests * do not subscribe to requests automatically * simplify request test * add Stream to Subcribe --- swarm/network/request_test.go | 146 +++++++++++++++++++++++++++ swarm/network/streamer.go | 8 -- swarm/network/streamer_test.go | 179 +-------------------------------- 3 files changed, 148 insertions(+), 185 deletions(-) create mode 100644 swarm/network/request_test.go diff --git a/swarm/network/request_test.go b/swarm/network/request_test.go new file mode 100644 index 0000000000..8b848611e3 --- /dev/null +++ b/swarm/network/request_test.go @@ -0,0 +1,146 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package network + +import ( + "testing" + "time" + + p2ptest "github.com/ethereum/go-ethereum/p2p/testing" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { + // TODO: we only need streamer + tester, streamer, _, teardown, err := newStreamerTester(t) + defer teardown() + if err != nil { + t.Fatal(err) + } + + err = waitForPeers(streamer, 1*time.Second) + if err != nil { + t.Fatal("timeout: peer is not created") + } + + peerId := tester.IDs[0] + + chunk := storage.NewChunk(storage.Key(hash0[:]), nil) + + peer := streamer.getPeer(peerId) + + peer.handleSubscribeMsg(&SubscribeMsg{ + Stream: retrieveRequestStream, + Key: nil, + From: 0, + To: 0, + Priority: Top, + }) + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "RetrieveRequestMsg", + Triggers: []p2ptest.Trigger{ + p2ptest.Trigger{ + Code: 5, + Msg: &RetrieveRequestMsg{ + Key: chunk.Key[:], + }, + Peer: peerId, + }, + }, + Expects: []p2ptest.Expect{ + p2ptest.Expect{ + Code: 1, + Msg: &OfferedHashesMsg{ + HandoverProof: nil, + Hashes: nil, + From: 0, + To: 0, + }, + Peer: peerId, + }, + }, + }) + + expectedError := "exchange 0: 'RetrieveRequestMsg' timed out" + if err == nil || err.Error() != expectedError { + t.Fatalf("Expected error %v, got %v", expectedError, err) + } +} + +func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { + // TODO: we only need streamer + tester, streamer, localStore, teardown, err := newStreamerTester(t) + defer teardown() + if err != nil { + t.Fatal(err) + } + + err = waitForPeers(streamer, 1*time.Second) + if err != nil { + t.Fatal("timeout: peer is not created") + } + + peerId := tester.IDs[0] + + chunk := storage.NewChunk(storage.Key(hash0[:]), nil) + + peer := streamer.getPeer(peerId) + + peer.handleSubscribeMsg(&SubscribeMsg{ + Stream: retrieveRequestStream, + Key: nil, + From: 0, + To: 0, + Priority: Top, + }) + + chunk.SData = hash0[:] + localStore.Put(chunk) + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "RetrieveRequestMsg", + Triggers: []p2ptest.Trigger{ + p2ptest.Trigger{ + Code: 5, + Msg: &RetrieveRequestMsg{ + Key: chunk.Key[:], + }, + Peer: peerId, + }, + }, + Expects: []p2ptest.Expect{ + p2ptest.Expect{ + Code: 1, + Msg: &OfferedHashesMsg{ + HandoverProof: nil, + Hashes: chunk.Key[:], + From: 0, + // TODO: why is this 32??? + To: 32, + Key: []byte{}, + Stream: retrieveRequestStream, + }, + Peer: peerId, + }, + }, + }) + + if err != nil { + t.Fatal(err) + } +} diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go index 365222ab17..24dbc3b379 100644 --- a/swarm/network/streamer.go +++ b/swarm/network/streamer.go @@ -667,18 +667,10 @@ var StreamerSpec = &protocols.Spec{ func (s *Streamer) Run(p *bzzPeer) error { sp := NewStreamerPeer(p, s) // load saved intervals - // autosubscribe to request handler to serve request only for non-light nodes - // sp.handleSubscribeMsg(&SubscribeMsg{ - // Stream: retrieveRequeststring, - // Priority: uint8(Top), - // }) - // subscribe to request handling ; only with non-light nodes s.setPeer(sp) defer s.deletePeer(sp) - - s.Subscribe(sp.ID(), retrieveRequestStream, nil, 0, 0, Top, true) defer close(sp.quit) return sp.Run(sp.HandleMsg) } diff --git a/swarm/network/streamer_test.go b/swarm/network/streamer_test.go index 6a75a163fc..a28a0bf260 100644 --- a/swarm/network/streamer_test.go +++ b/swarm/network/streamer_test.go @@ -24,7 +24,7 @@ import ( "testing" "time" - sha3 "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/protocols" @@ -68,7 +68,6 @@ func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Streamer, *stora to.On(bzzPeer) return streamer.Run(bzzPeer) } - protocolTester := p2ptest.NewProtocolTester(t, NewNodeIDFromAddr(addr), 1, run) return protocolTester, streamer, localStore, teardown, nil } @@ -226,6 +225,7 @@ func TestStreamerUpstreamSubscribeMsgExchange(t *testing.T) { p2ptest.Expect{ Code: 1, Msg: &OfferedHashesMsg{ + Stream: "foo", HandoverProof: nil, Hashes: make([]byte, HashSize), From: 6, @@ -383,181 +383,6 @@ func TestRetrieveRequest(t *testing.T) { } } -func TestUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { - // TODO: we only need streamer - tester, streamer, _, teardown, err := newStreamerTester(t) - defer teardown() - if err != nil { - t.Fatal(err) - } - - // streamer.RegisterOutgoingStreamer("foo", func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) { - // return &testOutgoingStreamer{ - // t: t, - // }, nil - // }) - - err = waitForPeers(streamer, 1*time.Second) - if err != nil { - t.Fatal("timeout: peer is not created") - } - - peerId := tester.IDs[0] - - err = tester.TestExchanges(p2ptest.Exchange{ - Label: "SubscribeMsg", - Expects: []p2ptest.Expect{ - p2ptest.Expect{ - Code: 4, - Msg: &SubscribeMsg{ - Stream: retrieveRequestStream, - Key: nil, - From: 0, - To: 0, - Priority: Top, - }, - Peer: peerId, - }, - }, - }) - - if err != nil { - t.Fatalf("Expected no error, got %v", err) - } - - chunk := storage.NewChunk(storage.Key(hash0[:]), nil) - - peer := streamer.getPeer(peerId) - - peer.handleSubscribeMsg(&SubscribeMsg{ - Stream: retrieveRequestStream, - Key: nil, - From: 0, - To: 0, - Priority: Top, - }) - - err = tester.TestExchanges(p2ptest.Exchange{ - Label: "RetrieveRequestMsg", - Triggers: []p2ptest.Trigger{ - p2ptest.Trigger{ - Code: 5, - Msg: &RetrieveRequestMsg{ - Key: chunk.Key[:], - }, - Peer: peerId, - }, - }, - Expects: []p2ptest.Expect{ - p2ptest.Expect{ - Code: 1, - Msg: &OfferedHashesMsg{ - HandoverProof: nil, - Hashes: nil, - From: 0, - To: 0, - }, - Peer: peerId, - }, - }, - }) - - expectedError := "exchange 0: 'RetrieveRequestMsg' timed out" - if err == nil || err.Error() != expectedError { - t.Fatalf("Expected error %v, got %v", expectedError, err) - } -} - -func TestUpstreamRetrieveRequestMsgExchange(t *testing.T) { - // TODO: we only need streamer - tester, streamer, localStore, teardown, err := newStreamerTester(t) - defer teardown() - if err != nil { - t.Fatal(err) - } - - // streamer.RegisterOutgoingStreamer("foo", func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) { - // return &testOutgoingStreamer{ - // t: t, - // }, nil - // }) - - err = waitForPeers(streamer, 1*time.Second) - if err != nil { - t.Fatal("timeout: peer is not created") - } - - peerId := tester.IDs[0] - - err = tester.TestExchanges(p2ptest.Exchange{ - Label: "SubscribeMsg", - Expects: []p2ptest.Expect{ - p2ptest.Expect{ - Code: 4, - Msg: &SubscribeMsg{ - Stream: retrieveRequestStream, - Key: nil, - From: 0, - To: 0, - Priority: Top, - }, - Peer: peerId, - }, - }, - }) - - if err != nil { - t.Fatalf("Expected no error, got %v", err) - } - - chunk := storage.NewChunk(storage.Key(hash0[:]), nil) - - peer := streamer.getPeer(peerId) - - peer.handleSubscribeMsg(&SubscribeMsg{ - Stream: retrieveRequestStream, - Key: nil, - From: 0, - To: 0, - Priority: Top, - }) - - chunk.SData = hash0[:] - localStore.Put(chunk) - - err = tester.TestExchanges(p2ptest.Exchange{ - Label: "RetrieveRequestMsg", - Triggers: []p2ptest.Trigger{ - p2ptest.Trigger{ - Code: 5, - Msg: &RetrieveRequestMsg{ - Key: chunk.Key[:], - }, - Peer: peerId, - }, - }, - Expects: []p2ptest.Expect{ - p2ptest.Expect{ - Code: 1, - Msg: &OfferedHashesMsg{ - HandoverProof: nil, - Hashes: chunk.Key[:], - From: 0, - // TODO: why is this 32??? - To: 32, - Key: []byte{}, - Stream: retrieveRequestStream, - }, - Peer: peerId, - }, - }, - }) - - if err != nil { - t.Fatal(err) - } -} - func waitForPeers(streamer *Streamer, timeout time.Duration) error { ticker := time.NewTicker(10 * time.Millisecond) timeoutTimer := time.NewTimer(timeout) From b4695523711ae47068e9a547cfd05a60cfcd8052 Mon Sep 17 00:00:00 2001 From: zelig Date: Sat, 13 Jan 2018 17:02:54 +0100 Subject: [PATCH 043/210] storage/network: refactor retrieve requests --- swarm/network/request_test.go | 106 ++++++++++++++++------ swarm/network/requests.go | 160 ++++++++++++++++++++++++++------- swarm/network/streamer.go | 134 +++------------------------ swarm/network/streamer_test.go | 84 ++++------------- 4 files changed, 238 insertions(+), 246 deletions(-) diff --git a/swarm/network/request_test.go b/swarm/network/request_test.go index 8b848611e3..8e032c6a3b 100644 --- a/swarm/network/request_test.go +++ b/swarm/network/request_test.go @@ -18,12 +18,42 @@ package network import ( "testing" - "time" p2ptest "github.com/ethereum/go-ethereum/p2p/testing" "github.com/ethereum/go-ethereum/swarm/storage" ) +func TestStreamerRetrieveRequest(t *testing.T) { + // TODO: we only need streamer + tester, streamer, _, teardown, err := newStreamerTester(t) + defer teardown() + if err != nil { + t.Fatal(err) + } + + peerID := tester.IDs[0] + + streamer.delivery.RequestFromPeers(hash0[:], true) + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "RetrieveRequestMsg", + Expects: []p2ptest.Expect{ + p2ptest.Expect{ + Code: 5, + Msg: &RetrieveRequestMsg{ + Key: hash0[:], + SkipCheck: true, + }, + Peer: peerID, + }, + }, + }) + + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } +} + func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { // TODO: we only need streamer tester, streamer, _, teardown, err := newStreamerTester(t) @@ -32,16 +62,11 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { t.Fatal(err) } - err = waitForPeers(streamer, 1*time.Second) - if err != nil { - t.Fatal("timeout: peer is not created") - } - - peerId := tester.IDs[0] + peerID := tester.IDs[0] chunk := storage.NewChunk(storage.Key(hash0[:]), nil) - peer := streamer.getPeer(peerId) + peer := streamer.getPeer(peerID) peer.handleSubscribeMsg(&SubscribeMsg{ Stream: retrieveRequestStream, @@ -59,7 +84,7 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { Msg: &RetrieveRequestMsg{ Key: chunk.Key[:], }, - Peer: peerId, + Peer: peerID, }, }, Expects: []p2ptest.Expect{ @@ -71,7 +96,7 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { From: 0, To: 0, }, - Peer: peerId, + Peer: peerID, }, }, }) @@ -82,6 +107,8 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { } } +// upstream request server receives a retrieve Request and responds with +// offered hashes or delivery if skipHash is set to true func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { // TODO: we only need streamer tester, streamer, localStore, teardown, err := newStreamerTester(t) @@ -90,16 +117,8 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { t.Fatal(err) } - err = waitForPeers(streamer, 1*time.Second) - if err != nil { - t.Fatal("timeout: peer is not created") - } - - peerId := tester.IDs[0] - - chunk := storage.NewChunk(storage.Key(hash0[:]), nil) - - peer := streamer.getPeer(peerId) + peerID := tester.IDs[0] + peer := streamer.getPeer(peerID) peer.handleSubscribeMsg(&SubscribeMsg{ Stream: retrieveRequestStream, @@ -109,8 +128,11 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { Priority: Top, }) - chunk.SData = hash0[:] + hash := storage.Key(hash0[:]) + chunk := storage.NewChunk(hash, nil) + chunk.SData = hash localStore.Put(chunk) + chunk.WaitToStore() err = tester.TestExchanges(p2ptest.Exchange{ Label: "RetrieveRequestMsg", @@ -118,9 +140,9 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { p2ptest.Trigger{ Code: 5, Msg: &RetrieveRequestMsg{ - Key: chunk.Key[:], + Key: hash, }, - Peer: peerId, + Peer: peerID, }, }, Expects: []p2ptest.Expect{ @@ -128,14 +150,48 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { Code: 1, Msg: &OfferedHashesMsg{ HandoverProof: nil, - Hashes: chunk.Key[:], + Hashes: hash, From: 0, // TODO: why is this 32??? To: 32, Key: []byte{}, Stream: retrieveRequestStream, }, - Peer: peerId, + Peer: peerID, + }, + }, + }) + + if err != nil { + t.Fatal(err) + } + + hash = storage.Key(hash1[:]) + chunk = storage.NewChunk(hash, nil) + chunk.SData = hash1[:] + localStore.Put(chunk) + chunk.WaitToStore() + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "RetrieveRequestMsg", + Triggers: []p2ptest.Trigger{ + p2ptest.Trigger{ + Code: 5, + Msg: &RetrieveRequestMsg{ + Key: hash, + SkipCheck: true, + }, + Peer: peerID, + }, + }, + Expects: []p2ptest.Expect{ + p2ptest.Expect{ + Code: 6, + Msg: &ChunkDeliveryMsg{ + Key: hash, + SData: hash, + }, + Peer: peerID, }, }, }) diff --git a/swarm/network/requests.go b/swarm/network/requests.go index ce68afd35d..e269e89d49 100644 --- a/swarm/network/requests.go +++ b/swarm/network/requests.go @@ -16,54 +16,48 @@ package network -import "github.com/ethereum/go-ethereum/swarm/storage" +import ( + "errors" + "fmt" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/swarm/storage" +) const retrieveRequestStream = "RETRIEVE_REQUEST" -// Intervals is a stream specific history of downloaded intervals -// for historical streams -type Intervals struct { - streamer *Streamer - key string +type Delivery struct { + dbAccess *DbAccess + overlay Overlay + receiveC chan *ChunkDeliveryMsg + getPeer func(discover.NodeID) *StreamerPeer + quit chan struct{} } -// func (s *Intervals) load() error { -// return s.streamer.load(s.key) -// } -// -// func (s *Intervals) save() error { -// return s.streamer.save(s.key) -// } -// -// func (s *Intervals) get() []uint64 { -// return s.streamer.get(s.key) -// } -// -// func (s *Intervals) set(v []uint64) { -// s.streamer.set(s.key, v) -// } -// -// func NewIntervals(key string, s *Streamer) *Intervals { -// return &Intervals{ -// streamer: s, -// key: key, -// } -// } +func NewDelivery(overlay Overlay, dbAccess *DbAccess) *Delivery { + return &Delivery{ + dbAccess: dbAccess, + overlay: overlay, + receiveC: make(chan *ChunkDeliveryMsg, 10), + } +} // RetrieveRequestStreamer implements OutgoingStreamer type RetrieveRequestStreamer struct { deliveryC chan *storage.Chunk batchC chan []byte - db *DbAccess + dbAccess *DbAccess currentLen uint64 } // NewRetrieveRequestStreamer is RetrieveRequestStreamer constructor -func NewRetrieveRequestStreamer(db *DbAccess) *RetrieveRequestStreamer { +func NewRetrieveRequestStreamer(dbAccess *DbAccess) *RetrieveRequestStreamer { s := &RetrieveRequestStreamer{ deliveryC: make(chan *storage.Chunk), batchC: make(chan []byte), - db: db, + dbAccess: dbAccess, } go s.processDeliveries() return s @@ -95,6 +89,108 @@ func (s *RetrieveRequestStreamer) SetNextBatch(_, _ uint64) (hashes []byte, from // GetData retrives chunk data from db store func (s *RetrieveRequestStreamer) GetData(key []byte) []byte { - chunk, _ := s.db.get(storage.Key(key)) + chunk, _ := s.dbAccess.get(storage.Key(key)) return chunk.SData } + +// RetrieveRequestMsg is the protocol msg for chunk retrieve requests +type RetrieveRequestMsg struct { + Key storage.Key + SkipCheck bool +} + +func (self *Delivery) handleRetrieveRequestMsg(sp *StreamerPeer, req *RetrieveRequestMsg) error { + s, err := sp.getOutgoingStreamer(retrieveRequestStream) + if err != nil { + return err + } + streamer := s.OutgoingStreamer.(*RetrieveRequestStreamer) + chunk, created := self.dbAccess.getOrCreateRequest(req.Key) + if chunk.ReqC != nil { + if created { + if err := self.RequestFromPeers(chunk.Key[:], false, sp.ID()); err != nil { + return nil + } + } + go func() { + t := time.NewTimer(3 * time.Minute) + defer t.Stop() + + select { + case <-chunk.ReqC: + case <-self.quit: + return + case <-t.C: + return + } + + if req.SkipCheck { + sp.Deliver(chunk, s.priority) + return + } + streamer.deliveryC <- chunk + }() + return nil + } + // TODO: call the retrieve function of the outgoing syncer + if req.SkipCheck { + sp.Deliver(chunk, s.priority) + return nil + } + streamer.deliveryC <- chunk + return nil +} + +type ChunkDeliveryMsg struct { + Key storage.Key + SData []byte // the stored chunk Data (incl size) +} + +func (self *Delivery) handleChunkDeliveryMsg(req *ChunkDeliveryMsg) error { + chunk, err := self.dbAccess.get(req.Key) + if err != nil { + return err + } + + self.receiveC <- req + + log.Trace(fmt.Sprintf("delivery of %v from %v", chunk, self)) + return nil +} + +func (self *Delivery) processReceivedChunks() { + for req := range self.receiveC { + chunk, err := self.dbAccess.get(req.Key) + if err != nil { + continue + } + chunk.SData = req.SData + self.dbAccess.put(chunk) + close(chunk.ReqC) + } +} + +// RequestFromPeers sends a chunk retrieve request to +func (self *Delivery) RequestFromPeers(hash []byte, skipCheck bool, peersToSkip ...discover.NodeID) error { + var success bool + self.overlay.EachConn(hash, 255, func(p OverlayConn, po int, nn bool) bool { + spId := p.(Peer).ID() + for _, p := range peersToSkip { + if p == spId { + return true + } + } + sp := self.getPeer(spId) + // TODO: skip light nodes that do not accept retrieve requests + sp.SendPriority(&RetrieveRequestMsg{ + Key: hash, + SkipCheck: skipCheck, + }, Top) + success = true + return false + }) + if success { + return nil + } + return errors.New("no peer found") +} diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go index 24dbc3b379..2d50a31390 100644 --- a/swarm/network/streamer.go +++ b/swarm/network/streamer.go @@ -21,9 +21,7 @@ import ( "errors" "fmt" "sync" - "time" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/protocols" bv "github.com/ethereum/go-ethereum/swarm/network/bitvector" @@ -91,18 +89,6 @@ type OfferedHashesMsg struct { *HandoverProof `rlp:"nil"` // HandoverProof } -/* - store requests are put in netstore so they are stored and then - forwarded to the peers in their kademlia proximity bin by the syncer -*/ -type ChunkDeliveryMsg struct { - Key storage.Key - SData []byte // the stored chunk Data (incl size) - // optional - Id uint64 // request ID. if delivery, the ID is retrieve request ID - from Peer // [not serialised] protocol registers the requester -} - // String pretty prints OfferedHashesMsg func (self OfferedHashesMsg) String() string { return fmt.Sprintf("Stream '%v' [%v-%v] (%v)", self.Stream, self.From, self.To, len(self.Hashes)/HashSize) @@ -129,28 +115,24 @@ type Streamer struct { peersLock sync.RWMutex outgoing map[string]func(*StreamerPeer, []byte) (OutgoingStreamer, error) incoming map[string]func(*StreamerPeer, []byte) (IncomingStreamer, error) - - dbAccess *DbAccess - overlay Overlay - receiveC chan *ChunkDeliveryMsg - peers map[discover.NodeID]*StreamerPeer + peers map[discover.NodeID]*StreamerPeer + delivery *Delivery } // NewStreamer is Streamer constructor -func NewStreamer(overlay Overlay, dbAccess *DbAccess) *Streamer { +func NewStreamer(delivery *Delivery) *Streamer { streamer := &Streamer{ outgoing: make(map[string]func(*StreamerPeer, []byte) (OutgoingStreamer, error)), incoming: make(map[string]func(*StreamerPeer, []byte) (IncomingStreamer, error)), - dbAccess: dbAccess, - overlay: overlay, - receiveC: make(chan *ChunkDeliveryMsg, 10), peers: make(map[discover.NodeID]*StreamerPeer), + delivery: delivery, } + delivery.getPeer = streamer.getPeer streamer.RegisterOutgoingStreamer(retrieveRequestStream, func(_ *StreamerPeer, t []byte) (OutgoingStreamer, error) { - return NewRetrieveRequestStreamer(dbAccess), nil + return NewRetrieveRequestStreamer(delivery.dbAccess), nil }) streamer.RegisterIncomingStreamer(retrieveRequestStream, func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { - return NewIncomingSwarmSyncer(p, dbAccess, nil) + return NewIncomingSwarmSyncer(p, delivery.dbAccess, nil) }) return streamer } @@ -215,7 +197,6 @@ type OutgoingStreamer interface { type incomingStreamer struct { IncomingStreamer priority uint8 - intervals *Intervals sessionAt uint64 live bool quit chan struct{} @@ -260,82 +241,13 @@ func NewStreamerPeer(p Peer, streamer *Streamer) *StreamerPeer { return self } -// RetrieveRequestMsg is the protocol msg for chunk retrieve requests -type RetrieveRequestMsg struct { - Key storage.Key -} - -func (self *StreamerPeer) handleRetrieveRequestMsg(req *RetrieveRequestMsg) error { - chunk, created := self.streamer.dbAccess.getOrCreateRequest(req.Key) - s, err := self.getOutgoingStreamer(retrieveRequestStream) - if err != nil { - return err - } - streamer := s.OutgoingStreamer.(*RetrieveRequestStreamer) - if chunk.ReqC != nil { - if created { - if err := self.streamer.Retrieve(chunk, self.ID()); err != nil { - return nil - } - } - go func() { - t := time.NewTicker(3 * time.Minute) - defer t.Stop() - - select { - case <-chunk.ReqC: - case <-self.quit: - return - case <-t.C: - return - } - - streamer.deliveryC <- chunk - }() - return nil - } - // TODO: call the retrieve function of the outgoing syncer - streamer.deliveryC <- chunk - return nil -} - -// Retrieve sends a chunk retrieve request to -func (self *Streamer) Retrieve(chunk *storage.Chunk, peersToSkip ...discover.NodeID) error { - var success bool - self.overlay.EachConn(chunk.Key[:], 255, func(p OverlayConn, po int, nn bool) bool { - spId := p.(Peer).ID() - for _, p := range peersToSkip { - if p == spId { - return true - } - } - sp := self.getPeer(spId) - // TODO: skip light nodes that do not accept retrieve requests - sp.SendPriority(&RetrieveRequestMsg{ - Key: chunk.Key[:], - }, Top) - success = true - return false - }) - if success { - return nil - } - return errors.New("no peer found") -} - func (self *Streamer) getPeer(peerId discover.NodeID) *StreamerPeer { - if self.peers == nil { - return nil - } self.peersLock.RLock() defer self.peersLock.RUnlock() return self.peers[peerId] } func (self *Streamer) setPeer(peer *StreamerPeer) { - if self.peers == nil { - self.peers = make(map[discover.NodeID]*StreamerPeer) - } self.peersLock.Lock() self.peers[peer.ID()] = peer self.peersLock.Unlock() @@ -347,33 +259,6 @@ func (self *Streamer) deletePeer(peer *StreamerPeer) { self.peersLock.Unlock() } -func (self *StreamerPeer) handleChunkDeliveryMsg(req *ChunkDeliveryMsg) error { - chunk, err := self.streamer.dbAccess.get(req.Key) - if err != nil { - return err - } - - self.streamer.receiveC <- req - - log.Trace(fmt.Sprintf("delivery of %v from %v", chunk, self)) - return nil -} - -func (self *Streamer) processReceivedChunks() { - for { - select { - case req := <-self.receiveC: - chunk, err := self.dbAccess.get(req.Key) - if err != nil { - continue - } - chunk.SData = req.SData - self.dbAccess.put(chunk) - close(chunk.ReqC) - } - } -} - func (self *StreamerPeer) getOutgoingStreamer(s string) (*outgoingStreamer, error) { self.outgoingLock.RLock() defer self.outgoingLock.RUnlock() @@ -660,6 +545,7 @@ var StreamerSpec = &protocols.Spec{ TakeoverProofMsg{}, SubscribeMsg{}, RetrieveRequestMsg{}, + ChunkDeliveryMsg{}, }, } @@ -692,10 +578,10 @@ func (self *StreamerPeer) HandleMsg(msg interface{}) error { return self.handleWantedHashesMsg(msg) case *ChunkDeliveryMsg: - return self.handleChunkDeliveryMsg(msg) + return self.streamer.delivery.handleChunkDeliveryMsg(msg) case *RetrieveRequestMsg: - return self.handleRetrieveRequestMsg(msg) + return self.streamer.delivery.handleRetrieveRequestMsg(self, msg) default: return fmt.Errorf("unknown message type: %T", msg) diff --git a/swarm/network/streamer_test.go b/swarm/network/streamer_test.go index a28a0bf260..0d58d552c2 100644 --- a/swarm/network/streamer_test.go +++ b/swarm/network/streamer_test.go @@ -57,8 +57,8 @@ func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Streamer, *stora } dbAccess := NewDbAccess(localStore) - streamer := NewStreamer(to, dbAccess) - + delivery := NewDelivery(to, dbAccess) + streamer := NewStreamer(delivery) run := func(p *p2p.Peer, rw p2p.MsgReadWriter) error { bzzPeer := &bzzPeer{ Peer: protocols.NewPeer(p, rw, StreamerSpec), @@ -69,6 +69,12 @@ func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Streamer, *stora return streamer.Run(bzzPeer) } protocolTester := p2ptest.NewProtocolTester(t, NewNodeIDFromAddr(addr), 1, run) + + err = waitForPeers(streamer, 1*time.Second) + if err != nil { + return nil, nil, nil, nil, errors.New("timeout: peer is not created") + } + return protocolTester, streamer, localStore, teardown, nil } @@ -151,14 +157,9 @@ func TestStreamerDownstreamSubscribeMsgExchange(t *testing.T) { }, nil }) - err = waitForPeers(streamer, 1*time.Second) - if err != nil { - t.Fatal("timeout: peer is not created") - } + peerID := tester.IDs[0] - peerId := tester.IDs[0] - - err = streamer.Subscribe(peerId, "foo", []byte{}, 5, 8, Top, true) + err = streamer.Subscribe(peerID, "foo", []byte{}, 5, 8, Top, true) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -175,7 +176,7 @@ func TestStreamerDownstreamSubscribeMsgExchange(t *testing.T) { To: 8, Priority: Top, }, - Peer: peerId, + Peer: peerID, }, }, }) @@ -199,12 +200,7 @@ func TestStreamerUpstreamSubscribeMsgExchange(t *testing.T) { }, nil }) - err = waitForPeers(streamer, 1*time.Second) - if err != nil { - t.Fatal("timeout: peer is not created") - } - - peerId := tester.IDs[0] + peerID := tester.IDs[0] err = tester.TestExchanges(p2ptest.Exchange{ Label: "Subscribe message", @@ -218,7 +214,7 @@ func TestStreamerUpstreamSubscribeMsgExchange(t *testing.T) { To: 8, Priority: Top, }, - Peer: peerId, + Peer: peerID, }, }, Expects: []p2ptest.Expect{ @@ -231,7 +227,7 @@ func TestStreamerUpstreamSubscribeMsgExchange(t *testing.T) { From: 6, To: 9, }, - Peer: peerId, + Peer: peerID, }, }, }) @@ -256,14 +252,9 @@ func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) { }, nil }) - err = waitForPeers(streamer, 1*time.Second) - if err != nil { - t.Fatal("timeout: peer is not created") - } + peerID := tester.IDs[0] - peerId := tester.IDs[0] - - err = streamer.Subscribe(peerId, "foo", []byte{}, 5, 8, Top, true) + err = streamer.Subscribe(peerID, "foo", []byte{}, 5, 8, Top, true) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -280,7 +271,7 @@ func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) { To: 8, Priority: Top, }, - Peer: peerId, + Peer: peerID, }, }, }, @@ -298,7 +289,7 @@ func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) { To: 8, Stream: "foo", }, - Peer: peerId, + Peer: peerID, }, }, Expects: []p2ptest.Expect{ @@ -310,7 +301,7 @@ func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) { From: 8, To: 0, }, - Peer: peerId, + Peer: peerID, }, }, }) @@ -346,43 +337,6 @@ func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) { } -func TestRetrieveRequest(t *testing.T) { - // TODO: we only need streamer - tester, streamer, _, teardown, err := newStreamerTester(t) - defer teardown() - if err != nil { - t.Fatal(err) - } - - err = waitForPeers(streamer, 1*time.Second) - if err != nil { - t.Fatal("timeout: peer is not created") - } - - peerId := tester.IDs[0] - - chunk := storage.NewChunk(storage.Key(hash0[:]), nil) - - streamer.Retrieve(chunk) - - err = tester.TestExchanges(p2ptest.Exchange{ - Label: "RetrieveRequestMsg", - Expects: []p2ptest.Expect{ - p2ptest.Expect{ - Code: 5, - Msg: &RetrieveRequestMsg{ - Key: chunk.Key[:], - }, - Peer: peerId, - }, - }, - }) - - if err != nil { - t.Fatalf("Expected no error, got %v", err) - } -} - func waitForPeers(streamer *Streamer, timeout time.Duration) error { ticker := time.NewTicker(10 * time.Millisecond) timeoutTimer := time.NewTimer(timeout) From 7427eab9c56a01bbbd559ddbb17ce11e5874880b Mon Sep 17 00:00:00 2001 From: zelig Date: Mon, 15 Jan 2018 10:14:04 +0100 Subject: [PATCH 044/210] swarm/network: first attempt retrieval functional test using simulation test --- swarm/network/request_test.go | 381 +++++++++++++++++++++++++++++++++ swarm/network/streamer_test.go | 8 +- swarm/storage/dpa.go | 2 +- swarm/storage/pyramid.go | 3 +- 4 files changed, 388 insertions(+), 6 deletions(-) diff --git a/swarm/network/request_test.go b/swarm/network/request_test.go index 8e032c6a3b..758559cf5f 100644 --- a/swarm/network/request_test.go +++ b/swarm/network/request_test.go @@ -17,9 +17,29 @@ package network import ( + "context" + crand "crypto/rand" + "errors" + "flag" + "fmt" + "io" + "io/ioutil" + "math/rand" + "os" + "sync" + "sync/atomic" "testing" + "time" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/protocols" + "github.com/ethereum/go-ethereum/p2p/simulations" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" p2ptest "github.com/ethereum/go-ethereum/p2p/testing" + "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm/storage" ) @@ -200,3 +220,364 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { t.Fatal(err) } } + +// serviceName is used with the exec adapter so the exec'd binary knows which +// service to execute +const serviceName = "delivery" + +var services = adapters.Services{ + serviceName: newService, +} + +var ( + adapter = flag.String("adapter", "sim", "type of simulation: sim|socket|exec|docker") + loglevel = flag.Int("loglevel", 5, "verbosity of logs") +) + +type roundRobinStore struct { + index uint32 + stores []storage.ChunkStore +} + +func newRoundRobinStore(stores ...storage.ChunkStore) *roundRobinStore { + return &roundRobinStore{ + stores: stores, + } +} + +func (rrs *roundRobinStore) Get(key storage.Key) (*storage.Chunk, error) { + return nil, errors.New("get not well defined on round robin store") +} + +func (rrs *roundRobinStore) Put(chunk *storage.Chunk) { + i := atomic.AddUint32(&rrs.index, 1) + idx := int(i) % len(rrs.stores) + log.Trace(fmt.Sprintf("put %v into localstore %v", chunk.Key, idx)) + rrs.stores[idx].Put(chunk) +} + +func (rrs *roundRobinStore) Close() { + for _, store := range rrs.stores { + store.Close() + } +} + +func init() { + flag.Parse() + // register the Delivery service which will run as a devp2p + // protocol when using the exec adapter + adapters.RegisterServices(services) + + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(false)))) +} + +func testSimulation(t *testing.T, simf func(adapters.NodeAdapter) (*simulations.StepResult, error)) { + var err error + var result *simulations.StepResult + startedAt := time.Now() + + switch *adapter { + case "sim": + t.Logf("simadapter") + result, err = simf(adapters.NewSimAdapter(services)) + case "socket": + result, err = simf(adapters.NewSocketAdapter(services)) + case "exec": + baseDir, err0 := ioutil.TempDir("", "swarm-test") + if err0 != nil { + t.Fatal(err0) + } + defer os.RemoveAll(baseDir) + result, err = simf(adapters.NewExecAdapter(baseDir)) + case "docker": + adapter, err0 := adapters.NewDockerAdapter() + if err0 != nil { + t.Fatal(err0) + } + result, err = simf(adapter) + default: + t.Fatal("adapter needs to be one of sim, socket, exec, docker") + } + if err != nil { + t.Fatal(err) + } + t.Logf("Simulation with %d nodes passed in %s", len(result.Passes), result.FinishedAt.Sub(result.StartedAt)) + var min, max time.Duration + var sum int + for _, pass := range result.Passes { + duration := pass.Sub(result.StartedAt) + if sum == 0 || duration < min { + min = duration + } + if duration > max { + max = duration + } + sum += int(duration.Nanoseconds()) + } + t.Logf("Min: %s, Max: %s, Average: %s", min, max, time.Duration(sum/len(result.Passes))*time.Nanosecond) + finishedAt := time.Now() + t.Logf("Setup: %s, shutdown: %s", result.StartedAt.Sub(startedAt), finishedAt.Sub(result.FinishedAt)) +} + +func TestDeliveryFromNodes(t *testing.T) { + testSimulation(t, testDeliveryFromNodes) +} + +var ( + delivery *Delivery + localStores []storage.ChunkStore + fileHash storage.Key + nodeCount int +) + +func setLocalStores(n int) (func(), error) { + var datadirs []string + localStores = make([]storage.ChunkStore, n) + var err error + for i := 0; i < n; i++ { + // TODO: remove temp datadir after test + var datadir string + datadir, err = ioutil.TempDir("", "streamer") + if err != nil { + break + } + var localStore *storage.LocalStore + localStore, err = storage.NewTestLocalStore(datadir) + if err != nil { + break + } + datadirs = append(datadirs, datadir) + localStores[i] = localStore + } + teardown := func() { + for _, datadir := range datadirs { + os.RemoveAll(datadir) + } + } + return teardown, err +} + +func mustReadAll(dpa *storage.DPA, hash storage.Key) (int, error) { + r := dpa.Retrieve(fileHash) + buf := make([]byte, 1024) + var n, total int + var err error + for (total == 0 || n > 0) && err == nil { + log.Warn(fmt.Sprintf("reading %v bytes at offset %v", len(buf), total)) + n, err = r.ReadAt(buf, int64(total)) + total += n + } + log.Warn(fmt.Sprintf("read %v bytes at offset %v", len(buf), total)) + if err != nil && err != io.EOF { + return total, err + } + return total, nil +} + +func testDeliveryFromNodes(adapter adapters.NodeAdapter) (*simulations.StepResult, error) { + nodes := 2 + conns := 0 + size := 8100 + skipCheck := true + + trigger := func(net *simulations.Network) chan discover.NodeID { + triggerC := make(chan discover.NodeID) + ticker := time.NewTicker(500 * time.Millisecond) + go func() { + defer ticker.Stop() + for range ticker.C { + triggerC <- net.Nodes[0].ID() + } + }() + return triggerC + } + + action := func(net *simulations.Network) func(context.Context) error { + rrdpa := storage.NewDPA(newRoundRobinStore(localStores[1:]...), storage.NewChunkerParams()) + rrdpa.Start() + dpacs := storage.NewDpaChunkStore(localStores[0].(*storage.LocalStore), func(chunk *storage.Chunk) error { return delivery.RequestFromPeers(chunk.Key[:], skipCheck) }) + dpa := storage.NewDPA(dpacs, storage.NewChunkerParams()) + dpa.Start() + return func(context.Context) error { + defer rrdpa.Stop() + hash, wait, err := rrdpa.Store(crand.Reader, int64(size)) + if err != nil { + return err + } + wait() + fileHash = hash + go func() { + defer dpa.Stop() + log.Debug(fmt.Sprintf("retrieve %v", fileHash)) + n, err := mustReadAll(dpa, fileHash) + log.Debug(fmt.Sprintf("retrieved %v", fileHash), "read", n, "err", err) + }() + return nil + } + } + + check := func(net *simulations.Network, dpa *storage.DPA) func(ctx context.Context, id discover.NodeID) (bool, error) { + return func(ctx context.Context, id discover.NodeID) (bool, error) { + select { + case <-ctx.Done(): + return false, ctx.Err() + default: + } + log.Warn(fmt.Sprintf("try to locally retrieve %v", fileHash)) + total, err := mustReadAll(dpa, fileHash) + if err != nil || total != size { + log.Warn(fmt.Sprintf("number of bytes read %v/%v (error: %v)", total, size, err)) + return false, nil + } + return true, nil + // node := net.GetNode(id) + // if node == nil { + // return false, fmt.Errorf("unknown node: %s", id) + // } + // client, err := node.Client() + // if err != nil { + // return false, fmt.Errorf("error getting node client: %s", err) + // } + // var response int + // if err := client.Call(&response, "test_haslocal", hash); err != nil { + // return false, fmt.Errorf("error getting bzz_has response: %s", err) + // } + // log.Debug(fmt.Sprintf("node has: %v\n%v", id, response)) + // return response == 0, nil + } + } + + result, err := runSimulation(nodes, conns, action, trigger, check, adapter) + if err != nil { + return nil, fmt.Errorf("Setting up simulation failed: %v", err) + } + if result.Error != nil { + return nil, fmt.Errorf("Simulation failed: %s", result.Error) + } + return result, err +} + +func runSimulation(nodes, conns int, action func(*simulations.Network) func(context.Context) error, trigger func(*simulations.Network) chan discover.NodeID, check func(*simulations.Network, *storage.DPA) func(context.Context, discover.NodeID) (bool, error), adapter adapters.NodeAdapter) (*simulations.StepResult, error) { + // create network + net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{ + ID: "0", + DefaultService: serviceName, + }) + defer net.Shutdown() + teardown, err := setLocalStores(nodes) + defer teardown() + if err != nil { + return nil, err + } + ids := make([]discover.NodeID, nodes) + nodeCount = 0 + for i := 0; i < nodes; i++ { + node, err := net.NewNode() + if err != nil { + return nil, fmt.Errorf("error starting node: %s", err) + } + if err := net.Start(node.ID()); err != nil { + return nil, fmt.Errorf("error starting node %s: %s", node.ID().TerminalString(), err) + } + ids[i] = node.ID() + } + + // run a simulation which connects the 10 nodes in a ring and waits + // for full peer discovery + var addrs [][]byte + wg := sync.WaitGroup{} + for i := range ids { + // collect the overlay addresses, to + addrs = append(addrs, ToOverlayAddr(ids[i].Bytes())) + for j := 0; j < conns; j++ { + var k int + if j == 0 { + k = i - 1 + } else { + k = rand.Intn(len(ids)) + } + if i > 0 { + wg.Add(1) + go func(i, k int) { + defer wg.Done() + net.Connect(ids[i], ids[k]) + }(i, k) + } + } + } + wg.Wait() + log.Debug(fmt.Sprintf("nodes: %v", len(addrs))) + + // 64 nodes ~ 1min + // 128 nodes ~ + dpa := storage.NewDPA(localStores[0], storage.NewChunkerParams()) + dpa.Start() + timeout := 300 * time.Second + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + result := simulations.NewSimulation(net).Run(ctx, &simulations.Step{ + Action: action(net), + Trigger: trigger(net), + Expect: &simulations.Expectation{ + Nodes: ids, + Check: check(net, dpa), + }, + }) + return result, nil +} + +func newService(ctx *adapters.ServiceContext) (node.Service, error) { + id := ctx.Config.ID + addr := NewAddrFromNodeID(id) + kad := NewKademlia(addr.Over(), NewKadParams()) + localStore := localStores[nodeCount] + dbAccess := NewDbAccess(localStore.(*storage.LocalStore)) + streamer := NewStreamer(NewDelivery(kad, dbAccess)) + if nodeCount == 0 { + delivery = streamer.delivery + } + nodeCount++ + run := func(p *p2p.Peer, rw p2p.MsgReadWriter) error { + bzzPeer := &bzzPeer{ + Peer: protocols.NewPeer(p, rw, StreamerSpec), + localAddr: addr, + BzzAddr: NewAddrFromNodeID(p.ID()), + } + kad.On(bzzPeer) + streamer.Subscribe(p.ID(), retrieveRequestStream, nil, 0, 0, Top, true) + return streamer.Run(bzzPeer) + } + + return &testDeliveryService{ + run: run, + }, nil +} + +type testDeliveryService struct { + run func(p *p2p.Peer, rw p2p.MsgReadWriter) error +} + +func (tds *testDeliveryService) Protocols() []p2p.Protocol { + return []p2p.Protocol{ + { + Name: StreamerSpec.Name, + Version: StreamerSpec.Version, + Length: StreamerSpec.Length(), + Run: tds.run, + // NodeInfo: , + // PeerInfo: , + }, + } +} + +func (b *testDeliveryService) APIs() []rpc.API { + return []rpc.API{} +} + +func (b *testDeliveryService) Start(server *p2p.Server) error { + return nil +} + +func (b *testDeliveryService) Stop() error { + return nil +} diff --git a/swarm/network/streamer_test.go b/swarm/network/streamer_test.go index 0d58d552c2..df1f92e3b5 100644 --- a/swarm/network/streamer_test.go +++ b/swarm/network/streamer_test.go @@ -25,16 +25,16 @@ import ( "time" "github.com/ethereum/go-ethereum/crypto/sha3" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/protocols" p2ptest "github.com/ethereum/go-ethereum/p2p/testing" "github.com/ethereum/go-ethereum/swarm/storage" ) -func init() { - log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlWarn, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) -} +// +// func init() { +// log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlWarn, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) +// } // TODO: extract newStreamer func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Streamer, *storage.LocalStore, func(), error) { diff --git a/swarm/storage/dpa.go b/swarm/storage/dpa.go index 5f0a95470e..508a712013 100644 --- a/swarm/storage/dpa.go +++ b/swarm/storage/dpa.go @@ -80,7 +80,7 @@ func NewLocalDPA(datadir string, basekey []byte) (*DPA, error) { } func NewDPA(store ChunkStore, params *ChunkerParams) *DPA { - chunker := NewTreeChunker(params) + chunker := NewPyramidChunker(params) return &DPA{ Chunker: chunker, ChunkStore: store, diff --git a/swarm/storage/pyramid.go b/swarm/storage/pyramid.go index 28736cf319..5e160a7fed 100644 --- a/swarm/storage/pyramid.go +++ b/swarm/storage/pyramid.go @@ -136,10 +136,11 @@ func NewPyramidChunker(params *ChunkerParams) (self *PyramidChunker) { return } -func (self *PyramidChunker) Join(key Key, chunkC chan *Chunk) LazySectionReader { +func (self *PyramidChunker) Join(key Key, chunkC chan *Chunk, depth int) LazySectionReader { return &LazyChunkReader{ key: key, chunkC: chunkC, + depth: depth, chunkSize: self.chunkSize, branches: self.branches, hashSize: self.hashSize, From 18a7d313386194be39e733ac3043988690f42464 Mon Sep 17 00:00:00 2001 From: Jim McDonald Date: Mon, 15 Jan 2018 10:57:06 +0000 Subject: [PATCH 045/210] miner: avoid unnecessary work (#15883) --- core/gaspool.go | 5 +++++ miner/worker.go | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/core/gaspool.go b/core/gaspool.go index c3ee5c198f..e3795c1ee9 100644 --- a/core/gaspool.go +++ b/core/gaspool.go @@ -44,6 +44,11 @@ func (gp *GasPool) SubGas(amount uint64) error { return nil } +// Gas returns the amount of gas remaining in the pool. +func (gp *GasPool) Gas() uint64 { + return uint64(*gp) +} + func (gp *GasPool) String() string { return fmt.Sprintf("%d", *gp) } diff --git a/miner/worker.go b/miner/worker.go index 638f759bf5..1520277e17 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -512,6 +512,11 @@ func (env *Work) commitTransactions(mux *event.TypeMux, txs *types.TransactionsB var coalescedLogs []*types.Log for { + // If we don't have enough gas for any further transactions then we're done + if gp.Gas() < params.TxGas { + log.Trace("Not enough gas for further transactions", "gp", gp) + break + } // Retrieve the next transaction and abort if all done tx := txs.Peek() if tx == nil { From 216e584899ed522088419438c9c605a20b5dc9ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 15 Jan 2018 15:32:14 +0200 Subject: [PATCH 046/210] Revert "trie: make fullnode children hash calculation concurrently (#15131)" (#15889) This reverts commit 0f7fbb85d6e939510a3e3bb6493a9a332ddfd8e8. --- trie/hasher.go | 111 +++++++++++--------------------------------- trie/secure_trie.go | 8 ++-- trie/trie.go | 1 + 3 files changed, 32 insertions(+), 88 deletions(-) diff --git a/trie/hasher.go b/trie/hasher.go index 5186d76698..4719aabf62 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -26,46 +26,27 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) -// calculator is a utility used by the hasher to calculate the hash value of the tree node. -type calculator struct { - sha hash.Hash - buffer *bytes.Buffer +type hasher struct { + tmp *bytes.Buffer + sha hash.Hash + cachegen, cachelimit uint16 } -// calculatorPool is a set of temporary calculators that may be individually saved and retrieved. -var calculatorPool = sync.Pool{ +// hashers live in a global pool. +var hasherPool = sync.Pool{ New: func() interface{} { - return &calculator{buffer: new(bytes.Buffer), sha: sha3.NewKeccak256()} + return &hasher{tmp: new(bytes.Buffer), sha: sha3.NewKeccak256()} }, } -// hasher hasher is used to calculate the hash value of the whole tree. -type hasher struct { - cachegen uint16 - cachelimit uint16 - threaded bool - mu sync.Mutex -} - func newHasher(cachegen, cachelimit uint16) *hasher { - h := &hasher{ - cachegen: cachegen, - cachelimit: cachelimit, - } + h := hasherPool.Get().(*hasher) + h.cachegen, h.cachelimit = cachegen, cachelimit return h } -// newCalculator retrieves a cleaned calculator from calculator pool. -func (h *hasher) newCalculator() *calculator { - calculator := calculatorPool.Get().(*calculator) - calculator.buffer.Reset() - calculator.sha.Reset() - return calculator -} - -// returnCalculator returns a no longer used calculator to the pool. -func (h *hasher) returnCalculator(calculator *calculator) { - calculatorPool.Put(calculator) +func returnHasherToPool(h *hasher) { + hasherPool.Put(h) } // hash collapses a node down into a hash node, also returning a copy of the @@ -142,48 +123,15 @@ func (h *hasher) hashChildren(original node, db DatabaseWriter) (node, node, err // Hash the full node's children, caching the newly hashed subtrees collapsed, cached := n.copy(), n.copy() - // hashChild is a helper to hash a single child, which is called either on the - // same thread as the caller or in a goroutine for the toplevel branching. - hashChild := func(index int, wg *sync.WaitGroup) { - if wg != nil { - defer wg.Done() + for i := 0; i < 16; i++ { + if n.Children[i] != nil { + collapsed.Children[i], cached.Children[i], err = h.hash(n.Children[i], db, false) + if err != nil { + return original, original, err + } + } else { + collapsed.Children[i] = valueNode(nil) // Ensure that nil children are encoded as empty strings. } - // Ensure that nil children are encoded as empty strings. - if collapsed.Children[index] == nil { - collapsed.Children[index] = valueNode(nil) - return - } - // Hash all other children properly - var herr error - collapsed.Children[index], cached.Children[index], herr = h.hash(n.Children[index], db, false) - if herr != nil { - h.mu.Lock() // rarely if ever locked, no congenstion - err = herr - h.mu.Unlock() - } - } - // If we're not running in threaded mode yet, span a goroutine for each child - if !h.threaded { - // Disable further threading - h.threaded = true - - // Hash all the children concurrently - var wg sync.WaitGroup - for i := 0; i < 16; i++ { - wg.Add(1) - go hashChild(i, &wg) - } - wg.Wait() - - // Reenable threading for subsequent hash calls - h.threaded = false - } else { - for i := 0; i < 16; i++ { - hashChild(i, nil) - } - } - if err != nil { - return original, original, err } cached.Children[16] = n.Children[16] if collapsed.Children[16] == nil { @@ -202,29 +150,24 @@ func (h *hasher) store(n node, db DatabaseWriter, force bool) (node, error) { if _, isHash := n.(hashNode); n == nil || isHash { return n, nil } - calculator := h.newCalculator() - defer h.returnCalculator(calculator) - // Generate the RLP encoding of the node - if err := rlp.Encode(calculator.buffer, n); err != nil { + h.tmp.Reset() + if err := rlp.Encode(h.tmp, n); err != nil { panic("encode error: " + err.Error()) } - if calculator.buffer.Len() < 32 && !force { + + if h.tmp.Len() < 32 && !force { return n, nil // Nodes smaller than 32 bytes are stored inside their parent } // Larger nodes are replaced by their hash and stored in the database. hash, _ := n.cache() if hash == nil { - calculator.sha.Write(calculator.buffer.Bytes()) - hash = hashNode(calculator.sha.Sum(nil)) + h.sha.Reset() + h.sha.Write(h.tmp.Bytes()) + hash = hashNode(h.sha.Sum(nil)) } if db != nil { - // db might be a leveldb batch, which is not safe for concurrent writes - h.mu.Lock() - err := db.Put(hash, calculator.buffer.Bytes()) - h.mu.Unlock() - - return hash, err + return hash, db.Put(hash, h.tmp.Bytes()) } return hash, nil } diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 1fde45165e..20c303f31c 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -199,10 +199,10 @@ func (t *SecureTrie) secKey(key []byte) []byte { // invalid on the next call to hashKey or secKey. func (t *SecureTrie) hashKey(key []byte) []byte { h := newHasher(0, 0) - calculator := h.newCalculator() - calculator.sha.Write(key) - buf := calculator.sha.Sum(t.hashKeyBuf[:0]) - h.returnCalculator(calculator) + h.sha.Reset() + h.sha.Write(key) + buf := h.sha.Sum(t.hashKeyBuf[:0]) + returnHasherToPool(h) return buf } diff --git a/trie/trie.go b/trie/trie.go index 7c1b5e1b61..8fe98d8351 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -501,5 +501,6 @@ func (t *Trie) hashRoot(db DatabaseWriter) (node, node, error) { return hashNode(emptyRoot.Bytes()), nil, nil } h := newHasher(t.cachegen, t.cachelimit) + defer returnHasherToPool(h) return h.hash(t.root, db, true) } From 05ab6ee5234d9c5881e903eb001e29d7bb7b7f35 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Mon, 15 Jan 2018 18:22:24 +0100 Subject: [PATCH 047/210] swarm/storage: Fix prepareChunks in pyramid chunker We need read in a loop until we receive an EOF or the buffer is full, because not all Reader guarantees that it reads the buffer full in one step --- swarm/storage/pyramid.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/swarm/storage/pyramid.go b/swarm/storage/pyramid.go index 5e160a7fed..a34b73993f 100644 --- a/swarm/storage/pyramid.go +++ b/swarm/storage/pyramid.go @@ -414,14 +414,24 @@ func (self *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEnt var n int var err error chunkData := make([]byte, self.chunkSize+8) + maxBuf := len(chunkData) + readBytes := 8 if unFinishedChunk != nil { copy(chunkData, unFinishedChunk.SData) - n, err = data.Read(chunkData[8+unFinishedChunk.Size:]) - n += int(unFinishedChunk.Size) - unFinishedChunk = nil - } else { - n, err = data.Read(chunkData[8:]) + readBytes += int(unFinishedChunk.Size) } + for readBytes < maxBuf { + n0, err0 := data.Read(chunkData[readBytes:]) + readBytes += n0 + n += n0 + if err0 != nil { + if err0 != io.EOF || (n0 == 0 && maxBuf == readBytes) || n == 0 || n0 != 0 { + err = err0 + } + break + } + } + unFinishedChunk = nil totalDataSize += n if err != nil { From 61f5aa4e01c5fd3f901ad5d0ee69667846bfbcde Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Mon, 15 Jan 2018 18:24:56 +0100 Subject: [PATCH 048/210] Fixed return value of LazyChunkReader.readAt It returned the full buffer size even when it was only partially filled --- swarm/storage/chunker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index 918c0fc45b..2ea81403bf 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -379,7 +379,7 @@ func (self *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) { return 0, err } if off+int64(len(b)) >= size { - return len(b), io.EOF + return int(size - int64(off)), io.EOF } return len(b), nil } From 91b6a66f034b652c06a666564c709788f0a78581 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Mon, 15 Jan 2018 18:25:59 +0100 Subject: [PATCH 049/210] swarm/storage: Store chunk size --- swarm/storage/dpa.go | 2 ++ swarm/storage/localstore.go | 1 + 2 files changed, 3 insertions(+) diff --git a/swarm/storage/dpa.go b/swarm/storage/dpa.go index 508a712013..b54c63804c 100644 --- a/swarm/storage/dpa.go +++ b/swarm/storage/dpa.go @@ -17,6 +17,7 @@ package storage import ( + "encoding/binary" "errors" "fmt" "io" @@ -211,6 +212,7 @@ func (self *dpaChunkStore) Get(key Key) (chunk *Chunk, err error) { return nil, notFound case <-chunk.ReqC: } + chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) return chunk, nil } diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index 4fcddbf735..32c495ac8a 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -59,6 +59,7 @@ func NewTestLocalStore(path string) (*LocalStore, error) { // LocalStore is itself a chunk store // unsafe, in that the data is not integrity checked func (self *LocalStore) Put(chunk *Chunk) { + chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) self.memStore.Put(chunk) go func() { self.DbStore.Put(chunk) From 40cfae13ef1491ce9af11d13fa24c27cca77218f Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Mon, 15 Jan 2018 18:32:34 +0100 Subject: [PATCH 050/210] swarm/storage: Temporarily disable chunker test TestDataAppend --- swarm/storage/chunker_test.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/swarm/storage/chunker_test.go b/swarm/storage/chunker_test.go index a0f82245d3..fb66b7c756 100644 --- a/swarm/storage/chunker_test.go +++ b/swarm/storage/chunker_test.go @@ -321,17 +321,19 @@ func TestSha3ForCorrectness(t *testing.T) { } -func TestDataAppend(t *testing.T) { - sizes := []int{1, 1, 1, 4095, 4096, 4097, 1, 1, 1, 123456, 2345678, 2345678} - appendSizes := []int{4095, 4096, 4097, 1, 1, 1, 8191, 8192, 8193, 9000, 3000, 5000} - - tester := &chunkerTester{t: t} - chunker := NewPyramidChunker(NewChunkerParams()) - for i, s := range sizes { - testRandomDataAppend(chunker, s, appendSizes[i], tester) - - } -} +// func TestDataAppend(t *testing.T) { +// // sizes := []int{1, 1, 1, 4095, 4096, 4097, 1, 1, 1, 123456, 2345678, 2345678} +// sizes := []int{1} +// // appendSizes := []int{4095, 4096, 4097, 1, 1, 1, 8191, 8192, 8193, 9000, 3000, 5000} +// appendSizes := []int{4095} +// +// tester := &chunkerTester{t: t} +// chunker := NewPyramidChunker(NewChunkerParams()) +// for i, s := range sizes { +// testRandomDataAppend(chunker, s, appendSizes[i], tester) +// +// } +// } func TestRandomData(t *testing.T) { sizes := []int{1, 60, 83, 179, 253, 1024, 4095, 4096, 4097, 8191, 8192, 8193, 12287, 12288, 12289, 123456, 2345678} From d075df71590ce3f39bc20e2982c93e855a9c4a5b Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Mon, 15 Jan 2018 18:33:24 +0100 Subject: [PATCH 051/210] swarm/network: HandoverProof should not be nil --- swarm/network/streamer.go | 10 +++++----- swarm/network/streamer_test.go | 17 +++++++++++------ 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go index 2d50a31390..61351bf392 100644 --- a/swarm/network/streamer.go +++ b/swarm/network/streamer.go @@ -82,11 +82,11 @@ type SubscribeMsg struct { // OfferedHashesMsg is the protocol msg for offering to hand over a // stream section type OfferedHashesMsg struct { - Stream string // name of Stream - Key []byte // subtype or key - From, To uint64 // peer and db-specific entry count - Hashes []byte // stream of hashes (128) - *HandoverProof `rlp:"nil"` // HandoverProof + Stream string // name of Stream + Key []byte // subtype or key + From, To uint64 // peer and db-specific entry count + Hashes []byte // stream of hashes (128) + *HandoverProof // HandoverProof } // String pretty prints OfferedHashesMsg diff --git a/swarm/network/streamer_test.go b/swarm/network/streamer_test.go index df1f92e3b5..5f2c5369ff 100644 --- a/swarm/network/streamer_test.go +++ b/swarm/network/streamer_test.go @@ -136,7 +136,10 @@ func (self *testIncomingStreamer) BatchDone(string, uint64, []byte, []byte) func } func (self *testOutgoingStreamer) SetNextBatch(from uint64, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) { - return make([]byte, HashSize), from + 1, to + 1, nil, nil + proof := &HandoverProof{ + Handover: &Handover{}, + } + return make([]byte, HashSize), from + 1, to + 1, proof, nil } func (self *testOutgoingStreamer) GetData([]byte) []byte { @@ -221,11 +224,13 @@ func TestStreamerUpstreamSubscribeMsgExchange(t *testing.T) { p2ptest.Expect{ Code: 1, Msg: &OfferedHashesMsg{ - Stream: "foo", - HandoverProof: nil, - Hashes: make([]byte, HashSize), - From: 6, - To: 9, + Stream: "foo", + HandoverProof: &HandoverProof{ + Handover: &Handover{}, + }, + Hashes: make([]byte, HashSize), + From: 6, + To: 9, }, Peer: peerID, }, From a4fbb7b2d97e5988863d21034245bc6cf6325444 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Mon, 15 Jan 2018 18:33:56 +0100 Subject: [PATCH 052/210] swarm/network: Test fixes in request_test --- swarm/network/request_test.go | 132 ++++++++++++++++++++++++++++++++-- swarm/network/requests.go | 14 ++-- 2 files changed, 136 insertions(+), 10 deletions(-) diff --git a/swarm/network/request_test.go b/swarm/network/request_test.go index 758559cf5f..54e5db14a1 100644 --- a/swarm/network/request_test.go +++ b/swarm/network/request_test.go @@ -17,6 +17,7 @@ package network import ( + "bytes" "context" crand "crypto/rand" "errors" @@ -221,6 +222,105 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { } } +func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { + // TODO: we only need streamer + tester, streamer, localStore, teardown, err := newStreamerTester(t) + defer teardown() + if err != nil { + t.Fatal(err) + } + + streamer.RegisterIncomingStreamer("foo", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { + return &testIncomingStreamer{ + t: t, + }, nil + }) + + peerID := tester.IDs[0] + + err = streamer.Subscribe(peerID, "foo", []byte{}, 5, 8, Top, true) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + chunkKey := hash0[:] + chunkData := hash1[:] + chunk, created := localStore.GetOrCreateRequest(chunkKey) + + if !created { + t.Fatal("chunk already exists") + } + select { + case <-chunk.ReqC: + t.Fatal("chunk is already received") + default: + } + + err = tester.TestExchanges(p2ptest.Exchange{ + Label: "Subscribe message", + Expects: []p2ptest.Expect{ + p2ptest.Expect{ + Code: 4, + Msg: &SubscribeMsg{ + Stream: "foo", + Key: []byte{}, + From: 5, + To: 8, + Priority: Top, + }, + Peer: peerID, + }, + }, + }, + p2ptest.Exchange{ + Label: "ChunkDeliveryRequest message", + Triggers: []p2ptest.Trigger{ + p2ptest.Trigger{ + Code: 6, + Msg: &ChunkDeliveryMsg{ + Key: chunkKey, + SData: chunkData, + }, + Peer: peerID, + }, + }, + // Expects: []p2ptest.Expect{ + // p2ptest.Expect{ + // Code: 2, + // Msg: &WantedHashesMsg{ + // Stream: "foo", + // Want: []byte{5}, + // From: 8, + // To: 0, + // }, + // Peer: peerID, + // }, + // }, + }) + + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + timeout := time.NewTimer(1 * time.Second) + + select { + case <-timeout.C: + t.Fatal("timeout receiving chunk") + case <-chunk.ReqC: + } + + storedChunk, err := localStore.Get(chunkKey) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + if !bytes.Equal(storedChunk.SData, chunkData) { + t.Fatal("Retrieved chunk has different data than original") + } + +} + // serviceName is used with the exec adapter so the exec'd binary knows which // service to execute const serviceName = "delivery" @@ -250,6 +350,7 @@ func (rrs *roundRobinStore) Get(key storage.Key) (*storage.Chunk, error) { } func (rrs *roundRobinStore) Put(chunk *storage.Chunk) { + log.Warn("chunksize", "size", chunk.Size, "sdata", len(chunk.SData)) i := atomic.AddUint32(&rrs.index, 1) idx := int(i) % len(rrs.stores) log.Trace(fmt.Sprintf("put %v into localstore %v", chunk.Key, idx)) @@ -367,7 +468,7 @@ func mustReadAll(dpa *storage.DPA, hash storage.Key) (int, error) { n, err = r.ReadAt(buf, int64(total)) total += n } - log.Warn(fmt.Sprintf("read %v bytes at offset %v", len(buf), total)) + log.Warn(fmt.Sprintf("read %v bytes at offset %v error %v", len(buf), total, err)) if err != nil && err != io.EOF { return total, err } @@ -376,7 +477,7 @@ func mustReadAll(dpa *storage.DPA, hash storage.Key) (int, error) { func testDeliveryFromNodes(adapter adapters.NodeAdapter) (*simulations.StepResult, error) { nodes := 2 - conns := 0 + conns := 1 size := 8100 skipCheck := true @@ -385,6 +486,9 @@ func testDeliveryFromNodes(adapter adapters.NodeAdapter) (*simulations.StepResul ticker := time.NewTicker(500 * time.Millisecond) go func() { defer ticker.Stop() + for i := 1; i < nodes; i++ { + triggerC <- net.Nodes[i].ID() + } for range ticker.C { triggerC <- net.Nodes[0].ID() } @@ -400,7 +504,7 @@ func testDeliveryFromNodes(adapter adapters.NodeAdapter) (*simulations.StepResul dpa.Start() return func(context.Context) error { defer rrdpa.Stop() - hash, wait, err := rrdpa.Store(crand.Reader, int64(size)) + hash, wait, err := rrdpa.Store(io.LimitReader(crand.Reader, int64(size)), int64(size)) if err != nil { return err } @@ -409,6 +513,7 @@ func testDeliveryFromNodes(adapter adapters.NodeAdapter) (*simulations.StepResul go func() { defer dpa.Stop() log.Debug(fmt.Sprintf("retrieve %v", fileHash)) + time.Sleep(2 * time.Second) n, err := mustReadAll(dpa, fileHash) log.Debug(fmt.Sprintf("retrieved %v", fileHash), "read", n, "err", err) }() @@ -418,6 +523,9 @@ func testDeliveryFromNodes(adapter adapters.NodeAdapter) (*simulations.StepResul check := func(net *simulations.Network, dpa *storage.DPA) func(ctx context.Context, id discover.NodeID) (bool, error) { return func(ctx context.Context, id discover.NodeID) (bool, error) { + if id != net.Nodes[0].ID() { + return true, nil + } select { case <-ctx.Done(): return false, ctx.Err() @@ -486,10 +594,13 @@ func runSimulation(nodes, conns int, action func(*simulations.Network) func(cont // for full peer discovery var addrs [][]byte wg := sync.WaitGroup{} + log.Warn("runSimulation 1") for i := range ids { + log.Warn("runSimulation 2") // collect the overlay addresses, to addrs = append(addrs, ToOverlayAddr(ids[i].Bytes())) for j := 0; j < conns; j++ { + log.Warn("runSimulation 3") var k int if j == 0 { k = i - 1 @@ -497,15 +608,18 @@ func runSimulation(nodes, conns int, action func(*simulations.Network) func(cont k = rand.Intn(len(ids)) } if i > 0 { + log.Warn("runSimulation 4") wg.Add(1) go func(i, k int) { defer wg.Done() + log.Warn("net.Connect") net.Connect(ids[i], ids[k]) }(i, k) } } } wg.Wait() + log.Debug(fmt.Sprintf("nodes: %v", len(addrs))) // 64 nodes ~ 1min @@ -543,11 +657,18 @@ func newService(ctx *adapters.ServiceContext) (node.Service, error) { localAddr: addr, BzzAddr: NewAddrFromNodeID(p.ID()), } + log.Warn("Run function kad On ", "local", id, "remote", p.ID()) kad.On(bzzPeer) - streamer.Subscribe(p.ID(), retrieveRequestStream, nil, 0, 0, Top, true) + go func() { + time.Sleep(1 * time.Second) + err := streamer.Subscribe(p.ID(), retrieveRequestStream, nil, 0, 0, Top, true) + if err != nil { + log.Warn("error in subscribe", "err", err) + } + }() return streamer.Run(bzzPeer) } - + log.Warn("new service created") return &testDeliveryService{ run: run, }, nil @@ -558,6 +679,7 @@ type testDeliveryService struct { } func (tds *testDeliveryService) Protocols() []p2p.Protocol { + log.Warn("Protocols function", "run", tds.run) return []p2p.Protocol{ { Name: StreamerSpec.Name, diff --git a/swarm/network/requests.go b/swarm/network/requests.go index e269e89d49..d2cf9fa344 100644 --- a/swarm/network/requests.go +++ b/swarm/network/requests.go @@ -37,11 +37,14 @@ type Delivery struct { } func NewDelivery(overlay Overlay, dbAccess *DbAccess) *Delivery { - return &Delivery{ + self := &Delivery{ dbAccess: dbAccess, overlay: overlay, receiveC: make(chan *ChunkDeliveryMsg, 10), } + + go self.processReceivedChunks() + return self } // RetrieveRequestStreamer implements OutgoingStreamer @@ -134,8 +137,7 @@ func (self *Delivery) handleRetrieveRequestMsg(sp *StreamerPeer, req *RetrieveRe } // TODO: call the retrieve function of the outgoing syncer if req.SkipCheck { - sp.Deliver(chunk, s.priority) - return nil + return sp.Deliver(chunk, s.priority) } streamer.deliveryC <- chunk return nil @@ -182,11 +184,13 @@ func (self *Delivery) RequestFromPeers(hash []byte, skipCheck bool, peersToSkip } sp := self.getPeer(spId) // TODO: skip light nodes that do not accept retrieve requests - sp.SendPriority(&RetrieveRequestMsg{ + err := sp.SendPriority(&RetrieveRequestMsg{ Key: hash, SkipCheck: skipCheck, }, Top) - success = true + if err == nil { + success = true + } return false }) if success { From cc939a9ca5fccf2c9e56b4bd1894e3cebfcb60e1 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Tue, 16 Jan 2018 12:21:22 +0100 Subject: [PATCH 053/210] swarm/network: Remove unnecessary comments --- swarm/network/request_test.go | 16 ---------------- swarm/network/streamer_test.go | 9 --------- 2 files changed, 25 deletions(-) diff --git a/swarm/network/request_test.go b/swarm/network/request_test.go index 54e5db14a1..7aa892dc13 100644 --- a/swarm/network/request_test.go +++ b/swarm/network/request_test.go @@ -45,7 +45,6 @@ import ( ) func TestStreamerRetrieveRequest(t *testing.T) { - // TODO: we only need streamer tester, streamer, _, teardown, err := newStreamerTester(t) defer teardown() if err != nil { @@ -76,7 +75,6 @@ func TestStreamerRetrieveRequest(t *testing.T) { } func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { - // TODO: we only need streamer tester, streamer, _, teardown, err := newStreamerTester(t) defer teardown() if err != nil { @@ -131,7 +129,6 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { // upstream request server receives a retrieve Request and responds with // offered hashes or delivery if skipHash is set to true func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { - // TODO: we only need streamer tester, streamer, localStore, teardown, err := newStreamerTester(t) defer teardown() if err != nil { @@ -223,7 +220,6 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { } func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { - // TODO: we only need streamer tester, streamer, localStore, teardown, err := newStreamerTester(t) defer teardown() if err != nil { @@ -284,18 +280,6 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { Peer: peerID, }, }, - // Expects: []p2ptest.Expect{ - // p2ptest.Expect{ - // Code: 2, - // Msg: &WantedHashesMsg{ - // Stream: "foo", - // Want: []byte{5}, - // From: 8, - // To: 0, - // }, - // Peer: peerID, - // }, - // }, }) if err != nil { diff --git a/swarm/network/streamer_test.go b/swarm/network/streamer_test.go index 5f2c5369ff..9d44934385 100644 --- a/swarm/network/streamer_test.go +++ b/swarm/network/streamer_test.go @@ -36,7 +36,6 @@ import ( // log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlWarn, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) // } -// TODO: extract newStreamer func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Streamer, *storage.LocalStore, func(), error) { // setup addr := RandomAddr() // tested peers peer address @@ -78,11 +77,6 @@ func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Streamer, *stora return protocolTester, streamer, localStore, teardown, nil } -// TODO -// func newStreamer() (*Streamer, error) { -// -// } - func TestStreamerSubscribe(t *testing.T) { tester, streamer, _, teardown, err := newStreamerTester(t) defer teardown() @@ -147,7 +141,6 @@ func (self *testOutgoingStreamer) GetData([]byte) []byte { } func TestStreamerDownstreamSubscribeMsgExchange(t *testing.T) { - // TODO: we only need streamer tester, streamer, _, teardown, err := newStreamerTester(t) defer teardown() if err != nil { @@ -190,7 +183,6 @@ func TestStreamerDownstreamSubscribeMsgExchange(t *testing.T) { } func TestStreamerUpstreamSubscribeMsgExchange(t *testing.T) { - // TODO: we only need streamer tester, streamer, _, teardown, err := newStreamerTester(t) defer teardown() if err != nil { @@ -244,7 +236,6 @@ func TestStreamerUpstreamSubscribeMsgExchange(t *testing.T) { } func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) { - // TODO: we only need streamer tester, streamer, _, teardown, err := newStreamerTester(t) defer teardown() if err != nil { From 8deb2d1900e479a479893971db7a718db46f5906 Mon Sep 17 00:00:00 2001 From: zelig Date: Tue, 16 Jan 2018 12:40:31 +0100 Subject: [PATCH 054/210] swarm/network: TestDeliveryFromNodes passes for 2,3 nodes --- swarm/network/request_test.go | 234 ++++++++++++++++++--------------- swarm/network/requests.go | 13 +- swarm/network/streamer.go | 11 +- swarm/network/streamer_test.go | 5 +- 4 files changed, 145 insertions(+), 118 deletions(-) diff --git a/swarm/network/request_test.go b/swarm/network/request_test.go index 7aa892dc13..36f6db7ed3 100644 --- a/swarm/network/request_test.go +++ b/swarm/network/request_test.go @@ -310,7 +310,7 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { const serviceName = "delivery" var services = adapters.Services{ - serviceName: newService, + serviceName: newDeliveryService, } var ( @@ -405,7 +405,10 @@ func testSimulation(t *testing.T, simf func(adapters.NodeAdapter) (*simulations. } func TestDeliveryFromNodes(t *testing.T) { - testSimulation(t, testDeliveryFromNodes) + testSimulation(t, testDeliveryFromNodes(2, 1, 8100, true)) + testSimulation(t, testDeliveryFromNodes(2, 1, 8100, false)) + testSimulation(t, testDeliveryFromNodes(3, 1, 8100, true)) + testSimulation(t, testDeliveryFromNodes(3, 1, 8100, false)) } var ( @@ -459,94 +462,102 @@ func mustReadAll(dpa *storage.DPA, hash storage.Key) (int, error) { return total, nil } -func testDeliveryFromNodes(adapter adapters.NodeAdapter) (*simulations.StepResult, error) { - nodes := 2 - conns := 1 - size := 8100 - skipCheck := true - - trigger := func(net *simulations.Network) chan discover.NodeID { - triggerC := make(chan discover.NodeID) - ticker := time.NewTicker(500 * time.Millisecond) - go func() { - defer ticker.Stop() - for i := 1; i < nodes; i++ { - triggerC <- net.Nodes[i].ID() - } - for range ticker.C { - triggerC <- net.Nodes[0].ID() - } - }() - return triggerC - } - - action := func(net *simulations.Network) func(context.Context) error { - rrdpa := storage.NewDPA(newRoundRobinStore(localStores[1:]...), storage.NewChunkerParams()) - rrdpa.Start() - dpacs := storage.NewDpaChunkStore(localStores[0].(*storage.LocalStore), func(chunk *storage.Chunk) error { return delivery.RequestFromPeers(chunk.Key[:], skipCheck) }) - dpa := storage.NewDPA(dpacs, storage.NewChunkerParams()) - dpa.Start() - return func(context.Context) error { - defer rrdpa.Stop() - hash, wait, err := rrdpa.Store(io.LimitReader(crand.Reader, int64(size)), int64(size)) - if err != nil { - return err - } - wait() - fileHash = hash +func testDeliveryFromNodes(nodes, conns, size int, skipCheck bool) func(adapter adapters.NodeAdapter) (*simulations.StepResult, error) { + return func(adapter adapters.NodeAdapter) (*simulations.StepResult, error) { + trigger := func(net *simulations.Network) chan discover.NodeID { + triggerC := make(chan discover.NodeID) + ticker := time.NewTicker(500 * time.Millisecond) go func() { - defer dpa.Stop() - log.Debug(fmt.Sprintf("retrieve %v", fileHash)) - time.Sleep(2 * time.Second) - n, err := mustReadAll(dpa, fileHash) - log.Debug(fmt.Sprintf("retrieved %v", fileHash), "read", n, "err", err) + defer ticker.Stop() + // we are only testing the pivot node (net.Nodes[0]) but simulation needs + // all nodes to pass the check so we trigger each and the check function + // will trivially return true + for i := 1; i < nodes; i++ { + triggerC <- net.Nodes[i].ID() + } + for range ticker.C { + triggerC <- net.Nodes[0].ID() + } }() - return nil + return triggerC } - } - check := func(net *simulations.Network, dpa *storage.DPA) func(ctx context.Context, id discover.NodeID) (bool, error) { - return func(ctx context.Context, id discover.NodeID) (bool, error) { - if id != net.Nodes[0].ID() { + action := func(net *simulations.Network) func(context.Context) error { + // here we distribute chunks of a random file into localstores of nodes 1 to nodes + rrdpa := storage.NewDPA(newRoundRobinStore(localStores[1:]...), storage.NewChunkerParams()) + rrdpa.Start() + // create a retriever dpa for the pivot node + dpacs := storage.NewDpaChunkStore(localStores[0].(*storage.LocalStore), func(chunk *storage.Chunk) error { return delivery.RequestFromPeers(chunk.Key[:], skipCheck) }) + dpa := storage.NewDPA(dpacs, storage.NewChunkerParams()) + dpa.Start() + return func(context.Context) error { + defer rrdpa.Stop() + // upload an actual random file of size size + hash, wait, err := rrdpa.Store(io.LimitReader(crand.Reader, int64(size)), int64(size)) + if err != nil { + return err + } + // wait until all chunks stored + wait() + // assign the fileHash to a global so that it is available for the check function + fileHash = hash + go func() { + defer dpa.Stop() + log.Debug(fmt.Sprintf("retrieve %v", fileHash)) + // start the retrieval on the pivot node - this will spawn retrieve requests for missing chunks + // we must wait for the peer connections to have started before requesting + time.Sleep(2 * time.Second) + n, err := mustReadAll(dpa, fileHash) + log.Debug(fmt.Sprintf("retrieved %v", fileHash), "read", n, "err", err) + }() + return nil + } + } + + check := func(net *simulations.Network, dpa *storage.DPA) func(ctx context.Context, id discover.NodeID) (bool, error) { + return func(ctx context.Context, id discover.NodeID) (bool, error) { + if id != net.Nodes[0].ID() { + return true, nil + } + select { + case <-ctx.Done(): + return false, ctx.Err() + default: + } + // try to locally retrieve the file to check if retrieve requests have been successful + log.Warn(fmt.Sprintf("try to locally retrieve %v", fileHash)) + total, err := mustReadAll(dpa, fileHash) + if err != nil || total != size { + log.Warn(fmt.Sprintf("number of bytes read %v/%v (error: %v)", total, size, err)) + return false, nil + } return true, nil + // node := net.GetNode(id) + // if node == nil { + // return false, fmt.Errorf("unknown node: %s", id) + // } + // client, err := node.Client() + // if err != nil { + // return false, fmt.Errorf("error getting node client: %s", err) + // } + // var response int + // if err := client.Call(&response, "test_haslocal", hash); err != nil { + // return false, fmt.Errorf("error getting bzz_has response: %s", err) + // } + // log.Debug(fmt.Sprintf("node has: %v\n%v", id, response)) + // return response == 0, nil } - select { - case <-ctx.Done(): - return false, ctx.Err() - default: - } - log.Warn(fmt.Sprintf("try to locally retrieve %v", fileHash)) - total, err := mustReadAll(dpa, fileHash) - if err != nil || total != size { - log.Warn(fmt.Sprintf("number of bytes read %v/%v (error: %v)", total, size, err)) - return false, nil - } - return true, nil - // node := net.GetNode(id) - // if node == nil { - // return false, fmt.Errorf("unknown node: %s", id) - // } - // client, err := node.Client() - // if err != nil { - // return false, fmt.Errorf("error getting node client: %s", err) - // } - // var response int - // if err := client.Call(&response, "test_haslocal", hash); err != nil { - // return false, fmt.Errorf("error getting bzz_has response: %s", err) - // } - // log.Debug(fmt.Sprintf("node has: %v\n%v", id, response)) - // return response == 0, nil } - } - result, err := runSimulation(nodes, conns, action, trigger, check, adapter) - if err != nil { - return nil, fmt.Errorf("Setting up simulation failed: %v", err) + result, err := runSimulation(nodes, conns, action, trigger, check, adapter) + if err != nil { + return nil, fmt.Errorf("Setting up simulation failed: %v", err) + } + if result.Error != nil { + return nil, fmt.Errorf("Simulation failed: %s", result.Error) + } + return result, err } - if result.Error != nil { - return nil, fmt.Errorf("Simulation failed: %s", result.Error) - } - return result, err } func runSimulation(nodes, conns int, action func(*simulations.Network) func(context.Context) error, trigger func(*simulations.Network) chan discover.NodeID, check func(*simulations.Network, *storage.DPA) func(context.Context, discover.NodeID) (bool, error), adapter adapters.NodeAdapter) (*simulations.StepResult, error) { @@ -556,6 +567,7 @@ func runSimulation(nodes, conns int, action func(*simulations.Network) func(cont DefaultService: serviceName, }) defer net.Shutdown() + // set nodes number of localstores globally available teardown, err := setLocalStores(nodes) defer teardown() if err != nil { @@ -563,6 +575,7 @@ func runSimulation(nodes, conns int, action func(*simulations.Network) func(cont } ids := make([]discover.NodeID, nodes) nodeCount = 0 + // start nodes for i := 0; i < nodes; i++ { node, err := net.NewNode() if err != nil { @@ -574,8 +587,7 @@ func runSimulation(nodes, conns int, action func(*simulations.Network) func(cont ids[i] = node.ID() } - // run a simulation which connects the 10 nodes in a ring and waits - // for full peer discovery + // run a simulation which connects the 10 nodes in a chain var addrs [][]byte wg := sync.WaitGroup{} log.Warn("runSimulation 1") @@ -606,10 +618,11 @@ func runSimulation(nodes, conns int, action func(*simulations.Network) func(cont log.Debug(fmt.Sprintf("nodes: %v", len(addrs))) - // 64 nodes ~ 1min - // 128 nodes ~ + // create an only locally retrieving dpa for the pivot node to test + // if retriee requests have arrived dpa := storage.NewDPA(localStores[0], storage.NewChunkerParams()) dpa.Start() + defer dpa.Stop() timeout := 300 * time.Second ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() @@ -624,7 +637,8 @@ func runSimulation(nodes, conns int, action func(*simulations.Network) func(cont return result, nil } -func newService(ctx *adapters.ServiceContext) (node.Service, error) { +// newDeliveryService +func newDeliveryService(ctx *adapters.ServiceContext) (node.Service, error) { id := ctx.Config.ID addr := NewAddrFromNodeID(id) kad := NewKademlia(addr.Over(), NewKadParams()) @@ -632,34 +646,23 @@ func newService(ctx *adapters.ServiceContext) (node.Service, error) { dbAccess := NewDbAccess(localStore.(*storage.LocalStore)) streamer := NewStreamer(NewDelivery(kad, dbAccess)) if nodeCount == 0 { + // the delivery service for the pivot node is assigned globally + // so that the simulation action call can use it for the + // swarm enabled dpa delivery = streamer.delivery } nodeCount++ - run := func(p *p2p.Peer, rw p2p.MsgReadWriter) error { - bzzPeer := &bzzPeer{ - Peer: protocols.NewPeer(p, rw, StreamerSpec), - localAddr: addr, - BzzAddr: NewAddrFromNodeID(p.ID()), - } - log.Warn("Run function kad On ", "local", id, "remote", p.ID()) - kad.On(bzzPeer) - go func() { - time.Sleep(1 * time.Second) - err := streamer.Subscribe(p.ID(), retrieveRequestStream, nil, 0, 0, Top, true) - if err != nil { - log.Warn("error in subscribe", "err", err) - } - }() - return streamer.Run(bzzPeer) - } + log.Warn("new service created") return &testDeliveryService{ - run: run, + addr: addr, + streamer: streamer, }, nil } type testDeliveryService struct { - run func(p *p2p.Peer, rw p2p.MsgReadWriter) error + addr *BzzAddr + streamer *Streamer } func (tds *testDeliveryService) Protocols() []p2p.Protocol { @@ -687,3 +690,24 @@ func (b *testDeliveryService) Start(server *p2p.Server) error { func (b *testDeliveryService) Stop() error { return nil } + +func (b *testDeliveryService) run(p *p2p.Peer, rw p2p.MsgReadWriter) error { + bzzPeer := &bzzPeer{ + Peer: protocols.NewPeer(p, rw, StreamerSpec), + localAddr: b.addr, + BzzAddr: NewAddrFromNodeID(p.ID()), + } + b.streamer.delivery.overlay.On(bzzPeer) + defer b.streamer.delivery.overlay.Off(bzzPeer) + go func() { + // each node Subscribes to each other's retrieveRequestStream + // need to wait till an aynchronous process registers the peers in streamer.peers + // that is used by Subscribe + time.Sleep(1 * time.Second) + err := b.streamer.Subscribe(p.ID(), retrieveRequestStream, nil, 0, 0, Top, true) + if err != nil { + log.Warn("error in subscribe", "err", err) + } + }() + return b.streamer.Run(bzzPeer) +} diff --git a/swarm/network/requests.go b/swarm/network/requests.go index d2cf9fa344..7cc0a47946 100644 --- a/swarm/network/requests.go +++ b/swarm/network/requests.go @@ -49,7 +49,7 @@ func NewDelivery(overlay Overlay, dbAccess *DbAccess) *Delivery { // RetrieveRequestStreamer implements OutgoingStreamer type RetrieveRequestStreamer struct { - deliveryC chan *storage.Chunk + deliveryC chan []byte batchC chan []byte dbAccess *DbAccess currentLen uint64 @@ -58,7 +58,7 @@ type RetrieveRequestStreamer struct { // NewRetrieveRequestStreamer is RetrieveRequestStreamer constructor func NewRetrieveRequestStreamer(dbAccess *DbAccess) *RetrieveRequestStreamer { s := &RetrieveRequestStreamer{ - deliveryC: make(chan *storage.Chunk), + deliveryC: make(chan []byte), batchC: make(chan []byte), dbAccess: dbAccess, } @@ -72,11 +72,12 @@ func (s *RetrieveRequestStreamer) processDeliveries() { var batchC chan []byte for { select { - case delivery := <-s.deliveryC: - hashes = append(hashes, delivery.Key[:]...) + case hash := <-s.deliveryC: + hashes = append(hashes, hash...) batchC = s.batchC case batchC <- hashes: hashes = nil + batchC = nil } } } @@ -131,7 +132,7 @@ func (self *Delivery) handleRetrieveRequestMsg(sp *StreamerPeer, req *RetrieveRe sp.Deliver(chunk, s.priority) return } - streamer.deliveryC <- chunk + streamer.deliveryC <- chunk.Key[:] }() return nil } @@ -139,7 +140,7 @@ func (self *Delivery) handleRetrieveRequestMsg(sp *StreamerPeer, req *RetrieveRe if req.SkipCheck { return sp.Deliver(chunk, s.priority) } - streamer.deliveryC <- chunk + streamer.deliveryC <- chunk.Key[:] return nil } diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go index 61351bf392..ea64734fd1 100644 --- a/swarm/network/streamer.go +++ b/swarm/network/streamer.go @@ -406,7 +406,7 @@ func (self *StreamerPeer) handleOfferedHashesMsg(req *OfferedHashesMsg) error { hashes := req.Hashes want, err := bv.New(len(hashes) / HashSize) if err != nil { - return err + return fmt.Errorf("error initiaising bitvector of length %v: %v", len(hashes)/HashSize, err) } wg := sync.WaitGroup{} for i := 0; i < len(hashes); i += HashSize { @@ -472,7 +472,7 @@ func (self *StreamerPeer) handleWantedHashesMsg(req *WantedHashesMsg) error { l := len(hashes) / HashSize want, err := bv.NewFromBytes(req.Want, l) if err != nil { - return err + return fmt.Errorf("error initiaising bitvector of length %v: %v", l, err) } for i := 0; i < l; i++ { if want.Get(i) { @@ -514,12 +514,17 @@ func (self *StreamerPeer) SendPriority(msg interface{}, priority uint8) error { return self.pq.Push(nil, msg, int(priority)) } -// OfferedHashes sends OfferedHashesMsg protocol msg +// SendOfferedHashes sends OfferedHashesMsg protocol msg func (self *StreamerPeer) SendOfferedHashes(s *outgoingStreamer, f, t uint64) error { hashes, from, to, proof, err := s.SetNextBatch(f, t) if err != nil { return err } + if proof == nil { + proof = &HandoverProof{ + Handover: &Handover{}, + } + } s.currentBatch = hashes msg := &OfferedHashesMsg{ HandoverProof: proof, diff --git a/swarm/network/streamer_test.go b/swarm/network/streamer_test.go index 9d44934385..86ba965071 100644 --- a/swarm/network/streamer_test.go +++ b/swarm/network/streamer_test.go @@ -130,10 +130,7 @@ func (self *testIncomingStreamer) BatchDone(string, uint64, []byte, []byte) func } func (self *testOutgoingStreamer) SetNextBatch(from uint64, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) { - proof := &HandoverProof{ - Handover: &Handover{}, - } - return make([]byte, HashSize), from + 1, to + 1, proof, nil + return make([]byte, HashSize), from + 1, to + 1, nil, nil } func (self *testOutgoingStreamer) GetData([]byte) []byte { From f08cd94fb755471cb78091af99ef7026afb392f3 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 16 Jan 2018 15:42:41 +0100 Subject: [PATCH 055/210] cmd/ethkey: fix formatting, review nits (#15807) This commit: - Adds a --msgfile option to read the message to sign from a file instead of command line argument. - Adds a unit test for signing subcommands. - Removes some weird whitespace in the code. --- cmd/ethkey/generate.go | 63 +++++++++++------------ cmd/ethkey/inspect.go | 1 + cmd/ethkey/main.go | 41 +++++++-------- cmd/ethkey/message.go | 97 ++++++++++++++++++++---------------- cmd/ethkey/message_test.go | 70 ++++++++++++++++++++++++++ cmd/ethkey/run_test.go | 54 ++++++++++++++++++++ internal/cmdtest/test_cmd.go | 8 +-- 7 files changed, 235 insertions(+), 99 deletions(-) create mode 100644 cmd/ethkey/message_test.go create mode 100644 cmd/ethkey/run_test.go diff --git a/cmd/ethkey/generate.go b/cmd/ethkey/generate.go index dee0e9d70e..6d57d17fb4 100644 --- a/cmd/ethkey/generate.go +++ b/cmd/ethkey/generate.go @@ -1,8 +1,23 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + package main import ( "crypto/ecdsa" - "crypto/rand" "fmt" "io/ioutil" "os" @@ -26,16 +41,16 @@ var commandGenerate = cli.Command{ ArgsUsage: "[ ]", Description: ` Generate a new keyfile. -If you want to use an existing private key to use in the keyfile, it can be -specified by setting --privatekey with the location of the file containing the -private key.`, + +If you want to encrypt an existing private key, it can be specified by setting +--privatekey with the location of the file containing the private key. +`, Flags: []cli.Flag{ passphraseFlag, jsonFlag, cli.StringFlag{ - Name: "privatekey", - Usage: "the file from where to read the private key to " + - "generate a keyfile for", + Name: "privatekey", + Usage: "file containing a raw private key to encrypt", }, }, Action: func(ctx *cli.Context) error { @@ -51,32 +66,19 @@ private key.`, } var privateKey *ecdsa.PrivateKey - - // First check if a private key file is provided. - privateKeyFile := ctx.String("privatekey") - if privateKeyFile != "" { - privateKeyBytes, err := ioutil.ReadFile(privateKeyFile) + var err error + if file := ctx.String("privatekey"); file != "" { + // Load private key from file. + privateKey, err = crypto.LoadECDSA(file) if err != nil { - utils.Fatalf("Failed to read the private key file '%s': %v", - privateKeyFile, err) + utils.Fatalf("Can't load private key: %v", err) } - - pk, err := crypto.HexToECDSA(string(privateKeyBytes)) - if err != nil { - utils.Fatalf( - "Could not construct ECDSA private key from file content: %v", - err) - } - privateKey = pk - } - - // If not loaded, generate random. - if privateKey == nil { - pk, err := ecdsa.GenerateKey(crypto.S256(), rand.Reader) + } else { + // If not loaded, generate random. + privateKey, err = crypto.GenerateKey() if err != nil { utils.Fatalf("Failed to generate random private key: %v", err) } - privateKey = pk } // Create the keyfile object with a random UUID. @@ -89,8 +91,7 @@ private key.`, // Encrypt key with passphrase. passphrase := getPassPhrase(ctx, true) - keyjson, err := keystore.EncryptKey(key, passphrase, - keystore.StandardScryptN, keystore.StandardScryptP) + keyjson, err := keystore.EncryptKey(key, passphrase, keystore.StandardScryptN, keystore.StandardScryptP) if err != nil { utils.Fatalf("Error encrypting key: %v", err) } @@ -110,7 +111,7 @@ private key.`, if ctx.Bool(jsonFlag.Name) { mustPrintJSON(out) } else { - fmt.Println("Address: ", out.Address) + fmt.Println("Address:", out.Address) } return nil }, diff --git a/cmd/ethkey/inspect.go b/cmd/ethkey/inspect.go index 8a7aeef848..219a5460b8 100644 --- a/cmd/ethkey/inspect.go +++ b/cmd/ethkey/inspect.go @@ -23,6 +23,7 @@ var commandInspect = cli.Command{ ArgsUsage: "", Description: ` Print various information about the keyfile. + Private key information can be printed by using the --private flag; make sure to use this feature with great caution!`, Flags: []cli.Flag{ diff --git a/cmd/ethkey/main.go b/cmd/ethkey/main.go index b9b7a18e05..2a9e5ee483 100644 --- a/cmd/ethkey/main.go +++ b/cmd/ethkey/main.go @@ -28,30 +28,11 @@ const ( defaultKeyfileName = "keyfile.json" ) -var ( - gitCommit = "" // Git SHA1 commit hash of the release (set via linker flags) +// Git SHA1 commit hash of the release (set via linker flags) +var gitCommit = "" - app *cli.App // the main app instance -) +var app *cli.App -var ( // Commonly used command line flags. - passphraseFlag = cli.StringFlag{ - Name: "passwordfile", - Usage: "the file that contains the passphrase for the keyfile", - } - - jsonFlag = cli.BoolFlag{ - Name: "json", - Usage: "output JSON instead of human-readable format", - } - - messageFlag = cli.StringFlag{ - Name: "message", - Usage: "the file that contains the message to sign/verify", - } -) - -// Configure the app instance. func init() { app = utils.NewApp(gitCommit, "an Ethereum key manager") app.Commands = []cli.Command{ @@ -62,6 +43,22 @@ func init() { } } +// Commonly used command line flags. +var ( + passphraseFlag = cli.StringFlag{ + Name: "passwordfile", + Usage: "the file that contains the passphrase for the keyfile", + } + jsonFlag = cli.BoolFlag{ + Name: "json", + Usage: "output JSON instead of human-readable format", + } + messageFlag = cli.StringFlag{ + Name: "message", + Usage: "the file that contains the message to sign/verify", + } +) + func main() { if err := app.Run(os.Args); err != nil { fmt.Fprintln(os.Stderr, err) diff --git a/cmd/ethkey/message.go b/cmd/ethkey/message.go index ae6b6552d3..531a931c82 100644 --- a/cmd/ethkey/message.go +++ b/cmd/ethkey/message.go @@ -1,11 +1,25 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + package main import ( "encoding/hex" "fmt" "io/ioutil" - "os" - "strings" "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/cmd/utils" @@ -18,26 +32,33 @@ type outputSign struct { Signature string } +var msgfileFlag = cli.StringFlag{ + Name: "msgfile", + Usage: "file containing the message to sign/verify", +} + var commandSignMessage = cli.Command{ Name: "signmessage", Usage: "sign a message", - ArgsUsage: " ", + ArgsUsage: " ", Description: ` Sign the message with a keyfile. -It is possible to refer to a file containing the message.`, + +To sign a message contained in a file, use the --msgfile flag. +`, Flags: []cli.Flag{ passphraseFlag, jsonFlag, + msgfileFlag, }, Action: func(ctx *cli.Context) error { - keyfilepath := ctx.Args().First() - message := []byte(ctx.Args().Get(1)) + message := getMessage(ctx, 1) // Load the keyfile. + keyfilepath := ctx.Args().First() keyjson, err := ioutil.ReadFile(keyfilepath) if err != nil { - utils.Fatalf("Failed to read the keyfile at '%s': %v", - keyfilepath, err) + utils.Fatalf("Failed to read the keyfile at '%s': %v", keyfilepath, err) } // Decrypt key with passphrase. @@ -47,29 +68,15 @@ It is possible to refer to a file containing the message.`, utils.Fatalf("Error decrypting key: %v", err) } - if len(message) == 0 { - utils.Fatalf("A message must be provided") - } - // Read message if file. - if _, err := os.Stat(string(message)); err == nil { - message, err = ioutil.ReadFile(string(message)) - if err != nil { - utils.Fatalf("Failed to read the message file: %v", err) - } - } - signature, err := crypto.Sign(signHash(message), key.PrivateKey) if err != nil { utils.Fatalf("Failed to sign message: %v", err) } - - out := outputSign{ - Signature: hex.EncodeToString(signature), - } + out := outputSign{Signature: hex.EncodeToString(signature)} if ctx.Bool(jsonFlag.Name) { mustPrintJSON(out) } else { - fmt.Println("Signature: ", out.Signature) + fmt.Println("Signature:", out.Signature) } return nil }, @@ -84,53 +91,40 @@ type outputVerify struct { var commandVerifyMessage = cli.Command{ Name: "verifymessage", Usage: "verify the signature of a signed message", - ArgsUsage: "
", + ArgsUsage: "
", Description: ` Verify the signature of the message. It is possible to refer to a file containing the message.`, Flags: []cli.Flag{ jsonFlag, + msgfileFlag, }, Action: func(ctx *cli.Context) error { addressStr := ctx.Args().First() signatureHex := ctx.Args().Get(1) - message := []byte(ctx.Args().Get(2)) + message := getMessage(ctx, 2) - // Determine whether it is a keyfile, public key or address. if !common.IsHexAddress(addressStr) { utils.Fatalf("Invalid address: %s", addressStr) } address := common.HexToAddress(addressStr) - signature, err := hex.DecodeString(signatureHex) if err != nil { utils.Fatalf("Signature encoding is not hexadecimal: %v", err) } - if len(message) == 0 { - utils.Fatalf("A message must be provided") - } - // Read message if file. - if _, err := os.Stat(string(message)); err == nil { - message, err = ioutil.ReadFile(string(message)) - if err != nil { - utils.Fatalf("Failed to read the message file: %v", err) - } - } - recoveredPubkey, err := crypto.SigToPub(signHash(message), signature) if err != nil || recoveredPubkey == nil { utils.Fatalf("Signature verification failed: %v", err) } recoveredPubkeyBytes := crypto.FromECDSAPub(recoveredPubkey) recoveredAddress := crypto.PubkeyToAddress(*recoveredPubkey) - success := address == recoveredAddress out := outputVerify{ Success: success, RecoveredPublicKey: hex.EncodeToString(recoveredPubkeyBytes), - RecoveredAddress: strings.ToLower(recoveredAddress.Hex()), + RecoveredAddress: recoveredAddress.Hex(), } if ctx.Bool(jsonFlag.Name) { mustPrintJSON(out) @@ -140,9 +134,26 @@ It is possible to refer to a file containing the message.`, } else { fmt.Println("Signature verification failed!") } - fmt.Println("Recovered public key: ", out.RecoveredPublicKey) - fmt.Println("Recovered address: ", out.RecoveredAddress) + fmt.Println("Recovered public key:", out.RecoveredPublicKey) + fmt.Println("Recovered address:", out.RecoveredAddress) } return nil }, } + +func getMessage(ctx *cli.Context, msgarg int) []byte { + if file := ctx.String("msgfile"); file != "" { + if len(ctx.Args()) > msgarg { + utils.Fatalf("Can't use --msgfile and message argument at the same time.") + } + msg, err := ioutil.ReadFile(file) + if err != nil { + utils.Fatalf("Can't read message file: %v", err) + } + return msg + } else if len(ctx.Args()) == msgarg+1 { + return []byte(ctx.Args().Get(msgarg)) + } + utils.Fatalf("Invalid number of arguments: want %d, got %d", msgarg+1, len(ctx.Args())) + return nil +} diff --git a/cmd/ethkey/message_test.go b/cmd/ethkey/message_test.go new file mode 100644 index 0000000000..fb16f03d02 --- /dev/null +++ b/cmd/ethkey/message_test.go @@ -0,0 +1,70 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +func TestMessageSignVerify(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "ethkey-test") + if err != nil { + t.Fatal("Can't create temporary directory:", err) + } + defer os.RemoveAll(tmpdir) + + keyfile := filepath.Join(tmpdir, "the-keyfile") + message := "test message" + + // Create the key. + generate := runEthkey(t, "generate", keyfile) + generate.Expect(` +!! Unsupported terminal, password will be echoed. +Passphrase: {{.InputLine "foobar"}} +Repeat passphrase: {{.InputLine "foobar"}} +`) + _, matches := generate.ExpectRegexp(`Address: (0x[0-9a-fA-F]{40})\n`) + address := matches[1] + generate.ExpectExit() + + // Sign a message. + sign := runEthkey(t, "signmessage", keyfile, message) + sign.Expect(` +!! Unsupported terminal, password will be echoed. +Passphrase: {{.InputLine "foobar"}} +`) + _, matches = sign.ExpectRegexp(`Signature: ([0-9a-f]+)\n`) + signature := matches[1] + sign.ExpectExit() + + // Verify the message. + verify := runEthkey(t, "verifymessage", address, signature, message) + _, matches = verify.ExpectRegexp(` +Signature verification successful! +Recovered public key: [0-9a-f]+ +Recovered address: (0x[0-9a-fA-F]{40}) +`) + recovered := matches[1] + verify.ExpectExit() + + if recovered != address { + t.Error("recovered address doesn't match generated key") + } +} diff --git a/cmd/ethkey/run_test.go b/cmd/ethkey/run_test.go new file mode 100644 index 0000000000..8ce4fe5cde --- /dev/null +++ b/cmd/ethkey/run_test.go @@ -0,0 +1,54 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "fmt" + "os" + "testing" + + "github.com/docker/docker/pkg/reexec" + "github.com/ethereum/go-ethereum/internal/cmdtest" +) + +type testEthkey struct { + *cmdtest.TestCmd +} + +// spawns ethkey with the given command line args. +func runEthkey(t *testing.T, args ...string) *testEthkey { + tt := new(testEthkey) + tt.TestCmd = cmdtest.NewTestCmd(t, tt) + tt.Run("ethkey-test", args...) + return tt +} + +func TestMain(m *testing.M) { + // Run the app if we've been exec'd as "ethkey-test" in runEthkey. + reexec.Register("ethkey-test", func() { + if err := app.Run(os.Args); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + os.Exit(0) + }) + // check if we have been reexec'd + if reexec.Init() { + return + } + os.Exit(m.Run()) +} diff --git a/internal/cmdtest/test_cmd.go b/internal/cmdtest/test_cmd.go index 541e51c4c0..fae61cfe32 100644 --- a/internal/cmdtest/test_cmd.go +++ b/internal/cmdtest/test_cmd.go @@ -25,6 +25,7 @@ import ( "os" "os/exec" "regexp" + "strings" "sync" "testing" "text/template" @@ -141,9 +142,10 @@ func (tt *TestCmd) matchExactOutput(want []byte) error { // Note that an arbitrary amount of output may be consumed by the // regular expression. This usually means that expect cannot be used // after ExpectRegexp. -func (tt *TestCmd) ExpectRegexp(resource string) (*regexp.Regexp, []string) { +func (tt *TestCmd) ExpectRegexp(regex string) (*regexp.Regexp, []string) { + regex = strings.TrimPrefix(regex, "\n") var ( - re = regexp.MustCompile(resource) + re = regexp.MustCompile(regex) rtee = &runeTee{in: tt.stdout} matches []int ) @@ -151,7 +153,7 @@ func (tt *TestCmd) ExpectRegexp(resource string) (*regexp.Regexp, []string) { output := rtee.buf.Bytes() if matches == nil { tt.Fatalf("Output did not match:\n---------------- (stdout text)\n%s\n---------------- (regular expression)\n%s", - output, resource) + output, regex) return re, nil } tt.Logf("Matched stdout text:\n%s", output) From 370dca4491de92ee81e0755cee1cb5df12f0f665 Mon Sep 17 00:00:00 2001 From: George Ornbo Date: Tue, 16 Jan 2018 15:45:13 +0000 Subject: [PATCH 056/210] core/vm: Fix comment typo --- core/vm/evm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/vm/evm.go b/core/vm/evm.go index 8796a633ec..46e7baff4c 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -391,7 +391,7 @@ func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.I return ret, contractAddr, contract.Gas, err } -// ChainConfig returns the evmironment's chain configuration +// ChainConfig returns the environment's chain configuration func (evm *EVM) ChainConfig() *params.ChainConfig { return evm.chainConfig } // Interpreter returns the EVM interpreter From 365bd3b6d2bd417c676bcbe6c9b82d133ae2ea4f Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Tue, 16 Jan 2018 18:37:32 +0100 Subject: [PATCH 057/210] swarm/network: Draft test for syncer --- swarm/network/bitvector/bitvector.go | 4 +- swarm/network/request_test.go | 31 ++-- swarm/network/streamer.go | 8 +- swarm/network/syncer.go | 48 +++--- swarm/network/syncer_test.go | 220 +++++++++++++++++++++++++++ swarm/swarm.go | 4 +- 6 files changed, 275 insertions(+), 40 deletions(-) create mode 100644 swarm/network/syncer_test.go diff --git a/swarm/network/bitvector/bitvector.go b/swarm/network/bitvector/bitvector.go index 93e55a9d09..256c9fd5f3 100644 --- a/swarm/network/bitvector/bitvector.go +++ b/swarm/network/bitvector/bitvector.go @@ -1,6 +1,8 @@ package bitvector -import "errors" +import ( + "errors" +) var errInvalidLength = errors.New("invalid length") diff --git a/swarm/network/request_test.go b/swarm/network/request_test.go index 36f6db7ed3..cd014a4509 100644 --- a/swarm/network/request_test.go +++ b/swarm/network/request_test.go @@ -305,12 +305,9 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { } -// serviceName is used with the exec adapter so the exec'd binary knows which -// service to execute -const serviceName = "delivery" - var services = adapters.Services{ - serviceName: newDeliveryService, + "delivery": newDeliveryService, + "syncer": newSyncerService, } var ( @@ -498,6 +495,7 @@ func testDeliveryFromNodes(nodes, conns, size int, skipCheck bool) func(adapter return err } // wait until all chunks stored + // TODO: is wait() necessary? wait() // assign the fileHash to a global so that it is available for the check function fileHash = hash @@ -549,7 +547,7 @@ func testDeliveryFromNodes(nodes, conns, size int, skipCheck bool) func(adapter } } - result, err := runSimulation(nodes, conns, action, trigger, check, adapter) + result, err := runSimulation(nodes, conns, "delivery", action, trigger, check, adapter) if err != nil { return nil, fmt.Errorf("Setting up simulation failed: %v", err) } @@ -560,7 +558,7 @@ func testDeliveryFromNodes(nodes, conns, size int, skipCheck bool) func(adapter } } -func runSimulation(nodes, conns int, action func(*simulations.Network) func(context.Context) error, trigger func(*simulations.Network) chan discover.NodeID, check func(*simulations.Network, *storage.DPA) func(context.Context, discover.NodeID) (bool, error), adapter adapters.NodeAdapter) (*simulations.StepResult, error) { +func runSimulation(nodes, conns int, serviceName string, action func(*simulations.Network) func(context.Context) error, trigger func(*simulations.Network) chan discover.NodeID, check func(*simulations.Network, *storage.DPA) func(context.Context, discover.NodeID) (bool, error), adapter adapters.NodeAdapter) (*simulations.StepResult, error) { // create network net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{ ID: "0", @@ -654,18 +652,21 @@ func newDeliveryService(ctx *adapters.ServiceContext) (node.Service, error) { nodeCount++ log.Warn("new service created") - return &testDeliveryService{ + self := &testStreamerService{ addr: addr, streamer: streamer, - }, nil + } + self.run = self.runDelivery + return self, nil } -type testDeliveryService struct { +type testStreamerService struct { addr *BzzAddr streamer *Streamer + run func(p *p2p.Peer, rw p2p.MsgReadWriter) error } -func (tds *testDeliveryService) Protocols() []p2p.Protocol { +func (tds *testStreamerService) Protocols() []p2p.Protocol { log.Warn("Protocols function", "run", tds.run) return []p2p.Protocol{ { @@ -679,19 +680,19 @@ func (tds *testDeliveryService) Protocols() []p2p.Protocol { } } -func (b *testDeliveryService) APIs() []rpc.API { +func (b *testStreamerService) APIs() []rpc.API { return []rpc.API{} } -func (b *testDeliveryService) Start(server *p2p.Server) error { +func (b *testStreamerService) Start(server *p2p.Server) error { return nil } -func (b *testDeliveryService) Stop() error { +func (b *testStreamerService) Stop() error { return nil } -func (b *testDeliveryService) run(p *p2p.Peer, rw p2p.MsgReadWriter) error { +func (b *testStreamerService) runDelivery(p *p2p.Peer, rw p2p.MsgReadWriter) error { bzzPeer := &bzzPeer{ Peer: protocols.NewPeer(p, rw, StreamerSpec), localAddr: b.addr, diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go index ea64734fd1..3c367f7b6f 100644 --- a/swarm/network/streamer.go +++ b/swarm/network/streamer.go @@ -22,6 +22,7 @@ import ( "fmt" "sync" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/protocols" bv "github.com/ethereum/go-ethereum/swarm/network/bitvector" @@ -264,7 +265,7 @@ func (self *StreamerPeer) getOutgoingStreamer(s string) (*outgoingStreamer, erro defer self.outgoingLock.RUnlock() streamer := self.outgoing[s] if streamer == nil { - return nil, fmt.Errorf("stream '%v' not provided", s) + return nil, fmt.Errorf("outgoing stream '%v' not provided to peer %v", s, self.ID()) } return streamer, nil } @@ -274,7 +275,7 @@ func (self *StreamerPeer) getIncomingStreamer(s string) (*incomingStreamer, erro defer self.incomingLock.RUnlock() streamer := self.incoming[s] if streamer == nil { - return nil, fmt.Errorf("stream '%v' not provided", s) + return nil, fmt.Errorf("incoming stream '%v' not provided to peer %v", s, self.ID()) } return streamer, nil } @@ -348,6 +349,7 @@ func (self *incomingStreamer) nextBatch(from uint64) (nextFrom uint64, nextTo ui // Subscribe initiates the streamer func (self *Streamer) Subscribe(peerId discover.NodeID, s string, t []byte, from, to uint64, priority uint8, live bool) error { + log.Warn("!!!!!! Subscribe ", "peer", peerId) f, err := self.GetIncomingStreamer(s) if err != nil { return err @@ -362,7 +364,7 @@ func (self *Streamer) Subscribe(peerId discover.NodeID, s string, t []byte, from if err != nil { return err } - err = peer.setIncomingStreamer(s, is, priority, live) + err = peer.setIncomingStreamer(s+string(t), is, priority, live) if err != nil { return err } diff --git a/swarm/network/syncer.go b/swarm/network/syncer.go index c9fe1d3555..db2e551826 100644 --- a/swarm/network/syncer.go +++ b/swarm/network/syncer.go @@ -21,6 +21,8 @@ import ( "errors" "fmt" "io" + "math" + "time" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/discover" @@ -58,6 +60,7 @@ func (self *DbAccess) iterator(from uint64, to uint64, po uint8, f func(storage. // to obtain the chunks from key or request db entry only func (self *DbAccess) getOrCreateRequest(key storage.Key) (*storage.Chunk, bool) { + log.Warn("getOrCreateRequest", "self", self) return self.loc.GetOrCreateRequest(key) } @@ -95,9 +98,11 @@ func NewOutgoingSwarmSyncer(live bool, po uint8, db *DbAccess) (*OutgoingSwarmSy const maxPO = 32 -func RegisterOutgoingSyncers(streamer *Streamer, db *DbAccess) { +func RegisterOutgoingSyncer(streamer *Streamer, db *DbAccess) { streamer.RegisterOutgoingStreamer("SYNC", func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) { syncType, po := parseSyncLabel(t) + // TODO: make this work for HISTORY too + syncType = "LIVE" switch syncType { case "LIVE": return NewOutgoingSwarmSyncer(true, po, db) @@ -128,17 +133,29 @@ func (self *OutgoingSwarmSyncer) SetNextBatch(from, to uint64) ([]byte, uint64, if from == 0 { from = self.start } - err := self.db.iterator(from, to, self.po, func(key storage.Key, idx uint64) bool { - batch = append(batch, key[:]...) - i++ - to = idx - return i < batchSize - }) - if err != nil { - return nil, 0, 0, nil, err + if to <= from { + to = math.MaxUint64 } + log.Warn("!!!!!!!!!!!!! setNextBatch", "from", from, "to", to, "currentStoreCount", self.db.currentBucketStorageIndex(1)) + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + for range ticker.C { + err := self.db.iterator(from, to, self.po, func(key storage.Key, idx uint64) bool { + batch = append(batch, key[:]...) + i++ + to = idx + return i < batchSize + }) + if err != nil { + return nil, 0, 0, nil, err + } + if len(batch) > 0 { + break + } + } + log.Debug("Swarm batch", "po", self.po, "len", i, "from", from, "to", to) - return batch, from, to, nil, nil + return batch, from, to + 1, nil, nil } // IncomingSwarmSyncer @@ -212,16 +229,9 @@ func StartSyncing(s *Streamer, peerId discover.NodeID, po uint8, nn bool) { } } -func RegisterIncomingSyncers(streamer *Streamer, db *DbAccess) { +func RegisterIncomingSyncer(streamer *Streamer, db *DbAccess) { streamer.RegisterIncomingStreamer("SYNC", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { - syncType, _ := parseSyncLabel(t) - switch syncType { - case "LIVE": - return NewIncomingSwarmSyncer(p, nil, nil) - case "HISTORY": - return NewIncomingSwarmSyncer(p, nil, nil) - } - return nil, fmt.Errorf("unknown sync type %q", syncType) + return NewIncomingSwarmSyncer(p, db, nil) }) // stream = fmt.Sprintf("SYNC-%02d-delete", po) // streamer.RegisterIncomingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) { diff --git a/swarm/network/syncer_test.go b/swarm/network/syncer_test.go new file mode 100644 index 0000000000..5962f79583 --- /dev/null +++ b/swarm/network/syncer_test.go @@ -0,0 +1,220 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package network + +import ( + "context" + crand "crypto/rand" + "fmt" + "io" + "math" + "net" + "testing" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/protocols" + "github.com/ethereum/go-ethereum/p2p/simulations" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +var nodeAddrById map[discover.NodeID]*BzzAddr + +func TestSyncerSimulation(t *testing.T) { + testSimulation(t, testSyncBetweenNodes(2, 1, 81000, true, 1)) +} + +func testSyncBetweenNodes(nodes, conns, size int, skipCheck bool, po uint8) func(adapter adapters.NodeAdapter) (*simulations.StepResult, error) { + return func(adapter adapters.NodeAdapter) (*simulations.StepResult, error) { + nodeAddrById = make(map[discover.NodeID]*BzzAddr) + trigger := func(net *simulations.Network) chan discover.NodeID { + triggerC := make(chan discover.NodeID) + ticker := time.NewTicker(500 * time.Millisecond) + go func() { + defer ticker.Stop() + // we are only testing the pivot node (net.Nodes[0]) but simulation needs + // all nodes to pass the check so we trigger each and the check function + // will trivially return true + for i := 1; i < nodes; i++ { + triggerC <- net.Nodes[i].ID() + } + for range ticker.C { + triggerC <- net.Nodes[0].ID() + } + }() + return triggerC + } + + action := func(net *simulations.Network) func(context.Context) error { + // here we distribute chunks of a random file into localstores of nodes 1 to nodes + rrdpa := storage.NewDPA(newRoundRobinStore(localStores[1:]...), storage.NewChunkerParams()) + rrdpa.Start() + // create a retriever dpa for the pivot node + dpacs := storage.NewDpaChunkStore(localStores[0].(*storage.LocalStore), func(chunk *storage.Chunk) error { return delivery.RequestFromPeers(chunk.Key[:], skipCheck) }) + dpa := storage.NewDPA(dpacs, storage.NewChunkerParams()) + dpa.Start() + return func(context.Context) error { + defer rrdpa.Stop() + // upload an actual random file of size size + _, _, err := rrdpa.Store(io.LimitReader(crand.Reader, int64(size)), int64(size)) + if err != nil { + return err + } + // // wait until all chunks stored + // wait() + // // assign the fileHash to a global so that it is available for the check function + // fileHash = hash + // go func() { + // defer dpa.Stop() + // log.Debug(fmt.Sprintf("retrieve %v", fileHash)) + // // start the retrieval on the pivot node - this will spawn retrieve requests for missing chunks + // // we must wait for the peer connections to have started before requesting + // time.Sleep(2 * time.Second) + // n, err := mustReadAll(dpa, fileHash) + // log.Debug(fmt.Sprintf("retrieved %v", fileHash), "read", n, "err", err) + // }() + return nil + } + } + + check := func(net *simulations.Network, dpa *storage.DPA) func(ctx context.Context, id discover.NodeID) (bool, error) { + dbAccesses := make([]*DbAccess, nodes) + + for i := 0; i < nodes; i++ { + dbAccesses[i] = NewDbAccess(localStores[i].(*storage.LocalStore)) + } + return func(ctx context.Context, id discover.NodeID) (bool, error) { + var found, total int + dbAccesses[1].iterator(0, math.MaxUint64, po, func(key storage.Key, index uint64) bool { + _, err := dbAccesses[0].get(key) + if err == nil { + found++ + } + total++ + return true + }) + + // + // if id != net.Nodes[0].ID() { + // return true, nil + // } + select { + case <-ctx.Done(): + return false, ctx.Err() + default: + } + return found == total, nil + // // try to locally retrieve the file to check if retrieve requests have been successful + // log.Warn(fmt.Sprintf("try to locally retrieve %v", fileHash)) + // total, err := mustReadAll(dpa, fileHash) + // if err != nil || total != size { + // log.Warn(fmt.Sprintf("number of bytes read %v/%v (error: %v)", total, size, err)) + // return false, nil + // } + // return true, nil + // node := net.GetNode(id) + // if node == nil { + // return false, fmt.Errorf("unknown node: %s", id) + // } + // client, err := node.Client() + // if err != nil { + // return false, fmt.Errorf("error getting node client: %s", err) + // } + // var response int + // if err := client.Call(&response, "test_haslocal", hash); err != nil { + // return false, fmt.Errorf("error getting bzz_has response: %s", err) + // } + // log.Debug(fmt.Sprintf("node has: %v\n%v", id, response)) + // return response == 0, nil + } + } + + result, err := runSimulation(nodes, conns, "syncer", action, trigger, check, adapter) + if err != nil { + return nil, fmt.Errorf("Setting up simulation failed: %v", err) + } + if result.Error != nil { + return nil, fmt.Errorf("Simulation failed: %s", result.Error) + } + return result, err + } +} + +func newSyncerService(ctx *adapters.ServiceContext) (node.Service, error) { + id := ctx.Config.ID + addr := NewAddrFromNodeID(id) + kad := NewKademlia(addr.Over(), NewKadParams()) + localStore := localStores[nodeCount] + dbAccess := NewDbAccess(localStore.(*storage.LocalStore)) + streamer := NewStreamer(NewDelivery(kad, dbAccess)) + log.Warn("!!!!!!!! Registering syncers") + RegisterIncomingSyncer(streamer, dbAccess) + RegisterOutgoingSyncer(streamer, dbAccess) + addrBytes := addr.Address() + if nodeCount == 0 { + // the delivery service for the pivot node is assigned globally + // so that the simulation action call can use it for the + // swarm enabled dpa + delivery = streamer.delivery + addrBytes[0] = 0x0 + } else { + addrBytes[0] = 0xF0 + } + addr = &BzzAddr{ + OAddr: addrBytes, + UAddr: []byte(discover.NewNode(id, net.IP{127, 0, 0, 1}, 30303, 30303).String()), + } + nodeAddrById[id] = addr + + //else { + // RegisterOutgoingSyncer(streamer, dbAccess) + // } + nodeCount++ + + log.Warn("new service created") + self := &testStreamerService{ + addr: addr, + streamer: streamer, + } + self.run = self.runSyncer + return self, nil +} + +func (b *testStreamerService) runSyncer(p *p2p.Peer, rw p2p.MsgReadWriter) error { + bzzPeer := &bzzPeer{ + Peer: protocols.NewPeer(p, rw, StreamerSpec), + localAddr: b.addr, + BzzAddr: nodeAddrById[p.ID()], + } + b.streamer.delivery.overlay.On(bzzPeer) + defer b.streamer.delivery.overlay.Off(bzzPeer) + go func() { + // each node Subscribes to each other's retrieveRequestStream + // need to wait till an aynchronous process registers the peers in streamer.peers + // that is used by Subscribe + time.Sleep(1 * time.Second) + err := b.streamer.Subscribe(p.ID(), "SYNC", []byte{uint8(1)}, 0, 0, Top, true) + if err != nil { + log.Warn("error in subscribe", "err", err) + } + }() + return b.streamer.Run(bzzPeer) +} diff --git a/swarm/swarm.go b/swarm/swarm.go index 11f5cf25f2..b0a1bc04d3 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -128,8 +128,8 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e dbAccess := network.NewDbAccess(self.lstore) self.streamer = network.NewStreamer(to, dbAccess) - network.RegisterOutgoingSyncers(self.streamer, dbAccess) - network.RegisterIncomingSyncers(self.streamer, dbAccess) + network.RegisterOutgoingSyncer(self.streamer, dbAccess) + network.RegisterIncomingSyncer(self.streamer, dbAccess) self.bzz = network.NewBzz(bzzconfig, to, nil, self.streamer) From 22b4959709080b561f1162c0124554e765809e41 Mon Sep 17 00:00:00 2001 From: zelig Date: Wed, 17 Jan 2018 04:10:15 +0100 Subject: [PATCH 058/210] swarm/storage: NewLocalStoreFromAddr and set bucketCnt to access index --- swarm/storage/dbstore.go | 3 ++- swarm/storage/localstore.go | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/swarm/storage/dbstore.go b/swarm/storage/dbstore.go index d559a7b27b..ec2ac57454 100644 --- a/swarm/storage/dbstore.go +++ b/swarm/storage/dbstore.go @@ -542,10 +542,11 @@ func (s *DbStore) doPut(chunk *Chunk, ikey []byte, index *dpaDBIndex, po uint8) data := encodeData(chunk) s.batch.Put(getDataKey(s.dataIdx, po), data) index.Idx = s.dataIdx + s.bucketCnt[po] = s.dataIdx s.entryCnt++ s.dataIdx++ - s.bucketCnt[po]++ + // s.bucketCnt[po]++ cntKey := make([]byte, 2) cntKey[0] = keyDistanceCnt cntKey[1] = po diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index 32c495ac8a..30b95780f5 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -56,6 +56,19 @@ func NewTestLocalStore(path string) (*LocalStore, error) { return localStore, nil } +func NewTestLocalStoreForAddr(path string, basekey []byte) (*LocalStore, error) { + hasher := MakeHashFunc("SHA3") + dbStore, err := NewDbStore(path, hasher, singletonSwarmDbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) }) + if err != nil { + return nil, err + } + localStore := &LocalStore{ + memStore: NewMemStore(dbStore, singletonSwarmDbCapacity), + DbStore: dbStore, + } + return localStore, nil +} + // LocalStore is itself a chunk store // unsafe, in that the data is not integrity checked func (self *LocalStore) Put(chunk *Chunk) { From debf122a285c6c9d701dcf628eda2e84c263276d Mon Sep 17 00:00:00 2001 From: zelig Date: Wed, 17 Jan 2018 04:53:26 +0100 Subject: [PATCH 059/210] swarm/network: fix RegisterAndConnect test --- swarm/network/hive_test.go | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/swarm/network/hive_test.go b/swarm/network/hive_test.go index 37aebd363e..daa9389291 100644 --- a/swarm/network/hive_test.go +++ b/swarm/network/hive_test.go @@ -43,18 +43,12 @@ func TestRegisterAndConnect(t *testing.T) { pp.Start(s.Server) defer pp.Stop() // retrieve and broadcast - err := s.TestExchanges(p2ptest.Exchange{ - Label: "getPeersMsg message", - Expects: []p2ptest.Expect{ - p2ptest.Expect{ - Code: 2, - Msg: &subPeersMsg{0}, - Peer: id, - }, - }, + err := s.TestDisconnected(&p2ptest.Disconnect{ + Peer: s.IDs[0], + Error: nil, }) - if err != nil { - t.Fatal(err) + if err == nil || err.Error() != "timed out waiting for peers to disconnect" { + t.Fatalf("expected peer to connect") } } From c3f02d0fd489958695842576f3869fe171af4ddf Mon Sep 17 00:00:00 2001 From: zelig Date: Wed, 17 Jan 2018 04:53:47 +0100 Subject: [PATCH 060/210] swarm/network: refactor test code and make all tests pass - only one direction for syncing (works otherwise too after double close on chunk.ReqC was pseudofixed - extract streamer helpers in streamer_common_test - simplify address creation - fix setIncomingSyncer - fix keys for streamer lookup - include Key field universally in msgs - weed out logs --- swarm/network/request_test.go | 288 +-------------------- swarm/network/requests.go | 8 +- swarm/network/streamer.go | 66 +++-- swarm/network/streamer_common_test.go | 349 ++++++++++++++++++++++++++ swarm/network/streamer_test.go | 68 +---- swarm/network/syncer.go | 71 ++---- swarm/network/syncer_test.go | 126 +++------- 7 files changed, 473 insertions(+), 503 deletions(-) create mode 100644 swarm/network/streamer_common_test.go diff --git a/swarm/network/request_test.go b/swarm/network/request_test.go index cd014a4509..12e820b2ad 100644 --- a/swarm/network/request_test.go +++ b/swarm/network/request_test.go @@ -20,15 +20,8 @@ import ( "bytes" "context" crand "crypto/rand" - "errors" - "flag" "fmt" "io" - "io/ioutil" - "math/rand" - "os" - "sync" - "sync/atomic" "testing" "time" @@ -40,7 +33,6 @@ import ( "github.com/ethereum/go-ethereum/p2p/simulations" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" p2ptest "github.com/ethereum/go-ethereum/p2p/testing" - "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm/storage" ) @@ -167,9 +159,11 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { p2ptest.Expect{ Code: 1, Msg: &OfferedHashesMsg{ - HandoverProof: nil, - Hashes: hash, - From: 0, + HandoverProof: &HandoverProof{ + Handover: &Handover{}, + }, + Hashes: hash, + From: 0, // TODO: why is this 32??? To: 32, Key: []byte{}, @@ -305,102 +299,6 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { } -var services = adapters.Services{ - "delivery": newDeliveryService, - "syncer": newSyncerService, -} - -var ( - adapter = flag.String("adapter", "sim", "type of simulation: sim|socket|exec|docker") - loglevel = flag.Int("loglevel", 5, "verbosity of logs") -) - -type roundRobinStore struct { - index uint32 - stores []storage.ChunkStore -} - -func newRoundRobinStore(stores ...storage.ChunkStore) *roundRobinStore { - return &roundRobinStore{ - stores: stores, - } -} - -func (rrs *roundRobinStore) Get(key storage.Key) (*storage.Chunk, error) { - return nil, errors.New("get not well defined on round robin store") -} - -func (rrs *roundRobinStore) Put(chunk *storage.Chunk) { - log.Warn("chunksize", "size", chunk.Size, "sdata", len(chunk.SData)) - i := atomic.AddUint32(&rrs.index, 1) - idx := int(i) % len(rrs.stores) - log.Trace(fmt.Sprintf("put %v into localstore %v", chunk.Key, idx)) - rrs.stores[idx].Put(chunk) -} - -func (rrs *roundRobinStore) Close() { - for _, store := range rrs.stores { - store.Close() - } -} - -func init() { - flag.Parse() - // register the Delivery service which will run as a devp2p - // protocol when using the exec adapter - adapters.RegisterServices(services) - - log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(false)))) -} - -func testSimulation(t *testing.T, simf func(adapters.NodeAdapter) (*simulations.StepResult, error)) { - var err error - var result *simulations.StepResult - startedAt := time.Now() - - switch *adapter { - case "sim": - t.Logf("simadapter") - result, err = simf(adapters.NewSimAdapter(services)) - case "socket": - result, err = simf(adapters.NewSocketAdapter(services)) - case "exec": - baseDir, err0 := ioutil.TempDir("", "swarm-test") - if err0 != nil { - t.Fatal(err0) - } - defer os.RemoveAll(baseDir) - result, err = simf(adapters.NewExecAdapter(baseDir)) - case "docker": - adapter, err0 := adapters.NewDockerAdapter() - if err0 != nil { - t.Fatal(err0) - } - result, err = simf(adapter) - default: - t.Fatal("adapter needs to be one of sim, socket, exec, docker") - } - if err != nil { - t.Fatal(err) - } - t.Logf("Simulation with %d nodes passed in %s", len(result.Passes), result.FinishedAt.Sub(result.StartedAt)) - var min, max time.Duration - var sum int - for _, pass := range result.Passes { - duration := pass.Sub(result.StartedAt) - if sum == 0 || duration < min { - min = duration - } - if duration > max { - max = duration - } - sum += int(duration.Nanoseconds()) - } - t.Logf("Min: %s, Max: %s, Average: %s", min, max, time.Duration(sum/len(result.Passes))*time.Nanosecond) - finishedAt := time.Now() - t.Logf("Setup: %s, shutdown: %s", result.StartedAt.Sub(startedAt), finishedAt.Sub(result.FinishedAt)) -} - func TestDeliveryFromNodes(t *testing.T) { testSimulation(t, testDeliveryFromNodes(2, 1, 8100, true)) testSimulation(t, testDeliveryFromNodes(2, 1, 8100, false)) @@ -408,57 +306,6 @@ func TestDeliveryFromNodes(t *testing.T) { testSimulation(t, testDeliveryFromNodes(3, 1, 8100, false)) } -var ( - delivery *Delivery - localStores []storage.ChunkStore - fileHash storage.Key - nodeCount int -) - -func setLocalStores(n int) (func(), error) { - var datadirs []string - localStores = make([]storage.ChunkStore, n) - var err error - for i := 0; i < n; i++ { - // TODO: remove temp datadir after test - var datadir string - datadir, err = ioutil.TempDir("", "streamer") - if err != nil { - break - } - var localStore *storage.LocalStore - localStore, err = storage.NewTestLocalStore(datadir) - if err != nil { - break - } - datadirs = append(datadirs, datadir) - localStores[i] = localStore - } - teardown := func() { - for _, datadir := range datadirs { - os.RemoveAll(datadir) - } - } - return teardown, err -} - -func mustReadAll(dpa *storage.DPA, hash storage.Key) (int, error) { - r := dpa.Retrieve(fileHash) - buf := make([]byte, 1024) - var n, total int - var err error - for (total == 0 || n > 0) && err == nil { - log.Warn(fmt.Sprintf("reading %v bytes at offset %v", len(buf), total)) - n, err = r.ReadAt(buf, int64(total)) - total += n - } - log.Warn(fmt.Sprintf("read %v bytes at offset %v error %v", len(buf), total, err)) - if err != nil && err != io.EOF { - return total, err - } - return total, nil -} - func testDeliveryFromNodes(nodes, conns, size int, skipCheck bool) func(adapter adapters.NodeAdapter) (*simulations.StepResult, error) { return func(adapter adapters.NodeAdapter) (*simulations.StepResult, error) { trigger := func(net *simulations.Network) chan discover.NodeID { @@ -466,12 +313,7 @@ func testDeliveryFromNodes(nodes, conns, size int, skipCheck bool) func(adapter ticker := time.NewTicker(500 * time.Millisecond) go func() { defer ticker.Stop() - // we are only testing the pivot node (net.Nodes[0]) but simulation needs - // all nodes to pass the check so we trigger each and the check function - // will trivially return true - for i := 1; i < nodes; i++ { - triggerC <- net.Nodes[i].ID() - } + // we are only testing the pivot node (net.Nodes[0]) for range ticker.C { triggerC <- net.Nodes[0].ID() } @@ -523,10 +365,9 @@ func testDeliveryFromNodes(nodes, conns, size int, skipCheck bool) func(adapter default: } // try to locally retrieve the file to check if retrieve requests have been successful - log.Warn(fmt.Sprintf("try to locally retrieve %v", fileHash)) total, err := mustReadAll(dpa, fileHash) + log.Debug(fmt.Sprintf("check if %08x is available locally: number of bytes read %v/%v (error: %v)", fileHash, total, size, err)) if err != nil || total != size { - log.Warn(fmt.Sprintf("number of bytes read %v/%v (error: %v)", total, size, err)) return false, nil } return true, nil @@ -547,7 +388,7 @@ func testDeliveryFromNodes(nodes, conns, size int, skipCheck bool) func(adapter } } - result, err := runSimulation(nodes, conns, "delivery", action, trigger, check, adapter) + result, err := runSimulation(nodes, conns, "delivery", NewAddrFromNodeID, action, trigger, check, adapter) if err != nil { return nil, fmt.Errorf("Setting up simulation failed: %v", err) } @@ -558,83 +399,6 @@ func testDeliveryFromNodes(nodes, conns, size int, skipCheck bool) func(adapter } } -func runSimulation(nodes, conns int, serviceName string, action func(*simulations.Network) func(context.Context) error, trigger func(*simulations.Network) chan discover.NodeID, check func(*simulations.Network, *storage.DPA) func(context.Context, discover.NodeID) (bool, error), adapter adapters.NodeAdapter) (*simulations.StepResult, error) { - // create network - net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{ - ID: "0", - DefaultService: serviceName, - }) - defer net.Shutdown() - // set nodes number of localstores globally available - teardown, err := setLocalStores(nodes) - defer teardown() - if err != nil { - return nil, err - } - ids := make([]discover.NodeID, nodes) - nodeCount = 0 - // start nodes - for i := 0; i < nodes; i++ { - node, err := net.NewNode() - if err != nil { - return nil, fmt.Errorf("error starting node: %s", err) - } - if err := net.Start(node.ID()); err != nil { - return nil, fmt.Errorf("error starting node %s: %s", node.ID().TerminalString(), err) - } - ids[i] = node.ID() - } - - // run a simulation which connects the 10 nodes in a chain - var addrs [][]byte - wg := sync.WaitGroup{} - log.Warn("runSimulation 1") - for i := range ids { - log.Warn("runSimulation 2") - // collect the overlay addresses, to - addrs = append(addrs, ToOverlayAddr(ids[i].Bytes())) - for j := 0; j < conns; j++ { - log.Warn("runSimulation 3") - var k int - if j == 0 { - k = i - 1 - } else { - k = rand.Intn(len(ids)) - } - if i > 0 { - log.Warn("runSimulation 4") - wg.Add(1) - go func(i, k int) { - defer wg.Done() - log.Warn("net.Connect") - net.Connect(ids[i], ids[k]) - }(i, k) - } - } - } - wg.Wait() - - log.Debug(fmt.Sprintf("nodes: %v", len(addrs))) - - // create an only locally retrieving dpa for the pivot node to test - // if retriee requests have arrived - dpa := storage.NewDPA(localStores[0], storage.NewChunkerParams()) - dpa.Start() - defer dpa.Stop() - timeout := 300 * time.Second - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - result := simulations.NewSimulation(net).Run(ctx, &simulations.Step{ - Action: action(net), - Trigger: trigger(net), - Expect: &simulations.Expectation{ - Nodes: ids, - Check: check(net, dpa), - }, - }) - return result, nil -} - // newDeliveryService func newDeliveryService(ctx *adapters.ServiceContext) (node.Service, error) { id := ctx.Config.ID @@ -649,49 +413,15 @@ func newDeliveryService(ctx *adapters.ServiceContext) (node.Service, error) { // swarm enabled dpa delivery = streamer.delivery } - nodeCount++ - - log.Warn("new service created") self := &testStreamerService{ addr: addr, streamer: streamer, } self.run = self.runDelivery + nodeCount++ return self, nil } -type testStreamerService struct { - addr *BzzAddr - streamer *Streamer - run func(p *p2p.Peer, rw p2p.MsgReadWriter) error -} - -func (tds *testStreamerService) Protocols() []p2p.Protocol { - log.Warn("Protocols function", "run", tds.run) - return []p2p.Protocol{ - { - Name: StreamerSpec.Name, - Version: StreamerSpec.Version, - Length: StreamerSpec.Length(), - Run: tds.run, - // NodeInfo: , - // PeerInfo: , - }, - } -} - -func (b *testStreamerService) APIs() []rpc.API { - return []rpc.API{} -} - -func (b *testStreamerService) Start(server *p2p.Server) error { - return nil -} - -func (b *testStreamerService) Stop() error { - return nil -} - func (b *testStreamerService) runDelivery(p *p2p.Peer, rw p2p.MsgReadWriter) error { bzzPeer := &bzzPeer{ Peer: protocols.NewPeer(p, rw, StreamerSpec), diff --git a/swarm/network/requests.go b/swarm/network/requests.go index 7cc0a47946..3495cfceb9 100644 --- a/swarm/network/requests.go +++ b/swarm/network/requests.go @@ -168,8 +168,12 @@ func (self *Delivery) processReceivedChunks() { continue } chunk.SData = req.SData - self.dbAccess.put(chunk) - close(chunk.ReqC) + select { + case <-chunk.ReqC: + default: + self.dbAccess.put(chunk) + close(chunk.ReqC) + } } } diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go index 3c367f7b6f..66abf82f3a 100644 --- a/swarm/network/streamer.go +++ b/swarm/network/streamer.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "math" "sync" "github.com/ethereum/go-ethereum/log" @@ -109,6 +110,14 @@ func (self WantedHashesMsg) String() string { return fmt.Sprintf("Stream '%v', Want: %x, Next: [%v-%v]", self.Stream, self.Want, self.From, self.To) } +func keyToString(key []byte) string { + l := len(key) + if l == 0 { + return "" + } + return fmt.Sprintf("%s-%d", string(key[:l-1]), uint8(key[l-1])) +} + // Streamer registry for outgoing and incoming streamer constructors type Streamer struct { incomingLock sync.RWMutex @@ -187,6 +196,7 @@ type outgoingStreamer struct { priority uint8 currentBatch []byte stream string + key []byte } // OutgoingStreamer interface for outgoing peer Streamer @@ -200,6 +210,8 @@ type incomingStreamer struct { priority uint8 sessionAt uint64 live bool + stream string + key []byte quit chan struct{} next chan struct{} } @@ -280,26 +292,30 @@ func (self *StreamerPeer) getIncomingStreamer(s string) (*incomingStreamer, erro return streamer, nil } -func (self *StreamerPeer) setOutgoingStreamer(s string, o OutgoingStreamer, priority uint8) (*outgoingStreamer, error) { +func (self *StreamerPeer) setOutgoingStreamer(s string, key []byte, o OutgoingStreamer, priority uint8) (*outgoingStreamer, error) { self.outgoingLock.Lock() defer self.outgoingLock.Unlock() - if self.outgoing[s] != nil { - return nil, fmt.Errorf("stream %v already registered", s) + sk := s + keyToString(key) + if self.outgoing[sk] != nil { + return nil, fmt.Errorf("stream %v already registered", sk) } os := &outgoingStreamer{ OutgoingStreamer: o, priority: priority, stream: s, + key: key, } - self.outgoing[s] = os + self.outgoing[sk] = os return os, nil } -func (self *StreamerPeer) setIncomingStreamer(s string, i IncomingStreamer, priority uint8, live bool) error { +func (self *StreamerPeer) setIncomingStreamer(s string, key []byte, i IncomingStreamer, priority uint8, live bool) error { self.incomingLock.Lock() defer self.incomingLock.Unlock() - if self.incoming[s] != nil { - return fmt.Errorf("stream %v already registered", s) + + sk := s + keyToString(key) + if self.incoming[sk] != nil { + return fmt.Errorf("stream %v already registered", sk) } next := make(chan struct{}, 1) // var intervals *Intervals @@ -307,12 +323,14 @@ func (self *StreamerPeer) setIncomingStreamer(s string, i IncomingStreamer, prio // key := s + self.ID().String() // intervals = NewIntervals(key, self.streamer) // } - self.incoming[s] = &incomingStreamer{ + self.incoming[sk] = &incomingStreamer{ IncomingStreamer: i, // intervals: intervals, live: live, priority: priority, next: next, + stream: s, + key: key, } next <- struct{}{} // this is to allow wantedKeysMsg before first batch arrives return nil @@ -330,6 +348,8 @@ func (self *incomingStreamer) nextBatch(from uint64) (nextFrom uint64, nextTo ui nextFrom = from } else if from >= self.sessionAt { // history sync complete intervals = nil + nextFrom = from + nextTo = math.MaxUint64 } else if len(intervals) > 2 && from >= intervals[2] { // filled a gap in the intervals intervals = append(intervals[:1], intervals[3:]...) nextFrom = intervals[1] @@ -349,7 +369,6 @@ func (self *incomingStreamer) nextBatch(from uint64) (nextFrom uint64, nextTo ui // Subscribe initiates the streamer func (self *Streamer) Subscribe(peerId discover.NodeID, s string, t []byte, from, to uint64, priority uint8, live bool) error { - log.Warn("!!!!!! Subscribe ", "peer", peerId) f, err := self.GetIncomingStreamer(s) if err != nil { return err @@ -364,18 +383,21 @@ func (self *Streamer) Subscribe(peerId discover.NodeID, s string, t []byte, from if err != nil { return err } - err = peer.setIncomingStreamer(s+string(t), is, priority, live) + err = peer.setIncomingStreamer(s, t, is, priority, live) if err != nil { return err } msg := &SubscribeMsg{ - Stream: s, - Key: t, + Stream: s, + Key: t, + // Live: live, From: from, To: to, Priority: priority, } + log.Debug("Subscribe ", "peer", peerId, "stream", s, "key", t, "from", from, "to", to) + peer.SendPriority(msg, priority) return nil } @@ -389,11 +411,11 @@ func (self *StreamerPeer) handleSubscribeMsg(req *SubscribeMsg) error { if err != nil { return err } - key := req.Stream + string(req.Key) - os, err := self.setOutgoingStreamer(key, s, req.Priority) + os, err := self.setOutgoingStreamer(req.Stream, req.Key, s, req.Priority) if err != nil { return nil } + log.Debug("received subscription", "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To) go self.SendOfferedHashes(os, req.From, req.To) return nil } @@ -401,7 +423,9 @@ func (self *StreamerPeer) handleSubscribeMsg(req *SubscribeMsg) error { // handleOfferedHashesMsg protocol msg handler calls the incoming streamer interface // Filter method func (self *StreamerPeer) handleOfferedHashesMsg(req *OfferedHashesMsg) error { - s, err := self.getIncomingStreamer(req.Stream) + sk := req.Stream + sk += keyToString(req.Key) + s, err := self.getIncomingStreamer(sk) if err != nil { return err } @@ -440,11 +464,14 @@ func (self *StreamerPeer) handleOfferedHashesMsg(req *OfferedHashesMsg) error { s.sessionAt = req.From } from, to := s.nextBatch(req.To) + log.Debug("received batch", "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To) if from == to { return nil } + msg := &WantedHashesMsg{ Stream: req.Stream, + Key: req.Key, Want: want.Bytes(), From: from, To: to, @@ -455,6 +482,7 @@ func (self *StreamerPeer) handleOfferedHashesMsg(req *OfferedHashesMsg) error { case <-s.quit: return } + log.Debug("want batch", "stream", msg.Stream, "Key", msg.Key, "from", msg.From, "to", msg.To) self.SendPriority(msg, s.priority) }() return nil @@ -464,8 +492,10 @@ func (self *StreamerPeer) handleOfferedHashesMsg(req *OfferedHashesMsg) error { // * sends the next batch of unsynced keys // * sends the actual data chunks as per WantedHashesMsg func (self *StreamerPeer) handleWantedHashesMsg(req *WantedHashesMsg) error { - s, err := self.getOutgoingStreamer(req.Stream) + log.Debug("received wanted batch", "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To) + s, err := self.getOutgoingStreamer(req.Stream + keyToString(req.Key)) if err != nil { + log.Debug(err.Error()) return err } hashes := s.currentBatch @@ -534,9 +564,9 @@ func (self *StreamerPeer) SendOfferedHashes(s *outgoingStreamer, f, t uint64) er From: from, To: to, Stream: s.stream, - // TODO: use real key here - Key: []byte{}, + Key: s.key, } + log.Debug("Swarm syncer offer batch", "stream", s.stream, "key", s.key, "len", len(hashes), "from", from, "to", to) return self.SendPriority(msg, s.priority) } diff --git a/swarm/network/streamer_common_test.go b/swarm/network/streamer_common_test.go new file mode 100644 index 0000000000..7690ed75c8 --- /dev/null +++ b/swarm/network/streamer_common_test.go @@ -0,0 +1,349 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package network + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "io/ioutil" + "math/rand" + "os" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/protocols" + "github.com/ethereum/go-ethereum/p2p/simulations" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + p2ptest "github.com/ethereum/go-ethereum/p2p/testing" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +var services = adapters.Services{ + "delivery": newDeliveryService, + "syncer": newSyncerService, +} + +var ( + adapter = flag.String("adapter", "sim", "type of simulation: sim|socket|exec|docker") + loglevel = flag.Int("loglevel", 2, "verbosity of logs") +) + +func init() { + flag.Parse() + // register the Delivery service which will run as a devp2p + // protocol when using the exec adapter + adapters.RegisterServices(services) + + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(false)))) +} + +var ( + delivery *Delivery + localStores []storage.ChunkStore + addrs []Addr + fileHash storage.Key + nodeCount int +) + +func setLocalStores(addrs ...Addr) (func(), error) { + var datadirs []string + localStores = make([]storage.ChunkStore, len(addrs)) + var err error + for i, addr := range addrs { + // TODO: remove temp datadir after test + var datadir string + datadir, err = ioutil.TempDir("", "streamer") + if err != nil { + break + } + var localStore *storage.LocalStore + localStore, err = storage.NewTestLocalStoreForAddr(datadir, addr.Over()) + if err != nil { + break + } + datadirs = append(datadirs, datadir) + localStores[i] = localStore + } + teardown := func() { + for _, datadir := range datadirs { + os.RemoveAll(datadir) + } + } + return teardown, err +} + +func mustReadAll(dpa *storage.DPA, hash storage.Key) (int, error) { + r := dpa.Retrieve(fileHash) + buf := make([]byte, 1024) + var n, total int + var err error + for (total == 0 || n > 0) && err == nil { + n, err = r.ReadAt(buf, int64(total)) + total += n + } + if err != nil && err != io.EOF { + return total, err + } + return total, nil +} + +func testSimulation(t *testing.T, simf func(adapters.NodeAdapter) (*simulations.StepResult, error)) { + var err error + var result *simulations.StepResult + startedAt := time.Now() + + switch *adapter { + case "sim": + t.Logf("simadapter") + result, err = simf(adapters.NewSimAdapter(services)) + case "socket": + result, err = simf(adapters.NewSocketAdapter(services)) + case "exec": + baseDir, err0 := ioutil.TempDir("", "swarm-test") + if err0 != nil { + t.Fatal(err0) + } + defer os.RemoveAll(baseDir) + result, err = simf(adapters.NewExecAdapter(baseDir)) + case "docker": + adapter, err0 := adapters.NewDockerAdapter() + if err0 != nil { + t.Fatal(err0) + } + result, err = simf(adapter) + default: + t.Fatal("adapter needs to be one of sim, socket, exec, docker") + } + if err != nil { + t.Fatal(err) + } + t.Logf("Simulation with %d nodes passed in %s", len(result.Passes), result.FinishedAt.Sub(result.StartedAt)) + var min, max time.Duration + var sum int + for _, pass := range result.Passes { + duration := pass.Sub(result.StartedAt) + if sum == 0 || duration < min { + min = duration + } + if duration > max { + max = duration + } + sum += int(duration.Nanoseconds()) + } + t.Logf("Min: %s, Max: %s, Average: %s", min, max, time.Duration(sum/len(result.Passes))*time.Nanosecond) + finishedAt := time.Now() + t.Logf("Setup: %s, shutdown: %s", result.StartedAt.Sub(startedAt), finishedAt.Sub(result.FinishedAt)) +} + +func runSimulation(nodes, conns int, serviceName string, toAddr func(discover.NodeID) *BzzAddr, action func(*simulations.Network) func(context.Context) error, trigger func(*simulations.Network) chan discover.NodeID, check func(*simulations.Network, *storage.DPA) func(context.Context, discover.NodeID) (bool, error), adapter adapters.NodeAdapter) (*simulations.StepResult, error) { + // create network + net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{ + ID: "0", + DefaultService: serviceName, + }) + defer net.Shutdown() + ids := make([]discover.NodeID, nodes) + nodeCount = 0 + addrs = make([]Addr, nodes) + // start nodes + for i := 0; i < nodes; i++ { + node, err := net.NewNode() + if err != nil { + return nil, fmt.Errorf("error creating node: %s", err) + } + ids[i] = node.ID() + addrs[i] = toAddr(ids[i]) + } + // set nodes number of localstores globally available + teardown, err := setLocalStores(addrs...) + defer teardown() + if err != nil { + return nil, err + } + + for i := 0; i < nodes; i++ { + if err := net.Start(ids[i]); err != nil { + return nil, fmt.Errorf("error starting node %s: %s", ids[i].TerminalString(), err) + } + } + + // run a simulation which connects the 10 nodes in a chain + wg := sync.WaitGroup{} + for i := range ids { + // collect the overlay addresses, to + for j := 0; j < conns; j++ { + var k int + if j == 0 { + k = i - 1 + } else { + k = rand.Intn(len(ids)) + } + if i > 0 { + wg.Add(1) + go func(i, k int) { + defer wg.Done() + net.Connect(ids[i], ids[k]) + }(i, k) + } + } + } + wg.Wait() + + log.Debug(fmt.Sprintf("nodes: %v", len(addrs))) + + // create an only locally retrieving dpa for the pivot node to test + // if retriee requests have arrived + dpa := storage.NewDPA(localStores[0], storage.NewChunkerParams()) + dpa.Start() + defer dpa.Stop() + timeout := 300 * time.Second + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + result := simulations.NewSimulation(net).Run(ctx, &simulations.Step{ + Action: action(net), + Trigger: trigger(net), + Expect: &simulations.Expectation{ + Nodes: ids[0:1], + Check: check(net, dpa), + }, + }) + return result, nil +} + +func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Streamer, *storage.LocalStore, func(), error) { + // setup + addr := RandomAddr() // tested peers peer address + to := NewKademlia(addr.OAddr, NewKadParams()) + + // temp datadir + datadir, err := ioutil.TempDir("", "streamer") + if err != nil { + return nil, nil, nil, func() {}, err + } + teardown := func() { + os.RemoveAll(datadir) + } + + localStore, err := storage.NewTestLocalStoreForAddr(datadir, addr.Over()) + if err != nil { + return nil, nil, nil, teardown, err + } + + dbAccess := NewDbAccess(localStore) + delivery := NewDelivery(to, dbAccess) + streamer := NewStreamer(delivery) + run := func(p *p2p.Peer, rw p2p.MsgReadWriter) error { + bzzPeer := &bzzPeer{ + Peer: protocols.NewPeer(p, rw, StreamerSpec), + localAddr: addr, + BzzAddr: NewAddrFromNodeID(p.ID()), + } + to.On(bzzPeer) + return streamer.Run(bzzPeer) + } + protocolTester := p2ptest.NewProtocolTester(t, NewNodeIDFromAddr(addr), 1, run) + + err = waitForPeers(streamer, 1*time.Second) + if err != nil { + return nil, nil, nil, nil, errors.New("timeout: peer is not created") + } + + return protocolTester, streamer, localStore, teardown, nil +} + +type roundRobinStore struct { + index uint32 + stores []storage.ChunkStore +} + +func newRoundRobinStore(stores ...storage.ChunkStore) *roundRobinStore { + return &roundRobinStore{ + stores: stores, + } +} + +func (rrs *roundRobinStore) Get(key storage.Key) (*storage.Chunk, error) { + return nil, errors.New("get not well defined on round robin store") +} + +func (rrs *roundRobinStore) Put(chunk *storage.Chunk) { + i := atomic.AddUint32(&rrs.index, 1) + idx := int(i) % len(rrs.stores) + rrs.stores[idx].Put(chunk) +} + +func (rrs *roundRobinStore) Close() { + for _, store := range rrs.stores { + store.Close() + } +} + +func waitForPeers(streamer *Streamer, timeout time.Duration) error { + ticker := time.NewTicker(10 * time.Millisecond) + timeoutTimer := time.NewTimer(timeout) + for { + select { + case <-ticker.C: + if len(streamer.peers) > 0 { + return nil + } + case <-timeoutTimer.C: + return errors.New("timeout") + } + } +} + +type testStreamerService struct { + index int + addr *BzzAddr + streamer *Streamer + run func(p *p2p.Peer, rw p2p.MsgReadWriter) error +} + +func (tds *testStreamerService) Protocols() []p2p.Protocol { + return []p2p.Protocol{ + { + Name: StreamerSpec.Name, + Version: StreamerSpec.Version, + Length: StreamerSpec.Length(), + Run: tds.run, + // NodeInfo: , + // PeerInfo: , + }, + } +} + +func (b *testStreamerService) APIs() []rpc.API { + return []rpc.API{} +} + +func (b *testStreamerService) Start(server *p2p.Server) error { + return nil +} + +func (b *testStreamerService) Stop() error { + return nil +} diff --git a/swarm/network/streamer_test.go b/swarm/network/streamer_test.go index 86ba965071..447713a33d 100644 --- a/swarm/network/streamer_test.go +++ b/swarm/network/streamer_test.go @@ -18,65 +18,13 @@ package network import ( "bytes" - "errors" - "io/ioutil" - "os" "testing" "time" "github.com/ethereum/go-ethereum/crypto/sha3" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/protocols" p2ptest "github.com/ethereum/go-ethereum/p2p/testing" - "github.com/ethereum/go-ethereum/swarm/storage" ) -// -// func init() { -// log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlWarn, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) -// } - -func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Streamer, *storage.LocalStore, func(), error) { - // setup - addr := RandomAddr() // tested peers peer address - to := NewKademlia(addr.OAddr, NewKadParams()) - - // temp datadir - datadir, err := ioutil.TempDir("", "streamer") - if err != nil { - return nil, nil, nil, func() {}, err - } - teardown := func() { - os.RemoveAll(datadir) - } - - localStore, err := storage.NewTestLocalStore(datadir) - if err != nil { - return nil, nil, nil, teardown, err - } - - dbAccess := NewDbAccess(localStore) - delivery := NewDelivery(to, dbAccess) - streamer := NewStreamer(delivery) - run := func(p *p2p.Peer, rw p2p.MsgReadWriter) error { - bzzPeer := &bzzPeer{ - Peer: protocols.NewPeer(p, rw, StreamerSpec), - localAddr: addr, - BzzAddr: NewAddrFromNodeID(p.ID()), - } - to.On(bzzPeer) - return streamer.Run(bzzPeer) - } - protocolTester := p2ptest.NewProtocolTester(t, NewNodeIDFromAddr(addr), 1, run) - - err = waitForPeers(streamer, 1*time.Second) - if err != nil { - return nil, nil, nil, nil, errors.New("timeout: peer is not created") - } - - return protocolTester, streamer, localStore, teardown, nil -} - func TestStreamerSubscribe(t *testing.T) { tester, streamer, _, teardown, err := newStreamerTester(t) defer teardown() @@ -214,6 +162,7 @@ func TestStreamerUpstreamSubscribeMsgExchange(t *testing.T) { Code: 1, Msg: &OfferedHashesMsg{ Stream: "foo", + Key: []byte{}, HandoverProof: &HandoverProof{ Handover: &Handover{}, }, @@ -329,18 +278,3 @@ func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) { } } - -func waitForPeers(streamer *Streamer, timeout time.Duration) error { - ticker := time.NewTicker(10 * time.Millisecond) - timeoutTimer := time.NewTimer(timeout) - for { - select { - case <-ticker.C: - if len(streamer.peers) > 0 { - return nil - } - case <-timeoutTimer.C: - return errors.New("timeout") - } - } -} diff --git a/swarm/network/syncer.go b/swarm/network/syncer.go index db2e551826..ec8a42808e 100644 --- a/swarm/network/syncer.go +++ b/swarm/network/syncer.go @@ -25,12 +25,12 @@ import ( "time" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/swarm/storage" ) const ( - batchSize = 128 + batchSize = 2 + // batchSize = 128 ) // wrapper of db-s to provide mockable custom local chunk store access to syncer @@ -60,7 +60,6 @@ func (self *DbAccess) iterator(from uint64, to uint64, po uint8, f func(storage. // to obtain the chunks from key or request db entry only func (self *DbAccess) getOrCreateRequest(key storage.Key) (*storage.Chunk, bool) { - log.Warn("getOrCreateRequest", "self", self) return self.loc.GetOrCreateRequest(key) } @@ -100,17 +99,9 @@ const maxPO = 32 func RegisterOutgoingSyncer(streamer *Streamer, db *DbAccess) { streamer.RegisterOutgoingStreamer("SYNC", func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) { - syncType, po := parseSyncLabel(t) + po := uint8(t[0]) // TODO: make this work for HISTORY too - syncType = "LIVE" - switch syncType { - case "LIVE": - return NewOutgoingSwarmSyncer(true, po, db) - case "HISTORY": - return NewOutgoingSwarmSyncer(false, po, db) - default: - return nil, errors.New("invalid sync type") - } + return NewOutgoingSwarmSyncer(false, po, db) }) // streamer.RegisterOutgoingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) { // return NewOutgoingProvableSwarmSyncer(po, db) @@ -133,10 +124,9 @@ func (self *OutgoingSwarmSyncer) SetNextBatch(from, to uint64) ([]byte, uint64, if from == 0 { from = self.start } - if to <= from { + if to <= from || from >= self.sessionAt { to = math.MaxUint64 } - log.Warn("!!!!!!!!!!!!! setNextBatch", "from", from, "to", to, "currentStoreCount", self.db.currentBucketStorageIndex(1)) ticker := time.NewTicker(10 * time.Millisecond) defer ticker.Stop() for range ticker.C { @@ -154,7 +144,7 @@ func (self *OutgoingSwarmSyncer) SetNextBatch(from, to uint64) ([]byte, uint64, } } - log.Debug("Swarm batch", "po", self.po, "len", i, "from", from, "to", to) + log.Debug("Swarm syncer offer batch", "po", self.po, "len", i, "from", from, "to", to, "current store count", self.db.currentBucketStorageIndex(self.po)) return batch, from, to + 1, nil, nil } @@ -204,40 +194,24 @@ func NewIncomingSwarmSyncer(p Peer, dbAccess *DbAccess, chunker storage.Chunker) // return self // } -func newSyncLabel(typ string, po uint8) []byte { - t := []byte(typ) - t = append(t, byte(po)) - return t -} - -func parseSyncLabel(t []byte) (string, uint8) { - l := len(t) - 1 - return string(t[:l]), uint8(t[l]) -} - -// StartSyncing is called on the StreamerPeer to start the syncing process -// the idea is that it is called only after kademlia is close to healthy -func StartSyncing(s *Streamer, peerId discover.NodeID, po uint8, nn bool) { - lastPO := po - if nn { - lastPO = maxPO - } - - for i := po; i <= lastPO; i++ { - s.Subscribe(peerId, "SYNC", newSyncLabel("LIVE", po), 0, 0, High, true) - s.Subscribe(peerId, "SYNC", newSyncLabel("HISTORY", po), 0, 0, Mid, false) - } -} +// // StartSyncing is called on the StreamerPeer to start the syncing process +// // the idea is that it is called only after kademlia is close to healthy +// func StartSyncing(s *Streamer, peerId discover.NodeID, po uint8, nn bool) { +// lastPO := po +// if nn { +// lastPO = maxPO +// } +// +// for i := po; i <= lastPO; i++ { +// s.Subscribe(peerId, "SYNC", newSyncLabel("LIVE", po), 0, 0, High, true) +// s.Subscribe(peerId, "SYNC", newSyncLabel("HISTORY", po), 0, 0, Mid, false) +// } +// } func RegisterIncomingSyncer(streamer *Streamer, db *DbAccess) { streamer.RegisterIncomingStreamer("SYNC", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { return NewIncomingSwarmSyncer(p, db, nil) }) - // stream = fmt.Sprintf("SYNC-%02d-delete", po) - // streamer.RegisterIncomingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) { - // intervals := loadIntervals(p, po, true) - // return NewIncomingSwarmSyncer(po, Mid, sessionAt, intervals, p) - // }) } // NeedData @@ -287,9 +261,10 @@ func (self *IncomingSwarmSyncer) TakeoverProof(s string, from uint64, hashes []b self.end += uint64(len(hashes)) / HashSize takeover := &Takeover{ Stream: s, - Start: self.start, - End: self.end, - Root: root, + // Key: self.Key, + Start: self.start, + End: self.end, + Root: root, } // serialise and sign return &TakeoverProof{ diff --git a/swarm/network/syncer_test.go b/swarm/network/syncer_test.go index 5962f79583..893be367f5 100644 --- a/swarm/network/syncer_test.go +++ b/swarm/network/syncer_test.go @@ -22,7 +22,6 @@ import ( "fmt" "io" "math" - "net" "testing" "time" @@ -36,26 +35,19 @@ import ( "github.com/ethereum/go-ethereum/swarm/storage" ) -var nodeAddrById map[discover.NodeID]*BzzAddr - func TestSyncerSimulation(t *testing.T) { testSimulation(t, testSyncBetweenNodes(2, 1, 81000, true, 1)) + testSimulation(t, testSyncBetweenNodes(3, 1, 81000, true, 1)) } func testSyncBetweenNodes(nodes, conns, size int, skipCheck bool, po uint8) func(adapter adapters.NodeAdapter) (*simulations.StepResult, error) { return func(adapter adapters.NodeAdapter) (*simulations.StepResult, error) { - nodeAddrById = make(map[discover.NodeID]*BzzAddr) trigger := func(net *simulations.Network) chan discover.NodeID { triggerC := make(chan discover.NodeID) ticker := time.NewTicker(500 * time.Millisecond) go func() { defer ticker.Stop() - // we are only testing the pivot node (net.Nodes[0]) but simulation needs - // all nodes to pass the check so we trigger each and the check function - // will trivially return true - for i := 1; i < nodes; i++ { - triggerC <- net.Nodes[i].ID() - } + // we are only testing the pivot node (net.Nodes[0]) for range ticker.C { triggerC <- net.Nodes[0].ID() } @@ -68,29 +60,15 @@ func testSyncBetweenNodes(nodes, conns, size int, skipCheck bool, po uint8) func rrdpa := storage.NewDPA(newRoundRobinStore(localStores[1:]...), storage.NewChunkerParams()) rrdpa.Start() // create a retriever dpa for the pivot node - dpacs := storage.NewDpaChunkStore(localStores[0].(*storage.LocalStore), func(chunk *storage.Chunk) error { return delivery.RequestFromPeers(chunk.Key[:], skipCheck) }) - dpa := storage.NewDPA(dpacs, storage.NewChunkerParams()) - dpa.Start() return func(context.Context) error { defer rrdpa.Stop() // upload an actual random file of size size - _, _, err := rrdpa.Store(io.LimitReader(crand.Reader, int64(size)), int64(size)) + _, wait, err := rrdpa.Store(io.LimitReader(crand.Reader, int64(size)), int64(size)) if err != nil { return err } - // // wait until all chunks stored - // wait() - // // assign the fileHash to a global so that it is available for the check function - // fileHash = hash - // go func() { - // defer dpa.Stop() - // log.Debug(fmt.Sprintf("retrieve %v", fileHash)) - // // start the retrieval on the pivot node - this will spawn retrieve requests for missing chunks - // // we must wait for the peer connections to have started before requesting - // time.Sleep(2 * time.Second) - // n, err := mustReadAll(dpa, fileHash) - // log.Debug(fmt.Sprintf("retrieved %v", fileHash), "read", n, "err", err) - // }() + // wait until all chunks stored + wait() return nil } } @@ -102,52 +80,37 @@ func testSyncBetweenNodes(nodes, conns, size int, skipCheck bool, po uint8) func dbAccesses[i] = NewDbAccess(localStores[i].(*storage.LocalStore)) } return func(ctx context.Context, id discover.NodeID) (bool, error) { - var found, total int - dbAccesses[1].iterator(0, math.MaxUint64, po, func(key storage.Key, index uint64) bool { - _, err := dbAccesses[0].get(key) - if err == nil { - found++ - } - total++ - return true - }) - - // - // if id != net.Nodes[0].ID() { - // return true, nil - // } + if id != net.Nodes[0].ID() { + return true, nil + } select { case <-ctx.Done(): return false, ctx.Err() default: } + + var found, total int + for i := 1; i < nodes; i++ { + dbAccesses[i].iterator(0, math.MaxUint64, po, func(key storage.Key, index uint64) bool { + _, err := dbAccesses[0].get(key) + if err == nil { + found++ + } + total++ + return true + }) + } + log.Debug("sync check", "bin", po, "found", found, "total", total) return found == total, nil - // // try to locally retrieve the file to check if retrieve requests have been successful - // log.Warn(fmt.Sprintf("try to locally retrieve %v", fileHash)) - // total, err := mustReadAll(dpa, fileHash) - // if err != nil || total != size { - // log.Warn(fmt.Sprintf("number of bytes read %v/%v (error: %v)", total, size, err)) - // return false, nil - // } - // return true, nil - // node := net.GetNode(id) - // if node == nil { - // return false, fmt.Errorf("unknown node: %s", id) - // } - // client, err := node.Client() - // if err != nil { - // return false, fmt.Errorf("error getting node client: %s", err) - // } - // var response int - // if err := client.Call(&response, "test_haslocal", hash); err != nil { - // return false, fmt.Errorf("error getting bzz_has response: %s", err) - // } - // log.Debug(fmt.Sprintf("node has: %v\n%v", id, response)) - // return response == 0, nil } } + toAddr := func(id discover.NodeID) *BzzAddr { + addr := NewAddrFromNodeID(id) + addr.OAddr[0] = byte(0) + return addr + } - result, err := runSimulation(nodes, conns, "syncer", action, trigger, check, adapter) + result, err := runSimulation(nodes, conns, "syncer", toAddr, action, trigger, check, adapter) if err != nil { return nil, fmt.Errorf("Setting up simulation failed: %v", err) } @@ -161,60 +124,45 @@ func testSyncBetweenNodes(nodes, conns, size int, skipCheck bool, po uint8) func func newSyncerService(ctx *adapters.ServiceContext) (node.Service, error) { id := ctx.Config.ID addr := NewAddrFromNodeID(id) + // for the test we make all peers share 8 bits so that syncing full bins make sense + addr.OAddr[0] = byte(0) kad := NewKademlia(addr.Over(), NewKadParams()) localStore := localStores[nodeCount] dbAccess := NewDbAccess(localStore.(*storage.LocalStore)) streamer := NewStreamer(NewDelivery(kad, dbAccess)) - log.Warn("!!!!!!!! Registering syncers") RegisterIncomingSyncer(streamer, dbAccess) RegisterOutgoingSyncer(streamer, dbAccess) - addrBytes := addr.Address() - if nodeCount == 0 { - // the delivery service for the pivot node is assigned globally - // so that the simulation action call can use it for the - // swarm enabled dpa - delivery = streamer.delivery - addrBytes[0] = 0x0 - } else { - addrBytes[0] = 0xF0 - } - addr = &BzzAddr{ - OAddr: addrBytes, - UAddr: []byte(discover.NewNode(id, net.IP{127, 0, 0, 1}, 30303, 30303).String()), - } - nodeAddrById[id] = addr - //else { - // RegisterOutgoingSyncer(streamer, dbAccess) - // } - nodeCount++ - - log.Warn("new service created") self := &testStreamerService{ + index: nodeCount, addr: addr, streamer: streamer, } self.run = self.runSyncer + nodeCount++ return self, nil } func (b *testStreamerService) runSyncer(p *p2p.Peer, rw p2p.MsgReadWriter) error { + addr := NewAddrFromNodeID(p.ID()) + addr.OAddr[0] = byte(0) bzzPeer := &bzzPeer{ Peer: protocols.NewPeer(p, rw, StreamerSpec), localAddr: b.addr, - BzzAddr: nodeAddrById[p.ID()], + BzzAddr: addr, } b.streamer.delivery.overlay.On(bzzPeer) defer b.streamer.delivery.overlay.Off(bzzPeer) + // if len(addr) > b.index+1 && bytes.Equal(addrs[b.index+1], addr) { go func() { // each node Subscribes to each other's retrieveRequestStream // need to wait till an aynchronous process registers the peers in streamer.peers // that is used by Subscribe time.Sleep(1 * time.Second) - err := b.streamer.Subscribe(p.ID(), "SYNC", []byte{uint8(1)}, 0, 0, Top, true) - if err != nil { + if err := b.streamer.Subscribe(p.ID(), "SYNC", []byte{uint8(1)}, 0, 0, Top, false); err != nil { log.Warn("error in subscribe", "err", err) } }() + // } return b.streamer.Run(bzzPeer) } From e9c22d4da0bfd05b5de57b716922adaf9e936dae Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Wed, 17 Jan 2018 12:33:44 +0100 Subject: [PATCH 061/210] swarm/storage: Fix all tests --- swarm/storage/dbstore.go | 1 - swarm/storage/dbstore_test.go | 60 +++++++++++++++++----------------- swarm/storage/localstore.go | 13 ++++++++ swarm/storage/resource_test.go | 8 ++--- 4 files changed, 45 insertions(+), 37 deletions(-) diff --git a/swarm/storage/dbstore.go b/swarm/storage/dbstore.go index 59a87069c2..60acf4871b 100644 --- a/swarm/storage/dbstore.go +++ b/swarm/storage/dbstore.go @@ -574,7 +574,6 @@ func (s *DbStore) doPut(chunk *Chunk, ikey []byte, index *dpaDBIndex, po uint8) s.entryCnt++ s.dataIdx++ - // s.bucketCnt[po]++ cntKey := make([]byte, 2) cntKey[0] = keyDistanceCnt cntKey[1] = po diff --git a/swarm/storage/dbstore_test.go b/swarm/storage/dbstore_test.go index 3cdfb65247..3ef83a8177 100644 --- a/swarm/storage/dbstore_test.go +++ b/swarm/storage/dbstore_test.go @@ -253,33 +253,33 @@ func testMockDbStore(l int64, branches int64, t *testing.T) { } -func TestMockDbStore128_0x1000000(t *testing.T) { - testMockDbStore(0x1000000, 128, t) -} - -func TestMockDbStore128_10000_(t *testing.T) { - testMockDbStore(10000, 128, t) -} - -func TestMockDbStore128_1000_(t *testing.T) { - testMockDbStore(1000, 128, t) -} - -func TestMockDbStore128_100_(t *testing.T) { - testMockDbStore(100, 128, t) -} - -func TestMockDbStore2_100_(t *testing.T) { - testMockDbStore(100, 2, t) -} - -func TestMockDbStoreNotFound(t *testing.T) { - globalStore := mem.NewGlobalStore() - mockStore := globalStore.NewNodeStore(common.HexToAddress("0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed")) - m := initMockDbStore(t, mockStore) - defer m.Close() - _, err := m.Get(ZeroKey) - if err != notFound { - t.Errorf("Expected notFound, got %v", err) - } -} +// func TestMockDbStore128_0x1000000(t *testing.T) { +// testMockDbStore(0x1000000, 128, t) +// } +// +// func TestMockDbStore128_10000_(t *testing.T) { +// testMockDbStore(10000, 128, t) +// } +// +// func TestMockDbStore128_1000_(t *testing.T) { +// testMockDbStore(1000, 128, t) +// } +// +// func TestMockDbStore128_100_(t *testing.T) { +// testMockDbStore(100, 128, t) +// } +// +// func TestMockDbStore2_100_(t *testing.T) { +// testMockDbStore(100, 2, t) +// } +// +// func TestMockDbStoreNotFound(t *testing.T) { +// globalStore := mem.NewGlobalStore() +// mockStore := globalStore.NewNodeStore(common.HexToAddress("0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed")) +// m := initMockDbStore(t, mockStore) +// defer m.Close() +// _, err := m.Get(ZeroKey) +// if err != notFound { +// t.Errorf("Expected notFound, got %v", err) +// } +// } diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index 4fcddbf735..c12c706af5 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -56,6 +56,19 @@ func NewTestLocalStore(path string) (*LocalStore, error) { return localStore, nil } +func NewTestLocalStoreForAddr(path string, basekey []byte) (*LocalStore, error) { + hasher := MakeHashFunc("SHA3") + dbStore, err := NewDbStore(path, hasher, singletonSwarmDbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) }) + if err != nil { + return nil, err + } + localStore := &LocalStore{ + memStore: NewMemStore(dbStore, singletonSwarmDbCapacity), + DbStore: dbStore, + } + return localStore, nil +} + // LocalStore is itself a chunk store // unsafe, in that the data is not integrity checked func (self *LocalStore) Put(chunk *Chunk) { diff --git a/swarm/storage/resource_test.go b/swarm/storage/resource_test.go index b1cb91b2a2..d648d6b00c 100644 --- a/swarm/storage/resource_test.go +++ b/swarm/storage/resource_test.go @@ -24,7 +24,6 @@ import ( "github.com/ethereum/go-ethereum/contracts/ens/contract" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" ) @@ -37,10 +36,6 @@ var ( domainName = "føø.bar" ) -func init() { - log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) -} - // simulated backend does not have the blocknumber call // so we use this wrapper to fake returning the block count type fakeBackend struct { @@ -475,7 +470,8 @@ func newTestResourceHandler(datadir string, privkey *ecdsa.PrivateKey, rpcclient memStore: NewMemStore(dbStore, singletonSwarmDbCapacity), DbStore: dbStore, } - return NewResourceHandler(privkey, hasher, localStore, rpcclient, validator) + resourceChunkStore := newResourceChunkStore(path, hasher, localStore, func(*Chunk) error { return nil }) + return NewResourceHandler(privkey, hasher, resourceChunkStore, rpcclient, validator) } // Set up simulated ENS backend for use with ENSResourceHandler tests From 2844dd69c24ef37e7a9940f4f3ef0f1c58df957a Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Wed, 17 Jan 2018 12:59:43 +0100 Subject: [PATCH 062/210] swarm/storage: add mock store tests for DbStore --- swarm/storage/dbstore.go | 4 +- swarm/storage/dbstore_test.go | 178 ++++++++++++++++------------------ swarm/storage/dpa_test.go | 4 +- 3 files changed, 87 insertions(+), 99 deletions(-) diff --git a/swarm/storage/dbstore.go b/swarm/storage/dbstore.go index 60acf4871b..bed07bdd9b 100644 --- a/swarm/storage/dbstore.go +++ b/swarm/storage/dbstore.go @@ -628,10 +628,10 @@ func (s *DbStore) writeBatch(b *leveldb.Batch, entryCnt, dataIdx, accessCnt uint // not need to store the data, but still need to create the index. func newMockEncodeDataFunc(mockStore *mock.NodeStore) func(chunk *Chunk) []byte { return func(chunk *Chunk) []byte { - if err := mockStore.Put(chunk.Key, chunk.SData); err != nil { + if err := mockStore.Put(chunk.Key, encodeData(chunk)); err != nil { log.Error(fmt.Sprintf("%T: Chunk %v put: %v", mockStore, chunk.Key.Log(), err)) } - return nil + return chunk.Key[:] } } diff --git a/swarm/storage/dbstore_test.go b/swarm/storage/dbstore_test.go index 3ef83a8177..7f751594e5 100644 --- a/swarm/storage/dbstore_test.go +++ b/swarm/storage/dbstore_test.go @@ -21,13 +21,11 @@ import ( "fmt" "io/ioutil" "os" - "strings" "sync" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/swarm/storage/mock" "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" ) @@ -36,12 +34,22 @@ type testDbStore struct { dir string } -func newTestDbStore() (*testDbStore, error) { +func newTestDbStore(mock bool) (*testDbStore, error) { dir, err := ioutil.TempDir("", "bzz-storage-test") if err != nil { return nil, err } - db, err := NewDbStore(dir, MakeHashFunc(SHA3Hash), defaultDbCapacity, testPoFunc) + + var db *DbStore + if mock { + globalStore := mem.NewGlobalStore() + addr := common.HexToAddress("0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed") + mockStore := globalStore.NewNodeStore(addr) + + db, err = NewMockDbStore(dir, MakeHashFunc(SHA3Hash), defaultDbCapacity, testPoFunc, mockStore) + } else { + db, err = NewDbStore(dir, MakeHashFunc(SHA3Hash), defaultDbCapacity, testPoFunc) + } return &testDbStore{db, dir}, err } @@ -59,8 +67,8 @@ func (db *testDbStore) close() { } } -func testDbStoreRandom(n int, processors int, chunksize int, t *testing.T) { - db, err := newTestDbStore() +func testDbStoreRandom(n int, processors int, chunksize int, mock bool, t *testing.T) { + db, err := newTestDbStore(mock) if err != nil { t.Fatalf("init dbStore failed: %v", err) } @@ -69,8 +77,8 @@ func testDbStoreRandom(n int, processors int, chunksize int, t *testing.T) { testStoreRandom(db, processors, n, chunksize, t) } -func testDbStoreCorrect(n int, processors int, chunksize int, t *testing.T) { - db, err := newTestDbStore() +func testDbStoreCorrect(n int, processors int, chunksize int, mock bool, t *testing.T) { + db, err := newTestDbStore(mock) if err != nil { t.Fatalf("init dbStore failed: %v", err) } @@ -79,31 +87,55 @@ func testDbStoreCorrect(n int, processors int, chunksize int, t *testing.T) { } func TestDbStoreRandom_1(t *testing.T) { - testDbStoreRandom(1, 1, 0, t) + testDbStoreRandom(1, 1, 0, false, t) } func TestDbStoreCorrect_1(t *testing.T) { - testDbStoreCorrect(1, 1, 4096, t) + testDbStoreCorrect(1, 1, 4096, false, t) } func TestDbStoreRandom_1_5k(t *testing.T) { - testDbStoreRandom(8, 5000, 0, t) + testDbStoreRandom(8, 5000, 0, false, t) } func TestDbStoreRandom_8_5k(t *testing.T) { - testDbStoreRandom(8, 5000, 0, t) + testDbStoreRandom(8, 5000, 0, false, t) } func TestDbStoreCorrect_1_5k(t *testing.T) { - testDbStoreCorrect(1, 5000, 4096, t) + testDbStoreCorrect(1, 5000, 4096, false, t) } func TestDbStoreCorrect_8_5k(t *testing.T) { - testDbStoreCorrect(8, 5000, 4096, t) + testDbStoreCorrect(8, 5000, 4096, false, t) } -func TestDbStoreNotFound(t *testing.T) { - db, err := newTestDbStore() +func TestMockDbStoreRandom_1(t *testing.T) { + testDbStoreRandom(1, 1, 0, true, t) +} + +func TestMockDbStoreCorrect_1(t *testing.T) { + testDbStoreCorrect(1, 1, 4096, true, t) +} + +func TestMockDbStoreRandom_1_5k(t *testing.T) { + testDbStoreRandom(8, 5000, 0, true, t) +} + +func TestMockDbStoreRandom_8_5k(t *testing.T) { + testDbStoreRandom(8, 5000, 0, true, t) +} + +func TestMockDbStoreCorrect_1_5k(t *testing.T) { + testDbStoreCorrect(1, 5000, 4096, true, t) +} + +func TestMockDbStoreCorrect_8_5k(t *testing.T) { + testDbStoreCorrect(8, 5000, 4096, true, t) +} + +func testDbStoreNotFound(t *testing.T, mock bool) { + db, err := newTestDbStore(mock) if err != nil { t.Fatalf("init dbStore failed: %v", err) } @@ -115,7 +147,14 @@ func TestDbStoreNotFound(t *testing.T) { } } -func TestIterator(t *testing.T) { +func TestDbStoreNotFound(t *testing.T) { + testDbStoreNotFound(t, false) +} +func TestMockDbStoreNotFound(t *testing.T) { + testDbStoreNotFound(t, true) +} + +func testIterator(t *testing.T, mock bool) { var chunkcount int = 32 var i int var poc uint @@ -127,7 +166,7 @@ func TestIterator(t *testing.T) { chunks = append(chunks, NewChunk(nil, nil)) } - db, err := newTestDbStore() + db, err := newTestDbStore(mock) if err != nil { t.Fatalf("init dbStore failed: %v", err) } @@ -174,8 +213,15 @@ func TestIterator(t *testing.T) { } -func benchmarkDbStorePut(n int, processors int, chunksize int, b *testing.B) { - db, err := newTestDbStore() +func TestIterator(t *testing.T) { + testIterator(t, false) +} +func TestMockIterator(t *testing.T) { + testIterator(t, true) +} + +func benchmarkDbStorePut(n int, processors int, chunksize int, mock bool, b *testing.B) { + db, err := newTestDbStore(mock) if err != nil { b.Fatalf("init dbStore failed: %v", err) } @@ -184,8 +230,8 @@ func benchmarkDbStorePut(n int, processors int, chunksize int, b *testing.B) { benchmarkStorePut(db, processors, n, chunksize, b) } -func benchmarkDbStoreGet(n int, processors int, chunksize int, b *testing.B) { - db, err := newTestDbStore() +func benchmarkDbStoreGet(n int, processors int, chunksize int, mock bool, b *testing.B) { + db, err := newTestDbStore(mock) if err != nil { b.Fatalf("init dbStore failed: %v", err) } @@ -195,91 +241,33 @@ func benchmarkDbStoreGet(n int, processors int, chunksize int, b *testing.B) { } func BenchmarkDbStorePut_1_5k(b *testing.B) { - benchmarkDbStorePut(5000, 1, 4096, b) + benchmarkDbStorePut(5000, 1, 4096, false, b) } func BenchmarkDbStorePut_8_5k(b *testing.B) { - benchmarkDbStorePut(5000, 8, 4096, b) + benchmarkDbStorePut(5000, 8, 4096, false, b) } func BenchmarkDbStoreGet_1_5k(b *testing.B) { - benchmarkDbStoreGet(5000, 1, 4096, b) + benchmarkDbStoreGet(5000, 1, 4096, false, b) } func BenchmarkDbStoreGet_8_5k(b *testing.B) { - benchmarkDbStoreGet(5000, 8, 4096, b) + benchmarkDbStoreGet(5000, 8, 4096, false, b) } -func initMockDbStore(t *testing.T, mockStore *mock.NodeStore) *DbStore { - dir, err := ioutil.TempDir("", "bzz-storage-test-mock") - if err != nil { - t.Fatal(err) - } - m, err := NewMockDbStore(dir, MakeHashFunc(SHA3Hash), defaultDbCapacity, testPoFunc, mockStore) - if err != nil { - t.Fatal("can't create store:", err) - } - return m +func BenchmarkMockDbStorePut_1_5k(b *testing.B) { + benchmarkDbStorePut(5000, 1, 4096, true, b) } -// testMockDbStore runs the same tests as testDbStore but with mock store configured. -// It also verifies if mock global store is storing the chunk data. -func testMockDbStore(l int64, branches int64, t *testing.T) { - globalStore := mem.NewGlobalStore() - addr := common.HexToAddress("0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed") - mockStore := globalStore.NewNodeStore(addr) - m := initMockDbStore(t, mockStore) - defer m.Close() - - key := Key(common.Hex2Bytes("fed1911825fc6a02ebfd19ab218a20455d8d7d275f8bf4d8244eb04364fae6f7")) - data := common.Hex2BytesFixed(strings.Repeat("1234567890abcdf", 10), 4096) - - m.Put(&Chunk{ - Key: key, - SData: data, - }) - - _, err := globalStore.Get(addr, key) - if err != nil { - t.Errorf("unexpected error getting the data from global mock store: %v", err) - } - - if !globalStore.HasKey(addr, key) { - t.Error("key not found in global store") - } - - // TODO: fix this! - // testStoreRandom(m, 8, l, chunk.S, t) - +func BenchmarkMockDbStorePut_8_5k(b *testing.B) { + benchmarkDbStorePut(5000, 8, 4096, true, b) } -// func TestMockDbStore128_0x1000000(t *testing.T) { -// testMockDbStore(0x1000000, 128, t) -// } -// -// func TestMockDbStore128_10000_(t *testing.T) { -// testMockDbStore(10000, 128, t) -// } -// -// func TestMockDbStore128_1000_(t *testing.T) { -// testMockDbStore(1000, 128, t) -// } -// -// func TestMockDbStore128_100_(t *testing.T) { -// testMockDbStore(100, 128, t) -// } -// -// func TestMockDbStore2_100_(t *testing.T) { -// testMockDbStore(100, 2, t) -// } -// -// func TestMockDbStoreNotFound(t *testing.T) { -// globalStore := mem.NewGlobalStore() -// mockStore := globalStore.NewNodeStore(common.HexToAddress("0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed")) -// m := initMockDbStore(t, mockStore) -// defer m.Close() -// _, err := m.Get(ZeroKey) -// if err != notFound { -// t.Errorf("Expected notFound, got %v", err) -// } -// } +func BenchmarkMockDbStoreGet_1_5k(b *testing.B) { + benchmarkDbStoreGet(5000, 1, 4096, true, b) +} + +func BenchmarkMockDbStoreGet_8_5k(b *testing.B) { + benchmarkDbStoreGet(5000, 8, 4096, true, b) +} diff --git a/swarm/storage/dpa_test.go b/swarm/storage/dpa_test.go index 391418674b..20b07216a3 100644 --- a/swarm/storage/dpa_test.go +++ b/swarm/storage/dpa_test.go @@ -27,7 +27,7 @@ import ( const testDataSize = 0x1000000 func TestDPArandom(t *testing.T) { - tdb, err := newTestDbStore() + tdb, err := newTestDbStore(false) if err != nil { t.Fatalf("init dbStore failed: %v", err) } @@ -86,7 +86,7 @@ func TestDPArandom(t *testing.T) { } func TestDPA_capacity(t *testing.T) { - tdb, err := newTestDbStore() + tdb, err := newTestDbStore(false) if err != nil { t.Fatalf("init dbStore failed: %v", err) } From 0265566534bb180a70819583c7cb2b20174e1161 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Wed, 17 Jan 2018 18:51:08 +0100 Subject: [PATCH 063/210] Fix swarm api tests and rename dpaChunkStore to NetStore --- swarm/api/api_test.go | 5 +- swarm/api/config_test.go | 12 +- swarm/api/filesystem.go | 4 +- swarm/api/filesystem_test.go | 8 +- swarm/api/http/server_test.go | 2 +- swarm/api/manifest.go | 7 +- swarm/api/storage.go | 4 +- swarm/fuse/swarmfs_test.go | 2 +- swarm/network/protocol.go | 9 +- swarm/network/request_test.go | 2 +- .../simulations/discovery/discovery_test.go | 2 +- swarm/network/streamer.go | 4 + swarm/pss/client/client_test.go | 4 +- swarm/pss/pss_test.go | 6 +- swarm/storage/chunker_test.go | 8 +- swarm/storage/dpa.go | 55 +------ swarm/storage/localstore.go | 26 ++- swarm/storage/netstore.go | 148 ++++++------------ swarm/storage/resource.go | 2 +- swarm/swarm.go | 5 +- swarm/testutil/http.go | 3 +- 21 files changed, 115 insertions(+), 203 deletions(-) diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index 44bf8aadc2..da1d8bcf23 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -34,9 +34,8 @@ func testApi(t *testing.T, f func(*Api)) { if err != nil { t.Fatalf("unable to create temp dir: %v", err) } - os.RemoveAll(datadir) defer os.RemoveAll(datadir) - dpa, err := storage.NewLocalDPA(datadir) + dpa, err := storage.NewLocalDPA(datadir, make([]byte, 32)) if err != nil { return } @@ -114,7 +113,7 @@ func TestApiPut(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - resp := testGet(t, api, key.String(), "") + resp := testGet(t, api, key.Hex(), "") checkResponse(t, resp, exp) }) } diff --git a/swarm/api/config_test.go b/swarm/api/config_test.go index 5636b6dafb..993388686b 100644 --- a/swarm/api/config_test.go +++ b/swarm/api/config_test.go @@ -33,8 +33,8 @@ func TestConfig(t *testing.T) { t.Fatalf("failed to load private key: %v", err) } - one := NewDefaultConfig() - two := NewDefaultConfig() + one := NewConfig() + two := NewConfig() if equal := reflect.DeepEqual(one, two); !equal { t.Fatal("Two default configs are not equal") @@ -55,14 +55,6 @@ func TestConfig(t *testing.T) { t.Fatal("Failed to correctly initialize SwapParams") } - if one.SyncParams.RequestDbPath == one.Path { - t.Fatal("Failed to correctly initialize SyncParams") - } - - if one.HiveParams.KadDbPath == one.Path { - t.Fatal("Failed to correctly initialize HiveParams") - } - if one.StoreParams.ChunkDbPath == one.Path { t.Fatal("Failed to correctly initialize StoreParams") } diff --git a/swarm/api/filesystem.go b/swarm/api/filesystem.go index b6a2de8862..0074ea167d 100644 --- a/swarm/api/filesystem.go +++ b/swarm/api/filesystem.go @@ -116,7 +116,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) { var wait func() hash, wait, err = self.api.dpa.Store(f, stat.Size()) if hash != nil { - list[i].Hash = hash.String() + list[i].Hash = hash.Hex() } wait() awg.Done() @@ -164,7 +164,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) { err2 := trie.recalcAndStore() var hs string if err2 == nil { - hs = trie.hash.String() + hs = trie.hash.Hex() } awg.Wait() return hs, err2 diff --git a/swarm/api/filesystem_test.go b/swarm/api/filesystem_test.go index 8a15e735dc..6f1594e991 100644 --- a/swarm/api/filesystem_test.go +++ b/swarm/api/filesystem_test.go @@ -21,7 +21,6 @@ import ( "io/ioutil" "os" "path/filepath" - "sync" "testing" "github.com/ethereum/go-ethereum/common" @@ -105,9 +104,8 @@ func TestApiDirUploadModify(t *testing.T) { t.Errorf("unexpected error: %v", err) return } - wg := &sync.WaitGroup{} - hash, err := api.Store(bytes.NewReader(index), int64(len(index)), wg) - wg.Wait() + hash, wait, err := api.Store(bytes.NewReader(index), int64(len(index))) + wait() if err != nil { t.Errorf("unexpected error: %v", err) return @@ -122,7 +120,7 @@ func TestApiDirUploadModify(t *testing.T) { t.Errorf("unexpected error: %v", err) return } - bzzhash = key.String() + bzzhash = key.Hex() content := readPath(t, "testdata", "test0", "index.html") resp := testGet(t, api, bzzhash, "index2.html") diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 6a35d2c785..bd05d77fcc 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -120,7 +120,7 @@ func TestBzzGetPath(t *testing.T) { t.Fatalf("Read request body: %v", err) } - if string(respbody) != key[v].String() { + if string(respbody) != key[v].Hex() { isexpectedfailrequest := false for _, r := range expectedfailrequests { diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index f6279a4ced..b8b64caa89 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -90,7 +90,7 @@ func (m *ManifestWriter) AddEntry(data io.Reader, e *ManifestEntry) (storage.Key return nil, err } entry := newManifestTrieEntry(e, nil) - entry.Hash = key.String() + entry.Hash = key.Hex() m.trie.addEntry(entry, m.quitC) return key, nil } @@ -338,7 +338,7 @@ func (self *manifestTrie) recalcAndStore() error { if err != nil { return err } - entry.Hash = entry.subtrie.hash.String() + entry.Hash = entry.subtrie.hash.Hex() } list.Entries = append(list.Entries, entry.ManifestEntry) } @@ -351,7 +351,8 @@ func (self *manifestTrie) recalcAndStore() error { } sr := bytes.NewReader(manifest) - key, _, err2 := self.dpa.Store(sr, int64(len(manifest))) + key, wait, err2 := self.dpa.Store(sr, int64(len(manifest))) + wait() self.hash = key return err2 } diff --git a/swarm/api/storage.go b/swarm/api/storage.go index ae94e15cb9..4679fabad3 100644 --- a/swarm/api/storage.go +++ b/swarm/api/storage.go @@ -46,7 +46,7 @@ func (self *Storage) Put(content, contentType string) (string, error) { if err != nil { return "", err } - return key.String(), err + return key.Hex(), err } // Get retrieves the content from bzzpath and reads the response in full @@ -100,5 +100,5 @@ func (self *Storage) Modify(rootHash, path, contentHash, contentType string) (ne if err != nil { return "", err } - return key.String(), nil + return key.Hex(), nil } diff --git a/swarm/fuse/swarmfs_test.go b/swarm/fuse/swarmfs_test.go index 42af36345f..e768f86495 100644 --- a/swarm/fuse/swarmfs_test.go +++ b/swarm/fuse/swarmfs_test.go @@ -808,7 +808,7 @@ func TestFUSE(t *testing.T) { } os.RemoveAll(datadir) - dpa, err := storage.NewLocalDPA(datadir) + dpa, err := storage.NewLocalDPA(datadir, make([]byte, 32)) if err != nil { t.Fatal(err) } diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index 9374d844c8..4f5906b4ae 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -143,7 +143,7 @@ func (b *Bzz) NodeInfo() interface{} { // * handshake/hive // * discovery func (b *Bzz) Protocols() []p2p.Protocol { - return []p2p.Protocol{ + protocols := []p2p.Protocol{ { Name: BzzSpec.Name, Version: BzzSpec.Version, @@ -159,15 +159,18 @@ func (b *Bzz) Protocols() []p2p.Protocol { NodeInfo: b.Hive.NodeInfo, PeerInfo: b.Hive.PeerInfo, }, - { + } + if b.Streamer != nil { + protocols = append(protocols, p2p.Protocol{ Name: StreamerSpec.Name, Version: StreamerSpec.Version, Length: StreamerSpec.Length(), Run: b.RunProtocol(StreamerSpec, b.Streamer.Run), NodeInfo: b.Streamer.NodeInfo, PeerInfo: b.Streamer.PeerInfo, - }, + }) } + return protocols } // APIs returns the APIs offered by bzz diff --git a/swarm/network/request_test.go b/swarm/network/request_test.go index 12e820b2ad..8ae1d3c344 100644 --- a/swarm/network/request_test.go +++ b/swarm/network/request_test.go @@ -326,7 +326,7 @@ func testDeliveryFromNodes(nodes, conns, size int, skipCheck bool) func(adapter rrdpa := storage.NewDPA(newRoundRobinStore(localStores[1:]...), storage.NewChunkerParams()) rrdpa.Start() // create a retriever dpa for the pivot node - dpacs := storage.NewDpaChunkStore(localStores[0].(*storage.LocalStore), func(chunk *storage.Chunk) error { return delivery.RequestFromPeers(chunk.Key[:], skipCheck) }) + dpacs := storage.NewNetStore(localStores[0].(*storage.LocalStore), func(chunk *storage.Chunk) error { return delivery.RequestFromPeers(chunk.Key[:], skipCheck) }) dpa := storage.NewDPA(dpacs, storage.NewChunkerParams()) dpa.Start() return func(context.Context) error { diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go index fc8d6f70ca..33a1c04868 100644 --- a/swarm/network/simulations/discovery/discovery_test.go +++ b/swarm/network/simulations/discovery/discovery_test.go @@ -319,5 +319,5 @@ func newService(ctx *adapters.ServiceContext) (node.Service, error) { HiveParams: hp, } - return network.NewBzz(config, kad, nil), nil + return network.NewBzz(config, kad, nil, nil), nil } diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go index 66abf82f3a..b0e8b9eb3b 100644 --- a/swarm/network/streamer.go +++ b/swarm/network/streamer.go @@ -147,6 +147,10 @@ func NewStreamer(delivery *Delivery) *Streamer { return streamer } +func (self *Streamer) Retrieve(chunk *storage.Chunk) error { + return self.delivery.RequestFromPeers(chunk.Key[:], false) +} + // RegisterIncomingStreamer registers an incoming streamer constructor func (self *Streamer) RegisterIncomingStreamer(stream string, f func(*StreamerPeer, []byte) (IncomingStreamer, error)) { self.incomingLock.Lock() diff --git a/swarm/pss/client/client_test.go b/swarm/pss/client/client_test.go index e773018a16..fe11e37c91 100644 --- a/swarm/pss/client/client_test.go +++ b/swarm/pss/client/client_test.go @@ -233,7 +233,7 @@ func newServices() adapters.Services { if err != nil { return nil, fmt.Errorf("create pss cache tmpdir failed", "error", err) } - dpa, err := storage.NewLocalDPA(cachedir) + dpa, err := storage.NewLocalDPA(cachedir, make([]byte, 32)) if err != nil { return nil, fmt.Errorf("local dpa creation failed", "error", err) } @@ -260,7 +260,7 @@ func newServices() adapters.Services { UnderlayAddr: addr.Under(), HiveParams: hp, } - return network.NewBzz(config, kademlia(ctx.Config.ID), stateStore), nil + return network.NewBzz(config, kademlia(ctx.Config.ID), stateStore, nil), nil }, } } diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index 57d4a79170..bda1490c9d 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -1123,7 +1123,7 @@ func newServices() adapters.Services { if err != nil { return nil, fmt.Errorf("create pss cache tmpdir failed", "error", err) } - dpa, err := storage.NewLocalDPA(cachedir) + dpa, err := storage.NewLocalDPA(cachedir, network.NewAddrFromNodeID(ctx.Config.ID).Over()) if err != nil { return nil, fmt.Errorf("local dpa creation failed", "error", err) } @@ -1178,7 +1178,7 @@ func newServices() adapters.Services { UnderlayAddr: addr.Under(), HiveParams: hp, } - return network.NewBzz(config, kademlia(ctx.Config.ID), stateStore), nil + return network.NewBzz(config, kademlia(ctx.Config.ID), stateStore, nil), nil }, } } @@ -1195,7 +1195,7 @@ func newTestPss(privkey *ecdsa.PrivateKey, overlay network.Overlay, ppextra *Pss log.Error("create pss cache tmpdir failed", "error", err) os.Exit(1) } - dpa, err := storage.NewLocalDPA(cachedir) + dpa, err := storage.NewLocalDPA(cachedir, addr.Over()) if err != nil { log.Error("local dpa creation failed", "error", err) os.Exit(1) diff --git a/swarm/storage/chunker_test.go b/swarm/storage/chunker_test.go index fb66b7c756..cb9d73a93e 100644 --- a/swarm/storage/chunker_test.go +++ b/swarm/storage/chunker_test.go @@ -64,7 +64,7 @@ func (self *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, c return nil case chunk := <-chunkC: // self.chunks = append(self.chunks, chunk) - self.chunks[chunk.Key.String()] = chunk + self.chunks[chunk.Key.Hex()] = chunk close(chunk.dbStored) } @@ -101,10 +101,10 @@ func (self *chunkerTester) Append(chunker Splitter, rootKey Key, data io.Reader, return nil case chunk := <-chunkC: if chunk != nil { - stored, success := self.chunks[chunk.Key.String()] + stored, success := self.chunks[chunk.Key.Hex()] if !success { // Requesting data - self.chunks[chunk.Key.String()] = chunk + self.chunks[chunk.Key.Hex()] = chunk close(chunk.dbStored) } else { // getting data @@ -151,7 +151,7 @@ func (self *chunkerTester) Join(chunker Chunker, key Key, c int, chunkC chan *Ch return nil } // this just mocks the behaviour of a chunk store retrieval - stored, success := self.chunks[chunk.Key.String()] + stored, success := self.chunks[chunk.Key.Hex()] if !success { return errors.New("Not found") } diff --git a/swarm/storage/dpa.go b/swarm/storage/dpa.go index b54c63804c..b0aafe0343 100644 --- a/swarm/storage/dpa.go +++ b/swarm/storage/dpa.go @@ -17,7 +17,6 @@ package storage import ( - "encoding/binary" "errors" "fmt" "io" @@ -50,6 +49,9 @@ const ( var ( notFound = errors.New("not found") + + // timeout interval before retrieval is timed out + searchTimeout = 3 * time.Second ) type DPA struct { @@ -173,54 +175,3 @@ func (self *DPA) storeWorker() { } } } - -// DpaChunkStore implements the ChunkStore interface, -// this chunk access layer assumed 2 chunk stores -// local storage eg. LocalStore and network storage eg., NetStore -// access by calling network is blocking with a timeout - -type dpaChunkStore struct { - localStore *LocalStore - retrieve func(chunk *Chunk) error -} - -func NewDpaChunkStore(localStore *LocalStore, retrieve func(chunk *Chunk) error) *dpaChunkStore { - return &dpaChunkStore{localStore, retrieve} -} - -// Get is the entrypoint for local retrieve requests -// waits for response or times out -func (self *dpaChunkStore) Get(key Key) (chunk *Chunk, err error) { - var created bool - chunk, created = self.localStore.GetOrCreateRequest(key) - if chunk.ReqC == nil { - log.Trace(fmt.Sprintf("DPA.Get: %v found locally, %d bytes", key.Log(), len(chunk.SData))) - return - } - - if created { - if err := self.retrieve(chunk); err != nil { - return nil, err - } - } - t := time.NewTicker(searchTimeout) - defer t.Stop() - - select { - case <-t.C: - log.Trace(fmt.Sprintf("DPA.Get: %v request time out ", key.Log())) - return nil, notFound - case <-chunk.ReqC: - } - chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) - return chunk, nil -} - -// Put is the entrypoint for local store requests coming from storeLoop -func (self *dpaChunkStore) Put(chunk *Chunk) { - self.localStore.Put(chunk) -} - -// Close chunk store -func (self *dpaChunkStore) Close() { -} diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index c12c706af5..893670b232 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -19,10 +19,32 @@ package storage import ( "encoding/binary" "fmt" + "path/filepath" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/storage/mock" ) +type StoreParams struct { + ChunkDbPath string + DbCapacity uint64 + CacheCapacity uint +} + +//create params with default values +func NewDefaultStoreParams() (self *StoreParams) { + return &StoreParams{ + DbCapacity: defaultDbCapacity, + CacheCapacity: defaultCacheCapacity, + } +} + +//this can only finally be set after all config options (file, cmd line, env vars) +//have been evaluated +func (self *StoreParams) Init(path string) { + self.ChunkDbPath = filepath.Join(path, "chunks") +} + // LocalStore is a combination of inmemory db over a disk persisted db // implements a Get/Put with fallback (caching) logic using any 2 ChunkStores type LocalStore struct { @@ -31,8 +53,8 @@ type LocalStore struct { } // This constructor uses MemStore and DbStore as components -func NewLocalStore(hash SwarmHasher, params *StoreParams, basekey []byte) (*LocalStore, error) { - dbStore, err := NewDbStore(params.ChunkDbPath, hash, params.DbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) }) +func NewLocalStore(hash SwarmHasher, params *StoreParams, basekey []byte, mockStore *mock.NodeStore) (*LocalStore, error) { + dbStore, err := NewMockDbStore(params.ChunkDbPath, hash, params.DbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) }, mockStore) if err != nil { return nil, err } diff --git a/swarm/storage/netstore.go b/swarm/storage/netstore.go index 4a7caf7c2e..f01ffe4a69 100644 --- a/swarm/storage/netstore.go +++ b/swarm/storage/netstore.go @@ -17,116 +17,58 @@ package storage import ( - "path/filepath" + "encoding/binary" + "fmt" "time" + + "github.com/ethereum/go-ethereum/log" ) -// import ( -// "fmt" -// "path/filepath" -// "time" - -// "github.com/ethereum/go-ethereum/log" -// ) - -// /* -// NetStore is a cloud storage access abstaction layer for swarm -// it contains the shared logic of network served chunk store/retrieval requests -// both local (coming from DPA api) and remote (coming from peers via bzz protocol) -// it implements the ChunkStore interface and embeds LocalStore - -// It is called by the bzz protocol instances via Depo (the store/retrieve request handler) -// a protocol instance is running on each peer, so this is heavily parallelised. -// NetStore falls back to a backend (CloudStorage interface) -// implemented by bzz/network/forwarder. forwarder or IPFS or IPΞS -// */ -// type NetStore struct { -// hashfunc SwarmHasher -// localStore *LocalStore -// cloud CloudStore -// } - -// // backend engine for cloud store -// // It can be aggregate dispatching to several parallel implementations: -// // bzz/network/forwarder. forwarder or IPFS or IPΞS -// type CloudStore interface { -// Store(*Chunk) -// Deliver(*Chunk) -// Retrieve(*Chunk) -// } - -type StoreParams struct { - ChunkDbPath string - DbCapacity uint64 - CacheCapacity uint - Radius int +// NetStore implements the ChunkStore interface, +// this chunk access layer assumed 2 chunk stores +// local storage eg. LocalStore and network storage eg., NetStore +// access by calling network is blocking with a timeout +type NetStore struct { + localStore *LocalStore + retrieve func(chunk *Chunk) error } -//create params with default values -func NewDefaultStoreParams() (self *StoreParams) { - return &StoreParams{ - DbCapacity: defaultDbCapacity, - CacheCapacity: defaultCacheCapacity, - Radius: defaultRadius, +func NewNetStore(localStore *LocalStore, retrieve func(chunk *Chunk) error) *NetStore { + return &NetStore{localStore, retrieve} +} + +// Get is the entrypoint for local retrieve requests +// waits for response or times out +func (self *NetStore) Get(key Key) (chunk *Chunk, err error) { + var created bool + chunk, created = self.localStore.GetOrCreateRequest(key) + if chunk.ReqC == nil { + log.Trace(fmt.Sprintf("DPA.Get: %v found locally, %d bytes", key.Log(), len(chunk.SData))) + return } + + if created { + if err := self.retrieve(chunk); err != nil { + return nil, err + } + } + t := time.NewTicker(searchTimeout) + defer t.Stop() + + select { + case <-t.C: + log.Trace(fmt.Sprintf("DPA.Get: %v request time out ", key.Log())) + return nil, notFound + case <-chunk.ReqC: + } + chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) + return chunk, nil } -//this can only finally be set after all config options (file, cmd line, env vars) -//have been evaluated -func (self *StoreParams) Init(path string) { - self.ChunkDbPath = filepath.Join(path, "chunks") +// Put is the entrypoint for local store requests coming from storeLoop +func (self *NetStore) Put(chunk *Chunk) { + self.localStore.Put(chunk) } -// // netstore contructor, takes path argument that is used to initialise dbStore, -// // the persistent (disk) storage component of LocalStore -// // the second argument is the hive, the connection/logistics manager for the node -// func NewNetStore(hash SwarmHasher, lstore *LocalStore, cloud CloudStore, params *StoreParams) *NetStore { -// return &NetStore{ -// hashfunc: hash, -// localStore: lstore, -// cloud: cloud, -// } -// } - -// const ( -// // maximum number of peers that a retrieved message is delivered to -// requesterCount = 3 -// ) - -var ( - // timeout interval before retrieval is timed out - searchTimeout = 3 * time.Second -) - -// // store logic common to local and network chunk store requests -// // ~ unsafe put in localdb no check if exists no extra copy no hash validation -// // the chunk is forced to propagate (Cloud.Store) even if locally found! -// // caller needs to make sure if that is wanted -// func (self *NetStore) Put(entry *Chunk) { -// self.localStore.Put(entry) - -// // handle deliveries -// if entry.ReqC != nil { -// log.Trace(fmt.Sprintf("NetStore.Put: localStore.Put %v hit existing request...delivering", entry.Key.Log())) -// // closing C signals to other routines (local requests) -// // that the chunk is has been retrieved -// close(entry.ReqC) -// // deliver the chunk to requesters upstream -// go self.cloud.Deliver(entry) -// } else { -// log.Trace(fmt.Sprintf("NetStore.Put: localStore.Put %v stored locally", entry.Key.Log())) -// // handle propagating store requests -// // go self.cloud.Store(entry) -// go self.cloud.Store(entry) -// } -// } - -// // retrieve logic common for local and network chunk retrieval requests -// func (self *NetStore) Get(key Key) (*Chunk, error) { -// chunk, _ := self.localStore.GetOrCreateRequest(key) -// go self.cloud.Retrieve(chunk) -// return chunk, nil -// } - -// // Close netstore -// func (self *NetStore) Close() {} +// Close chunk store +func (self *NetStore) Close() {} diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index e285962591..3f27de5e3e 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -639,7 +639,7 @@ type resourceChunkStore struct { func newResourceChunkStore(path string, hasher SwarmHasher, localStore *LocalStore, request func(*Chunk) error) *resourceChunkStore { return &resourceChunkStore{ localStore: localStore, - netStore: NewDpaChunkStore(localStore, request), + netStore: NewNetStore(localStore, request), } } diff --git a/swarm/swarm.go b/swarm/swarm.go index 3657088383..bc6533875b 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -130,14 +130,15 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e } dbAccess := network.NewDbAccess(self.lstore) - self.streamer = network.NewStreamer(to, dbAccess) + delivery := network.NewDelivery(to, dbAccess) + self.streamer = network.NewStreamer(delivery) network.RegisterOutgoingSyncer(self.streamer, dbAccess) network.RegisterIncomingSyncer(self.streamer, dbAccess) self.bzz = network.NewBzz(bzzconfig, to, nil, self.streamer) // set up DPA, the cloud storage local access layer - dpaChunkStore := storage.NewDpaChunkStore(self.lstore, self.streamer.Retrieve) + dpaChunkStore := storage.NewNetStore(self.lstore, self.streamer.Retrieve) log.Debug(fmt.Sprintf("-> Local Access to Swarm")) // Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage self.dpa = storage.NewDPA(dpaChunkStore, self.config.ChunkerParams) diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go index f2922fab00..32c83abc4e 100644 --- a/swarm/testutil/http.go +++ b/swarm/testutil/http.go @@ -36,9 +36,8 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { ChunkDbPath: dir, DbCapacity: 5000000, CacheCapacity: 5000, - Radius: 0, } - localStore, err := storage.NewLocalStore(storage.MakeHashFunc("SHA3"), storeparams) + localStore, err := storage.NewLocalStore(storage.MakeHashFunc("SHA3"), storeparams, make([]byte, 32), nil) if err != nil { os.RemoveAll(dir) t.Fatal(err) From 35609bec2e59db4f2346980ef502e327c6129f1f Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Thu, 18 Jan 2018 17:53:29 +0100 Subject: [PATCH 064/210] swarm/network, swarm/storage: initial netowork/stream refactor --- swarm/network/discovery.go | 8 +- swarm/network/discovery_test.go | 2 +- swarm/network/hive.go | 4 +- swarm/network/kademlia_test.go | 2 +- swarm/network/{ => light}/lightnode.go | 55 +- swarm/network/protocol.go | 35 +- swarm/network/protocol_test.go | 8 +- swarm/network/stream/common_test.go | 130 ++++ .../{requests.go => stream/delivery.go} | 81 +-- .../delivery_test.go} | 16 +- swarm/network/stream/messages.go | 228 +++++++ swarm/network/stream/peer.go | 164 +++++ swarm/network/stream/stream.go | 316 +++++++++ swarm/network/{ => stream}/streamer_test.go | 10 +- swarm/network/{ => stream}/syncer.go | 151 ++--- swarm/network/{ => stream}/syncer_test.go | 37 +- .../testing/testing.go} | 159 +---- swarm/network/streamer.go | 630 ------------------ swarm/storage/dbaccess.go | 52 ++ swarm/swarm.go | 20 +- 20 files changed, 1119 insertions(+), 989 deletions(-) rename swarm/network/{ => light}/lightnode.go (67%) create mode 100644 swarm/network/stream/common_test.go rename swarm/network/{requests.go => stream/delivery.go} (61%) rename swarm/network/{request_test.go => stream/delivery_test.go} (97%) create mode 100644 swarm/network/stream/messages.go create mode 100644 swarm/network/stream/peer.go create mode 100644 swarm/network/stream/stream.go rename swarm/network/{ => stream}/streamer_test.go (94%) rename swarm/network/{ => stream}/syncer.go (50%) rename swarm/network/{ => stream}/syncer_test.go (85%) rename swarm/network/{streamer_common_test.go => stream/testing/testing.go} (57%) delete mode 100644 swarm/network/streamer.go create mode 100644 swarm/storage/dbaccess.go diff --git a/swarm/network/discovery.go b/swarm/network/discovery.go index fb7152cba1..71ed974d2f 100644 --- a/swarm/network/discovery.go +++ b/swarm/network/discovery.go @@ -25,9 +25,9 @@ import ( // discovery bzz extension for requesting and relaying node address records -// discPeer wraps bzzPeer and embeds an Overlay connectivity driver +// discPeer wraps BzzPeer and embeds an Overlay connectivity driver type discPeer struct { - *bzzPeer + *BzzPeer overlay Overlay sentPeers bool // whether we already sent peer closer to this address mtx sync.Mutex @@ -36,10 +36,10 @@ type discPeer struct { } // NewDiscovery constructs a discovery peer -func newDiscovery(p *bzzPeer, o Overlay) *discPeer { +func newDiscovery(p *BzzPeer, o Overlay) *discPeer { d := &discPeer{ overlay: o, - bzzPeer: p, + BzzPeer: p, peers: make(map[string]bool), } // record remote as seen so we never send a peer its own record diff --git a/swarm/network/discovery_test.go b/swarm/network/discovery_test.go index ee90683a73..50e1f468b6 100644 --- a/swarm/network/discovery_test.go +++ b/swarm/network/discovery_test.go @@ -33,7 +33,7 @@ func TestDiscovery(t *testing.T) { addr := RandomAddr() to := NewKademlia(addr.OAddr, NewKadParams()) - run := func(p *bzzPeer) error { + run := func(p *BzzPeer) error { dp := newDiscovery(p, to) to.On(p) defer to.Off(p) diff --git a/swarm/network/hive.go b/swarm/network/hive.go index d72d7c5e7c..a309d93983 100644 --- a/swarm/network/hive.go +++ b/swarm/network/hive.go @@ -159,7 +159,7 @@ func (h *Hive) connect() { } // Run protocol run function -func (h *Hive) Run(p *bzzPeer) error { +func (h *Hive) Run(p *BzzPeer) error { dp := newDiscovery(p, h) depth, changed := h.On(dp) // if we want discovery, advertise changed depth of depth @@ -191,7 +191,7 @@ func ToAddr(pa OverlayPeer) *BzzAddr { if p, ok := pa.(*discPeer); ok { return p.BzzAddr } - return pa.(*bzzPeer).BzzAddr + return pa.(*BzzPeer).BzzAddr } // loadPeers, savePeer implement persistence callback/ diff --git a/swarm/network/kademlia_test.go b/swarm/network/kademlia_test.go index 20bfd7daaf..01ed72c582 100644 --- a/swarm/network/kademlia_test.go +++ b/swarm/network/kademlia_test.go @@ -70,7 +70,7 @@ func newTestKademlia(b string) *testKademlia { } func (k *testKademlia) newTestKadPeer(s string) Peer { - return &testDropPeer{&bzzPeer{BzzAddr: testKadPeerAddr(s)}, k.dropc} + return &testDropPeer{&BzzPeer{BzzAddr: testKadPeerAddr(s)}, k.dropc} } func (k *testKademlia) On(ons ...string) *testKademlia { diff --git a/swarm/network/lightnode.go b/swarm/network/light/lightnode.go similarity index 67% rename from swarm/network/lightnode.go rename to swarm/network/light/lightnode.go index 8ee7f5b22e..7bf769d468 100644 --- a/swarm/network/lightnode.go +++ b/swarm/network/light/lightnode.go @@ -1,4 +1,4 @@ -// Copyright 2017 The go-ethereum Authors +// Copyright 2018 The go-ethereum Authors // This file is part of the go-ethereum library.d // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -14,17 +14,18 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package network +package light import ( "errors" + "github.com/ethereum/go-ethereum/swarm/network/stream" "github.com/ethereum/go-ethereum/swarm/storage" ) // RemoteReader implements IncomingStreamer type RemoteSectionReader struct { - db *DbAccess + db *storage.DBAPI start uint64 end uint64 hashes chan []byte @@ -35,7 +36,7 @@ type RemoteSectionReader struct { } // NewRemoteReader is the constructor for RemoteReader -func NewRemoteSectionReader(root []byte, db *DbAccess) *RemoteSectionReader { +func NewRemoteSectionReader(root []byte, db *storage.DBAPI) *RemoteSectionReader { return &RemoteSectionReader{ db: db, root: root, @@ -45,7 +46,7 @@ func NewRemoteSectionReader(root []byte, db *DbAccess) *RemoteSectionReader { } func (r *RemoteSectionReader) NeedData(key []byte) func() { - chunk, created := r.db.getOrCreateRequest(storage.Key(key)) + chunk, created := r.db.GetOrCreateRequest(storage.Key(key)) // TODO: we may want to request from this peer anyway even if the request exists if chunk.ReqC == nil || !created { return nil @@ -58,7 +59,7 @@ func (r *RemoteSectionReader) NeedData(key []byte) func() { } } -func (r *RemoteSectionReader) BatchDone(s string, from uint64, hashes []byte, root []byte) func() (*TakeoverProof, error) { +func (r *RemoteSectionReader) BatchDone(s string, from uint64, hashes []byte, root []byte) func() (*stream.TakeoverProof, error) { r.hashes <- hashes return nil } @@ -75,9 +76,9 @@ func (r *RemoteSectionReader) Read(b []byte) (n int64, err error) { return l, nil } var end bool - for i := 0; !end && i < len(r.currentHashes); i += HashSize { - hash := r.currentHashes[i : i+HashSize] - chunk, err := r.db.get(hash) + for i := 0; !end && i < len(r.currentHashes); i += stream.HashSize { + hash := r.currentHashes[i : i+stream.HashSize] + chunk, err := r.db.Get(hash) if err != nil { return n, err } @@ -96,9 +97,9 @@ func (r *RemoteSectionReader) Read(b []byte) (n int64, err error) { return n, errors.New("aborted") case hashes := <-r.hashes: var i int - for ; !end && i < len(hashes); i += HashSize { - hash := hashes[i : i+HashSize] - chunk, err := r.db.get(hash) + for ; !end && i < len(hashes); i += stream.HashSize { + hash := hashes[i : i+stream.HashSize] + chunk, err := r.db.Get(hash) if err != nil { return n, err } @@ -120,12 +121,12 @@ func (r *RemoteSectionReader) Read(b []byte) (n int64, err error) { type RemoteSectionServer struct { // quit chan struct{} root []byte - db *DbAccess + db *storage.DBAPI r *storage.LazyChunkReader } // NewRemoteReader is the constructor for RemoteReader -func NewRemoteSectionServer(db *DbAccess, r *storage.LazyChunkReader) *RemoteSectionServer { +func NewRemoteSectionServer(db *storage.DBAPI, r *storage.LazyChunkReader) *RemoteSectionServer { return &RemoteSectionServer{ db: db, r: r, @@ -134,7 +135,7 @@ func NewRemoteSectionServer(db *DbAccess, r *storage.LazyChunkReader) *RemoteSec // GetData retrieves the actual chunk from localstore func (s *RemoteSectionServer) GetData(key []byte) []byte { - chunk, err := s.db.get(storage.Key(key)) + chunk, err := s.db.Get(storage.Key(key)) if err != nil { return nil } @@ -142,26 +143,26 @@ func (s *RemoteSectionServer) GetData(key []byte) []byte { } // GetBatch retrieves the next batch of hashes from the dbstore -func (s *RemoteSectionServer) SetNextBatch(from, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) { - if to > from+batchSize { - to = from + batchSize +func (s *RemoteSectionServer) SetNextBatch(from, to uint64) ([]byte, uint64, uint64, *stream.HandoverProof, error) { + if to > from+stream.BatchSize { + to = from + stream.BatchSize } - batch := make([]byte, (to-from)*HashSize) + batch := make([]byte, (to-from)*stream.HashSize) s.r.ReadAt(batch, int64(from)) return batch, from, to, nil, nil } // RegisterRemoteSectionReader registers RemoteSectionReader on light downstream node -func RegisterRemoteSectionReader(s *Streamer, db *DbAccess) { - s.RegisterIncomingStreamer("REMOTE_SECTION", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { +func RegisterRemoteSectionReader(s *stream.Registry, db *storage.DBAPI) { + s.RegisterClientFunc("REMOTE_SECTION", func(p *stream.Peer, t []byte) (stream.Client, error) { return NewRemoteSectionReader(t, db), nil }) } // RegisterRemoteSectionServer registers RemoteSectionServer outgoing streamer on // upstream light server node -func RegisterRemoteSectionServer(s *Streamer, db *DbAccess, rf func([]byte) *storage.LazyChunkReader) { - s.RegisterOutgoingStreamer("REMOTE_SECTION", func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) { +func RegisterRemoteSectionServer(s *stream.Registry, db *storage.DBAPI, rf func([]byte) *storage.LazyChunkReader) { + s.RegisterServerFunc("REMOTE_SECTION", func(p *stream.Peer, t []byte) (stream.Server, error) { r := rf(t) return NewRemoteSectionServer(db, r), nil }) @@ -169,16 +170,16 @@ func RegisterRemoteSectionServer(s *Streamer, db *DbAccess, rf func([]byte) *sto // RegisterRemoteDownloader registers RemoteDownloader incoming streamer // on downstream light node -// func RegisterRemoteDownloader(s *Streamer, db *DbAccess) { -// s.RegisterIncomingStreamer("REMOTE_DOWNLOADER", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { +// func RegisterRemoteDownloader(s *Streamer, db *storage.DBAPI) { +// s.RegisterIncomingStreamer("REMOTE_DOWNLOADER", func(p *stream.Peer, t []byte) (IncomingStreamer, error) { // return NewRemoteDownloader(t, db), nil // }) // } // // // RegisterRemoteDownloadServer registers RemoteDownloadServer outgoing streamer on // // upstream light server node -// func RegisterRemoteDownloadServer(s *Streamer, db *DbAccess, rf func([]byte) *storage.LazyChunkReader) { -// s.RegisterOutgoingStreamer("REMOTE_DOWNLOADER", func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) { +// func RegisterRemoteDownloadServer(s *Streamer, db *storage.DBAPI, rf func([]byte) *storage.LazyChunkReader) { +// s.RegisterOutgoingStreamer("REMOTE_DOWNLOADER", func(p *stream.Peer, t []byte) (OutgoingStreamer, error) { // r := rf(t) // return NewRemoteDownloadServer(db, r), nil // }) diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index 4f5906b4ae..53776e5ba6 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -103,7 +103,6 @@ type BzzConfig struct { // Bzz is the swarm protocol bundle type Bzz struct { - Streamer *Streamer *Hive localAddr *BzzAddr mtx sync.Mutex @@ -115,9 +114,8 @@ type Bzz struct { // * bzz config // * overlay driver // * peer store -func NewBzz(config *BzzConfig, kad Overlay, store StateStore, streamer *Streamer) *Bzz { +func NewBzz(config *BzzConfig, kad Overlay, store StateStore) *Bzz { return &Bzz{ - Streamer: streamer, Hive: NewHive(config.HiveParams, kad, store), localAddr: &BzzAddr{config.OverlayAddr, config.UnderlayAddr}, handshakes: make(map[discover.NodeID]*HandshakeMsg), @@ -143,7 +141,7 @@ func (b *Bzz) NodeInfo() interface{} { // * handshake/hive // * discovery func (b *Bzz) Protocols() []p2p.Protocol { - protocols := []p2p.Protocol{ + return []p2p.Protocol{ { Name: BzzSpec.Name, Version: BzzSpec.Version, @@ -160,17 +158,6 @@ func (b *Bzz) Protocols() []p2p.Protocol { PeerInfo: b.Hive.PeerInfo, }, } - if b.Streamer != nil { - protocols = append(protocols, p2p.Protocol{ - Name: StreamerSpec.Name, - Version: StreamerSpec.Version, - Length: StreamerSpec.Length(), - Run: b.RunProtocol(StreamerSpec, b.Streamer.Run), - NodeInfo: b.Streamer.NodeInfo, - PeerInfo: b.Streamer.PeerInfo, - }) - } - return protocols } // APIs returns the APIs offered by bzz @@ -188,12 +175,12 @@ func (b *Bzz) APIs() []rpc.API { // returns a p2p protocol run function that can be assigned to p2p.Protocol#Run field // arguments: // * p2p protocol spec -// * run function taking bzzPeer as argument +// * run function taking BzzPeer as argument // this run function is meant to block for the duration of the protocol session // on return the session is terminated and the peer is disconnected // the protocol waits for the bzz handshake is negotiated -// the overlay address on the bzzPeer is set from the remote handshake -func (b *Bzz) RunProtocol(spec *protocols.Spec, run func(*bzzPeer) error) func(*p2p.Peer, p2p.MsgReadWriter) error { +// the overlay address on the BzzPeer is set from the remote handshake +func (b *Bzz) RunProtocol(spec *protocols.Spec, run func(*BzzPeer) error) func(*p2p.Peer, p2p.MsgReadWriter) error { return func(p *p2p.Peer, rw p2p.MsgReadWriter) error { // wait for the bzz protocol to perform the handshake handshake, _ := b.GetHandshake(p.ID()) @@ -206,8 +193,8 @@ func (b *Bzz) RunProtocol(spec *protocols.Spec, run func(*bzzPeer) error) func(* if handshake.err != nil { return fmt.Errorf("%08x: %s protocol closed: %v", b.BaseAddr()[:4], spec.Name, handshake.err) } - // the handshake has succeeded so construct the bzzPeer and run the protocol - peer := &bzzPeer{ + // the handshake has succeeded so construct the BzzPeer and run the protocol + peer := &BzzPeer{ Peer: protocols.NewPeer(p, rw, spec), localAddr: b.localAddr, BzzAddr: handshake.peerAddr, @@ -257,9 +244,9 @@ func (b *Bzz) runBzz(p *p2p.Peer, rw p2p.MsgReadWriter) error { return errors.New("received multiple handshakes") } -// bzzPeer is the bzz protocol view of a protocols.Peer (itself an extension of p2p.Peer) +// BzzPeer is the bzz protocol view of a protocols.Peer (itself an extension of p2p.Peer) // implements the Peer interface and all interfaces Peer implements: Addr, OverlayPeer -type bzzPeer struct { +type BzzPeer struct { *protocols.Peer // represents the connection for online peers localAddr *BzzAddr // local Peers address *BzzAddr // remote address -> implements Addr interface = protocols.Peer @@ -267,12 +254,12 @@ type bzzPeer struct { } // Off returns the overlay peer record for offline persistance -func (p *bzzPeer) Off() OverlayAddr { +func (p *BzzPeer) Off() OverlayAddr { return p.BzzAddr } // LastActive returns the time the peer was last active -func (p *bzzPeer) LastActive() time.Time { +func (p *BzzPeer) LastActive() time.Time { return p.lastActive } diff --git a/swarm/network/protocol_test.go b/swarm/network/protocol_test.go index 1d7e165f02..fdabddb1c3 100644 --- a/swarm/network/protocol_test.go +++ b/swarm/network/protocol_test.go @@ -78,16 +78,16 @@ func HandshakeMsgExchange(lhs, rhs *HandshakeMsg, id discover.NodeID) []p2ptest. } } -func newBzzBaseTester(t *testing.T, n int, addr *BzzAddr, spec *protocols.Spec, run func(*bzzPeer) error) *bzzTester { +func newBzzBaseTester(t *testing.T, n int, addr *BzzAddr, spec *protocols.Spec, run func(*BzzPeer) error) *bzzTester { cs := make(map[string]chan bool) - srv := func(p *bzzPeer) error { + srv := func(p *BzzPeer) error { defer close(cs[p.ID().String()]) return run(p) } protocall := func(p *p2p.Peer, rw p2p.MsgReadWriter) error { - return srv(&bzzPeer{ + return srv(&BzzPeer{ Peer: protocols.NewPeer(p, rw, spec), localAddr: addr, BzzAddr: NewAddrFromNodeID(p.ID()), @@ -115,7 +115,7 @@ type bzzTester struct { func newBzzTester(t *testing.T, n int, addr *BzzAddr, pp *p2ptest.TestPeerPool, spec *protocols.Spec, services func(Peer) error) *bzzTester { - extraservices := func(p *bzzPeer) error { + extraservices := func(p *BzzPeer) error { pp.Add(p) defer pp.Remove(p) if services == nil { diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go new file mode 100644 index 0000000000..c46cc47144 --- /dev/null +++ b/swarm/network/stream/common_test.go @@ -0,0 +1,130 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package stream + +import ( + "errors" + "flag" + "io" + "io/ioutil" + "os" + "testing" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/protocols" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + p2ptest "github.com/ethereum/go-ethereum/p2p/testing" + "github.com/ethereum/go-ethereum/swarm/network" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +var ( + adapter = flag.String("adapter", "sim", "type of simulation: sim|socket|exec|docker") + loglevel = flag.Int("loglevel", 2, "verbosity of logs") +) + +var services = adapters.Services{ + "delivery": newDeliveryService, + "syncer": newSyncerService, +} + +func init() { + flag.Parse() + // register the Delivery service which will run as a devp2p + // protocol when using the exec adapter + adapters.RegisterServices(services) + + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(false)))) +} + +var ( + delivery *Delivery + fileHash storage.Key +) + +func mustReadAll(dpa *storage.DPA, hash storage.Key) (int, error) { + r := dpa.Retrieve(fileHash) + buf := make([]byte, 1024) + var n, total int + var err error + for (total == 0 || n > 0) && err == nil { + n, err = r.ReadAt(buf, int64(total)) + total += n + } + if err != nil && err != io.EOF { + return total, err + } + return total, nil +} + +func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Registry, *storage.LocalStore, func(), error) { + // setup + addr := network.RandomAddr() // tested peers peer address + to := network.NewKademlia(addr.OAddr, network.NewKadParams()) + + // temp datadir + datadir, err := ioutil.TempDir("", "streamer") + if err != nil { + return nil, nil, nil, func() {}, err + } + teardown := func() { + os.RemoveAll(datadir) + } + + localStore, err := storage.NewTestLocalStoreForAddr(datadir, addr.Over()) + if err != nil { + return nil, nil, nil, teardown, err + } + + db := storage.NewDBAPI(localStore) + delivery := NewDelivery(to, db) + streamer := NewRegistry(delivery) + run := func(p *p2p.Peer, rw p2p.MsgReadWriter) error { + BzzPeer := &BzzPeer{ + Peer: protocols.NewPeer(p, rw, Spec), + localAddr: addr, + BzzAddr: network.NewAddrFromNodeID(p.ID()), + } + to.On(BzzPeer) + return streamer.Run(BzzPeer) + } + protocolTester := p2ptest.NewProtocolTester(t, network.NewNodeIDFromAddr(addr), 1, run) + + err = waitForPeers(streamer, 1*time.Second) + if err != nil { + return nil, nil, nil, nil, errors.New("timeout: peer is not created") + } + + return protocolTester, streamer, localStore, teardown, nil +} + +func waitForPeers(streamer *Registry, timeout time.Duration) error { + ticker := time.NewTicker(10 * time.Millisecond) + timeoutTimer := time.NewTimer(timeout) + for { + select { + case <-ticker.C: + if len(streamer.peers) > 0 { + return nil + } + case <-timeoutTimer.C: + return errors.New("timeout") + } + } +} diff --git a/swarm/network/requests.go b/swarm/network/stream/delivery.go similarity index 61% rename from swarm/network/requests.go rename to swarm/network/stream/delivery.go index 3495cfceb9..d242dd01f3 100644 --- a/swarm/network/requests.go +++ b/swarm/network/stream/delivery.go @@ -1,4 +1,4 @@ -// Copyright 2016 The go-ethereum Authors +// Copyright 2018 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package network +package stream import ( "errors" @@ -23,51 +23,52 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/storage" ) -const retrieveRequestStream = "RETRIEVE_REQUEST" +const swarmChunkServerStreamName = "RETRIEVE_REQUEST" type Delivery struct { - dbAccess *DbAccess - overlay Overlay + db *storage.DBAPI + overlay network.Overlay receiveC chan *ChunkDeliveryMsg - getPeer func(discover.NodeID) *StreamerPeer + getPeer func(discover.NodeID) *Peer quit chan struct{} } -func NewDelivery(overlay Overlay, dbAccess *DbAccess) *Delivery { - self := &Delivery{ - dbAccess: dbAccess, +func NewDelivery(overlay network.Overlay, db *storage.DBAPI) *Delivery { + d := &Delivery{ + db: db, overlay: overlay, receiveC: make(chan *ChunkDeliveryMsg, 10), } - go self.processReceivedChunks() - return self + go d.processReceivedChunks() + return d } -// RetrieveRequestStreamer implements OutgoingStreamer -type RetrieveRequestStreamer struct { +// SwarmChunkServer implements OutgoingStreamer +type SwarmChunkServer struct { deliveryC chan []byte batchC chan []byte - dbAccess *DbAccess + db *storage.DBAPI currentLen uint64 } -// NewRetrieveRequestStreamer is RetrieveRequestStreamer constructor -func NewRetrieveRequestStreamer(dbAccess *DbAccess) *RetrieveRequestStreamer { - s := &RetrieveRequestStreamer{ +// NewSwarmChunkServer is SwarmChunkServer constructor +func NewSwarmChunkServer(db *storage.DBAPI) *SwarmChunkServer { + s := &SwarmChunkServer{ deliveryC: make(chan []byte), batchC: make(chan []byte), - dbAccess: dbAccess, + db: db, } go s.processDeliveries() return s } // processDeliveries handles delivered chunk hashes -func (s *RetrieveRequestStreamer) processDeliveries() { +func (s *SwarmChunkServer) processDeliveries() { var hashes []byte var batchC chan []byte for { @@ -83,7 +84,7 @@ func (s *RetrieveRequestStreamer) processDeliveries() { } // SetNextBatch -func (s *RetrieveRequestStreamer) SetNextBatch(_, _ uint64) (hashes []byte, from uint64, to uint64, proof *HandoverProof, err error) { +func (s *SwarmChunkServer) SetNextBatch(_, _ uint64) (hashes []byte, from uint64, to uint64, proof *HandoverProof, err error) { hashes = <-s.batchC from = s.currentLen s.currentLen += uint64(len(hashes)) @@ -92,8 +93,8 @@ func (s *RetrieveRequestStreamer) SetNextBatch(_, _ uint64) (hashes []byte, from } // GetData retrives chunk data from db store -func (s *RetrieveRequestStreamer) GetData(key []byte) []byte { - chunk, _ := s.dbAccess.get(storage.Key(key)) +func (s *SwarmChunkServer) GetData(key []byte) []byte { + chunk, _ := s.db.Get(storage.Key(key)) return chunk.SData } @@ -103,16 +104,16 @@ type RetrieveRequestMsg struct { SkipCheck bool } -func (self *Delivery) handleRetrieveRequestMsg(sp *StreamerPeer, req *RetrieveRequestMsg) error { - s, err := sp.getOutgoingStreamer(retrieveRequestStream) +func (d *Delivery) handleRetrieveRequestMsg(sp *Peer, req *RetrieveRequestMsg) error { + s, err := sp.getServer(swarmChunkServerStreamName) if err != nil { return err } - streamer := s.OutgoingStreamer.(*RetrieveRequestStreamer) - chunk, created := self.dbAccess.getOrCreateRequest(req.Key) + streamer := s.Server.(*SwarmChunkServer) + chunk, created := d.db.GetOrCreateRequest(req.Key) if chunk.ReqC != nil { if created { - if err := self.RequestFromPeers(chunk.Key[:], false, sp.ID()); err != nil { + if err := d.RequestFromPeers(chunk.Key[:], false, sp.ID()); err != nil { return nil } } @@ -122,7 +123,7 @@ func (self *Delivery) handleRetrieveRequestMsg(sp *StreamerPeer, req *RetrieveRe select { case <-chunk.ReqC: - case <-self.quit: + case <-d.quit: return case <-t.C: return @@ -149,21 +150,21 @@ type ChunkDeliveryMsg struct { SData []byte // the stored chunk Data (incl size) } -func (self *Delivery) handleChunkDeliveryMsg(req *ChunkDeliveryMsg) error { - chunk, err := self.dbAccess.get(req.Key) +func (d *Delivery) handleChunkDeliveryMsg(req *ChunkDeliveryMsg) error { + chunk, err := d.db.Get(req.Key) if err != nil { return err } - self.receiveC <- req + d.receiveC <- req - log.Trace(fmt.Sprintf("delivery of %v from %v", chunk, self)) + log.Trace(fmt.Sprintf("delivery of %v from %v", chunk, d)) return nil } -func (self *Delivery) processReceivedChunks() { - for req := range self.receiveC { - chunk, err := self.dbAccess.get(req.Key) +func (d *Delivery) processReceivedChunks() { + for req := range d.receiveC { + chunk, err := d.db.Get(req.Key) if err != nil { continue } @@ -171,23 +172,23 @@ func (self *Delivery) processReceivedChunks() { select { case <-chunk.ReqC: default: - self.dbAccess.put(chunk) + d.db.Put(chunk) close(chunk.ReqC) } } } // RequestFromPeers sends a chunk retrieve request to -func (self *Delivery) RequestFromPeers(hash []byte, skipCheck bool, peersToSkip ...discover.NodeID) error { +func (d *Delivery) RequestFromPeers(hash []byte, skipCheck bool, peersToSkip ...discover.NodeID) error { var success bool - self.overlay.EachConn(hash, 255, func(p OverlayConn, po int, nn bool) bool { - spId := p.(Peer).ID() + d.overlay.EachConn(hash, 255, func(p network.OverlayConn, po int, nn bool) bool { + spId := p.(*network.BzzPeer).ID() for _, p := range peersToSkip { if p == spId { return true } } - sp := self.getPeer(spId) + sp := d.getPeer(spId) // TODO: skip light nodes that do not accept retrieve requests err := sp.SendPriority(&RetrieveRequestMsg{ Key: hash, diff --git a/swarm/network/request_test.go b/swarm/network/stream/delivery_test.go similarity index 97% rename from swarm/network/request_test.go rename to swarm/network/stream/delivery_test.go index 8ae1d3c344..317132be61 100644 --- a/swarm/network/request_test.go +++ b/swarm/network/stream/delivery_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 The go-ethereum Authors +// Copyright 2018 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package network +package stream import ( "bytes" @@ -405,8 +405,8 @@ func newDeliveryService(ctx *adapters.ServiceContext) (node.Service, error) { addr := NewAddrFromNodeID(id) kad := NewKademlia(addr.Over(), NewKadParams()) localStore := localStores[nodeCount] - dbAccess := NewDbAccess(localStore.(*storage.LocalStore)) - streamer := NewStreamer(NewDelivery(kad, dbAccess)) + db := NewDBAPI(localStore.(*storage.LocalStore)) + streamer := NewStreamerRegistry(NewDelivery(kad, db)) if nodeCount == 0 { // the delivery service for the pivot node is assigned globally // so that the simulation action call can use it for the @@ -423,13 +423,13 @@ func newDeliveryService(ctx *adapters.ServiceContext) (node.Service, error) { } func (b *testStreamerService) runDelivery(p *p2p.Peer, rw p2p.MsgReadWriter) error { - bzzPeer := &bzzPeer{ + BzzPeer := &BzzPeer{ Peer: protocols.NewPeer(p, rw, StreamerSpec), localAddr: b.addr, BzzAddr: NewAddrFromNodeID(p.ID()), } - b.streamer.delivery.overlay.On(bzzPeer) - defer b.streamer.delivery.overlay.Off(bzzPeer) + b.streamer.delivery.overlay.On(BzzPeer) + defer b.streamer.delivery.overlay.Off(BzzPeer) go func() { // each node Subscribes to each other's retrieveRequestStream // need to wait till an aynchronous process registers the peers in streamer.peers @@ -440,5 +440,5 @@ func (b *testStreamerService) runDelivery(p *p2p.Peer, rw p2p.MsgReadWriter) err log.Warn("error in subscribe", "err", err) } }() - return b.streamer.Run(bzzPeer) + return b.streamer.Run(BzzPeer) } diff --git a/swarm/network/stream/messages.go b/swarm/network/stream/messages.go new file mode 100644 index 0000000000..1e5e281b91 --- /dev/null +++ b/swarm/network/stream/messages.go @@ -0,0 +1,228 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package stream + +import ( + "errors" + "fmt" + "sync" + + "github.com/ethereum/go-ethereum/log" + bv "github.com/ethereum/go-ethereum/swarm/network/bitvector" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +// Handover represents a statement that the upstream peer hands over the stream section +type Handover struct { + Stream string // name of stream + Start, End uint64 // index of hashes + Root []byte // Root hash for indexed segment inclusion proofs +} + +// HandoverProof represents a signed statement that the upstream peer handed over the stream section +type HandoverProof struct { + Sig []byte // Sign(Hash(Serialisation(Handover))) + *Handover +} + +// Takeover represents a statement that downstream peer took over (stored all data) +// handed over +type Takeover Handover + +// TakeoverProof represents a signed statement that the downstream peer took over +// the stream section +type TakeoverProof struct { + Sig []byte // Sign(Hash(Serialisation(Takeover))) + *Takeover +} + +// TakeoverProofMsg is the protocol msg sent by downstream peer +type TakeoverProofMsg TakeoverProof + +// String pretty prints TakeoverProofMsg +func (m TakeoverProofMsg) String() string { + return fmt.Sprintf("Stream: '%v' [%v-%v], Root: %x, Sig: %x", m.Stream, m.Start, m.End, m.Root, m.Sig) +} + +// SubcribeMsg is the protocol msg for requesting a stream(section) +type SubscribeMsg struct { + Stream string + Key []byte + From, To uint64 + Priority uint8 // delivered on priority channel +} + +func (p *Peer) handleSubscribeMsg(req *SubscribeMsg) error { + f, err := p.streamer.GetServerFunc(req.Stream) + if err != nil { + return err + } + s, err := f(p, req.Key) + if err != nil { + return err + } + os, err := p.setServer(req.Stream, req.Key, s, req.Priority) + if err != nil { + return nil + } + log.Debug("received subscription", "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To) + go p.SendOfferedHashes(os, req.From, req.To) + return nil +} + +// OfferedHashesMsg is the protocol msg for offering to hand over a +// stream section +type OfferedHashesMsg struct { + Stream string // name of Stream + Key []byte // subtype or key + From, To uint64 // peer and db-specific entry count + Hashes []byte // stream of hashes (128) + *HandoverProof // HandoverProof +} + +// String pretty prints OfferedHashesMsg +func (m OfferedHashesMsg) String() string { + return fmt.Sprintf("Stream '%v' [%v-%v] (%v)", m.Stream, m.From, m.To, len(m.Hashes)/HashSize) +} + +// handleOfferedHashesMsg protocol msg handler calls the incoming streamer interface +// Filter method +func (p *Peer) handleOfferedHashesMsg(req *OfferedHashesMsg) error { + sk := req.Stream + sk += keyToString(req.Key) + s, err := p.getClient(sk) + if err != nil { + return err + } + hashes := req.Hashes + want, err := bv.New(len(hashes) / HashSize) + if err != nil { + return fmt.Errorf("error initiaising bitvector of length %v: %v", len(hashes)/HashSize, err) + } + wg := sync.WaitGroup{} + for i := 0; i < len(hashes); i += HashSize { + hash := hashes[i : i+HashSize] + if wait := s.NeedData(hash); wait != nil { + want.Set(i/HashSize, true) + wg.Add(1) + // create request and wait until the chunk data arrives and is stored + go func(w func()) { + w() + wg.Done() + }(wait) + } + } + go func() { + wg.Wait() + if tf := s.BatchDone(req.Stream, req.From, hashes, req.Root); tf != nil { + tp, err := tf() + if err != nil { + return + } + p.SendPriority(tp, s.priority) + } + s.next <- struct{}{} + }() + // only send wantedKeysMsg if all missing chunks of the previous batch arrived + // except + if s.live { + s.sessionAt = req.From + } + from, to := s.nextBatch(req.To) + log.Debug("received batch", "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To) + if from == to { + return nil + } + + msg := &WantedHashesMsg{ + Stream: req.Stream, + Key: req.Key, + Want: want.Bytes(), + From: from, + To: to, + } + go func() { + select { + case <-s.next: + case <-s.quit: + return + } + log.Debug("want batch", "stream", msg.Stream, "Key", msg.Key, "from", msg.From, "to", msg.To) + p.SendPriority(msg, s.priority) + }() + return nil +} + +// WantedHashesMsg is the protocol msg data for signaling which hashes +// offered in OfferedHashesMsg downstream peer actually wants sent over +type WantedHashesMsg struct { + Stream string // name of stream + Key []byte // subtype or key + Want []byte // bitvector indicating which keys of the batch needed + From, To uint64 // next interval offset - empty if not to be continued +} + +// String pretty prints WantedHashesMsg +func (m WantedHashesMsg) String() string { + return fmt.Sprintf("Stream '%v', Want: %x, Next: [%v-%v]", m.Stream, m.Want, m.From, m.To) +} + +// handleWantedHashesMsg protocol msg handler +// * sends the next batch of unsynced keys +// * sends the actual data chunks as per WantedHashesMsg +func (p *Peer) handleWantedHashesMsg(req *WantedHashesMsg) error { + log.Debug("received wanted batch", "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To) + s, err := p.getServer(req.Stream + keyToString(req.Key)) + if err != nil { + log.Debug(err.Error()) + return err + } + hashes := s.currentBatch + // launch in go routine since GetBatch blocks until new hashes arrive + go p.SendOfferedHashes(s, req.From, req.To) + l := len(hashes) / HashSize + want, err := bv.NewFromBytes(req.Want, l) + if err != nil { + return fmt.Errorf("error initiaising bitvector of length %v: %v", l, err) + } + for i := 0; i < l; i++ { + if want.Get(i) { + hash := hashes[i*HashSize : (i+1)*HashSize] + data := s.GetData(hash) + if data == nil { + return errors.New("not found") + } + chunk := storage.NewChunk(hash, nil) + chunk.SData = data + if err := p.Deliver(chunk, s.priority); err != nil { + return err + } + } + } + return nil +} + +func (p *Peer) handleTakeoverProofMsg(req *TakeoverProofMsg) error { + _, err := p.getServer(req.Stream) + if err != nil { + return err + } + // store the strongest takeoverproof for the stream in streamer + return nil +} + +type UnsubscribeMsg struct{} diff --git a/swarm/network/stream/peer.go b/swarm/network/stream/peer.go new file mode 100644 index 0000000000..5d2461a3c6 --- /dev/null +++ b/swarm/network/stream/peer.go @@ -0,0 +1,164 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package stream + +import ( + "context" + "fmt" + "sync" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/protocols" + pq "github.com/ethereum/go-ethereum/swarm/network/priorityqueue" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +// Peer is the Peer extention for the streaming protocol +type Peer struct { + *protocols.Peer + streamer *Registry + pq *pq.PriorityQueue + outgoingMu sync.RWMutex + incomingMu sync.RWMutex + servers map[string]*server + clients map[string]*client + quit chan struct{} +} + +// NewPeer is the constructor for Peer +func NewPeer(peer *protocols.Peer, streamer *Registry) *Peer { + p := &Peer{ + Peer: peer, + pq: pq.New(int(PriorityQueue), PriorityQueueCap), + streamer: streamer, + servers: make(map[string]*server), + clients: make(map[string]*client), + quit: make(chan struct{}), + } + ctx, cancel := context.WithCancel(context.Background()) + go p.pq.Run(ctx, func(i interface{}) { p.Send(i) }) + go func() { + <-p.quit + cancel() + }() + return p +} + +// Deliver sends a storeRequestMsg protocol message to the peer +func (p *Peer) Deliver(chunk *storage.Chunk, priority uint8) error { + msg := &ChunkDeliveryMsg{ + Key: chunk.Key, + SData: chunk.SData, + } + return p.pq.Push(nil, msg, int(priority)) +} + +// Deliver sends a storeRequestMsg protocol message to the peer +func (p *Peer) SendPriority(msg interface{}, priority uint8) error { + return p.pq.Push(nil, msg, int(priority)) +} + +// SendOfferedHashes sends OfferedHashesMsg protocol msg +func (p *Peer) SendOfferedHashes(s *server, f, t uint64) error { + hashes, from, to, proof, err := s.SetNextBatch(f, t) + if err != nil { + return err + } + if proof == nil { + proof = &HandoverProof{ + Handover: &Handover{}, + } + } + s.currentBatch = hashes + msg := &OfferedHashesMsg{ + HandoverProof: proof, + Hashes: hashes, + From: from, + To: to, + Stream: s.stream, + Key: s.key, + } + log.Debug("Swarm syncer offer batch", "stream", s.stream, "key", s.key, "len", len(hashes), "from", from, "to", to) + return p.SendPriority(msg, s.priority) +} + +func (p *Peer) getServer(s string) (*server, error) { + p.outgoingMu.RLock() + defer p.outgoingMu.RUnlock() + + server := p.servers[s] + if server == nil { + return nil, fmt.Errorf("server '%v' not provided to peer %v", s, p.ID()) + } + return server, nil +} + +func (p *Peer) getClient(s string) (*client, error) { + p.incomingMu.RLock() + defer p.incomingMu.RUnlock() + + client := p.clients[s] + if client == nil { + return nil, fmt.Errorf("client '%v' not provided to peer %v", s, p.ID()) + } + return client, nil +} + +func (p *Peer) setServer(s string, key []byte, o Server, priority uint8) (*server, error) { + p.outgoingMu.Lock() + defer p.outgoingMu.Unlock() + + sk := s + keyToString(key) + if p.servers[sk] != nil { + return nil, fmt.Errorf("server %v already registered", sk) + } + os := &server{ + Server: o, + priority: priority, + stream: s, + key: key, + } + p.servers[sk] = os + return os, nil +} + +func (p *Peer) setClient(s string, key []byte, i Client, priority uint8, live bool) error { + p.incomingMu.Lock() + defer p.incomingMu.Unlock() + + sk := s + keyToString(key) + if p.clients[sk] != nil { + return fmt.Errorf("client %v already registered", sk) + } + next := make(chan struct{}, 1) + // var intervals *Intervals + // if !live { + // key := s + p.ID().String() + // intervals = NewIntervals(key, p.streamer) + // } + p.clients[sk] = &client{ + Client: i, + // intervals: intervals, + live: live, + priority: priority, + next: next, + stream: s, + key: key, + } + next <- struct{}{} // this is to allow wantedKeysMsg before first batch arrives + return nil +} diff --git a/swarm/network/stream/stream.go b/swarm/network/stream/stream.go new file mode 100644 index 0000000000..469ed9f126 --- /dev/null +++ b/swarm/network/stream/stream.go @@ -0,0 +1,316 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package stream + +import ( + "fmt" + "math" + "sync" + + "github.com/ethereum/go-ethereum/p2p" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/protocols" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +const ( + Low uint8 = iota + Mid + High + Top + PriorityQueue // number of queues + PriorityQueueCap = 3 // queue capacity + HashSize = 32 +) + +// Registry registry for outgoing and incoming streamer constructors +type Registry struct { + clientMu sync.RWMutex + serverMu sync.RWMutex + peersMu sync.RWMutex + serverFuncs map[string]func(*Peer, []byte) (Server, error) + clientFuncs map[string]func(*Peer, []byte) (Client, error) + peers map[discover.NodeID]*Peer + delivery *Delivery +} + +// NewRegistry is Streamer constructor +func NewRegistry(delivery *Delivery) *Registry { + streamer := &Registry{ + serverFuncs: make(map[string]func(*Peer, []byte) (Server, error)), + clientFuncs: make(map[string]func(*Peer, []byte) (Client, error)), + peers: make(map[discover.NodeID]*Peer), + delivery: delivery, + } + delivery.getPeer = streamer.getPeer + streamer.RegisterServerFunc(swarmChunkServerStreamName, func(_ *Peer, t []byte) (Server, error) { + return NewSwarmChunkServer(delivery.db), nil + }) + streamer.RegisterClientFunc(swarmChunkServerStreamName, func(p *Peer, t []byte) (Client, error) { + return NewSwarmSyncerClient(p, delivery.db, nil) + }) + return streamer +} + +// RegisterClient registers an incoming streamer constructor +func (r *Registry) RegisterClientFunc(stream string, f func(*Peer, []byte) (Client, error)) { + r.clientMu.Lock() + defer r.clientMu.Unlock() + + r.clientFuncs[stream] = f +} + +// RegisterServer registers an outgoing streamer constructor +func (r *Registry) RegisterServerFunc(stream string, f func(*Peer, []byte) (Server, error)) { + r.serverMu.Lock() + defer r.serverMu.Unlock() + + r.serverFuncs[stream] = f +} + +// GetClient accessor for incoming streamer constructors +func (r *Registry) GetClientFunc(stream string) (func(*Peer, []byte) (Client, error), error) { + r.clientMu.RLock() + defer r.clientMu.RUnlock() + + f := r.clientFuncs[stream] + if f == nil { + return nil, fmt.Errorf("stream %v not registered", stream) + } + return f, nil +} + +// GetServer accessor for incoming streamer constructors +func (r *Registry) GetServerFunc(stream string) (func(*Peer, []byte) (Server, error), error) { + r.serverMu.RLock() + defer r.serverMu.RUnlock() + + f := r.serverFuncs[stream] + if f == nil { + return nil, fmt.Errorf("stream %v not registered", stream) + } + return f, nil +} + +// Subscribe initiates the streamer +func (r *Registry) Subscribe(peerId discover.NodeID, s string, t []byte, from, to uint64, priority uint8, live bool) error { + f, err := r.GetClientFunc(s) + if err != nil { + return err + } + + peer := r.getPeer(peerId) + if peer == nil { + return fmt.Errorf("peer not found %v", peerId) + } + + is, err := f(peer, t) + if err != nil { + return err + } + err = peer.setClient(s, t, is, priority, live) + if err != nil { + return err + } + + msg := &SubscribeMsg{ + Stream: s, + Key: t, + // Live: live, + From: from, + To: to, + Priority: priority, + } + log.Debug("Subscribe ", "peer", peerId, "stream", s, "key", t, "from", from, "to", to) + + peer.SendPriority(msg, priority) + return nil +} + +func (r *Registry) Retrieve(chunk *storage.Chunk) error { + return r.delivery.RequestFromPeers(chunk.Key[:], false) +} + +func (r *Registry) NodeInfo() interface{} { + return nil +} + +func (r *Registry) PeerInfo(id discover.NodeID) interface{} { + return nil +} + +func (r *Registry) getPeer(peerId discover.NodeID) *Peer { + r.peersMu.RLock() + defer r.peersMu.RUnlock() + + return r.peers[peerId] +} + +func (r *Registry) setPeer(peer *Peer) { + r.peersMu.Lock() + r.peers[peer.ID()] = peer + r.peersMu.Unlock() +} + +func (r *Registry) deletePeer(peer *Peer) { + r.peersMu.Lock() + delete(r.peers, peer.ID()) + r.peersMu.Unlock() +} + +// Run protocol run function +func (r *Registry) Run(p *protocols.Peer) error { + sp := NewPeer(p, r) + // load saved intervals + + r.setPeer(sp) + + defer r.deletePeer(sp) + defer close(sp.quit) + return sp.Run(sp.HandleMsg) +} + +// HandleMsg is the message handler that delegates incoming messages +func (p *Peer) HandleMsg(msg interface{}) error { + switch msg := msg.(type) { + + case *SubscribeMsg: + return p.handleSubscribeMsg(msg) + + case *OfferedHashesMsg: + return p.handleOfferedHashesMsg(msg) + + case *TakeoverProofMsg: + return p.handleTakeoverProofMsg(msg) + + case *WantedHashesMsg: + return p.handleWantedHashesMsg(msg) + + case *ChunkDeliveryMsg: + return p.streamer.delivery.handleChunkDeliveryMsg(msg) + + case *RetrieveRequestMsg: + return p.streamer.delivery.handleRetrieveRequestMsg(p, msg) + + default: + return fmt.Errorf("unknown message type: %T", msg) + } +} + +func keyToString(key []byte) string { + l := len(key) + if l == 0 { + return "" + } + return fmt.Sprintf("%s-%d", string(key[:l-1]), uint8(key[l-1])) +} + +type server struct { + Server + priority uint8 + currentBatch []byte + stream string + key []byte +} + +// Server interface for outgoing peer Streamer +type Server interface { + SetNextBatch(uint64, uint64) (hashes []byte, from uint64, to uint64, proof *HandoverProof, err error) + GetData([]byte) []byte +} + +type client struct { + Client + priority uint8 + sessionAt uint64 + live bool + stream string + key []byte + quit chan struct{} + next chan struct{} +} + +// Client interface for incoming peer Streamer +type Client interface { + NeedData([]byte) func() + BatchDone(string, uint64, []byte, []byte) func() (*TakeoverProof, error) +} + +// NextBatch adjusts the indexes by inspecting the intervals +func (c *client) nextBatch(from uint64) (nextFrom uint64, nextTo uint64) { + var intervals []uint64 + if c.live { + if len(intervals) == 0 { + intervals = []uint64{c.sessionAt, from} + } else { + intervals[1] = from + } + nextFrom = from + } else if from >= c.sessionAt { // history sync complete + intervals = nil + nextFrom = from + nextTo = math.MaxUint64 + } else if len(intervals) > 2 && from >= intervals[2] { // filled a gap in the intervals + intervals = append(intervals[:1], intervals[3:]...) + nextFrom = intervals[1] + if len(intervals) > 2 { + nextTo = intervals[2] + } else { + nextTo = c.sessionAt + } + } else { + nextFrom = from + intervals[1] = from + nextTo = c.sessionAt + } + // b.intervals.set(intervals) + return nextFrom, nextTo +} + +// Spec is the spec of the streamer protocol. +var Spec = &protocols.Spec{ + Name: "stream", + Version: 1, + MaxMsgSize: 10 * 1024 * 1024, + Messages: []interface{}{ + UnsubscribeMsg{}, + OfferedHashesMsg{}, + WantedHashesMsg{}, + TakeoverProofMsg{}, + SubscribeMsg{}, + RetrieveRequestMsg{}, + ChunkDeliveryMsg{}, + }, +} + +func (r *Registry) Protocols() []p2p.Protocol { + return []p2p.Protocol{ + { + Name: Spec.Name, + Version: Spec.Version, + Length: Spec.Length(), + Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error { + peer := protocols.NewPeer(p, rw, Spec) + return r.Run(peer) + }, + NodeInfo: r.NodeInfo, + PeerInfo: r.PeerInfo, + }, + } +} diff --git a/swarm/network/streamer_test.go b/swarm/network/stream/streamer_test.go similarity index 94% rename from swarm/network/streamer_test.go rename to swarm/network/stream/streamer_test.go index 447713a33d..250573ade4 100644 --- a/swarm/network/streamer_test.go +++ b/swarm/network/stream/streamer_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 The go-ethereum Authors +// Copyright 2018 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package network +package stream import ( "bytes" @@ -92,7 +92,7 @@ func TestStreamerDownstreamSubscribeMsgExchange(t *testing.T) { t.Fatal(err) } - streamer.RegisterIncomingStreamer("foo", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { + streamer.RegisterClientFunc("foo", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { return &testIncomingStreamer{ t: t, }, nil @@ -134,7 +134,7 @@ func TestStreamerUpstreamSubscribeMsgExchange(t *testing.T) { t.Fatal(err) } - streamer.RegisterOutgoingStreamer("foo", func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) { + streamer.RegisterServerFunc("foo", func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) { return &testOutgoingStreamer{ t: t, }, nil @@ -188,7 +188,7 @@ func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) { t.Fatal(err) } - streamer.RegisterIncomingStreamer("foo", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { + streamer.RegisterClientFunc("foo", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { return &testIncomingStreamer{ t: t, }, nil diff --git a/swarm/network/syncer.go b/swarm/network/stream/syncer.go similarity index 50% rename from swarm/network/syncer.go rename to swarm/network/stream/syncer.go index ec8a42808e..9523e4e440 100644 --- a/swarm/network/syncer.go +++ b/swarm/network/stream/syncer.go @@ -1,4 +1,4 @@ -// Copyright 2016 The go-ethereum Authors +// Copyright 2018 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package network +package stream import ( "bytes" @@ -29,88 +29,52 @@ import ( ) const ( - batchSize = 2 - // batchSize = 128 + BatchSize = 2 + // BatchSize = 128 ) -// wrapper of db-s to provide mockable custom local chunk store access to syncer -type DbAccess struct { - db *storage.DbStore - loc *storage.LocalStore -} - -func NewDbAccess(loc *storage.LocalStore) *DbAccess { - return &DbAccess{loc.DbStore.(*storage.DbStore), loc} -} - -// to obtain the chunks from key or request db entry only -func (self *DbAccess) get(key storage.Key) (*storage.Chunk, error) { - return self.loc.Get(key) -} - -// current storage counter of chunk db -func (self *DbAccess) currentBucketStorageIndex(po uint8) uint64 { - return self.db.CurrentBucketStorageIndex(po) -} - -// iteration storage counter and proximity order -func (self *DbAccess) iterator(from uint64, to uint64, po uint8, f func(storage.Key, uint64) bool) error { - return self.db.SyncIterator(from, to, po, f) -} - -// to obtain the chunks from key or request db entry only -func (self *DbAccess) getOrCreateRequest(key storage.Key) (*storage.Chunk, bool) { - return self.loc.GetOrCreateRequest(key) -} - -// to obtain the chunks from key or request db entry only -func (self *DbAccess) put(chunk *storage.Chunk) { - self.loc.Put(chunk) -} - -// OutgoingSwarmSyncer implements an OutgoingStreamer for history syncing on bins +// SwarmSyncerServer implements an OutgoingStreamer for history syncing on bins // offered streams: // * live request delivery with or without checkback // * (live/non-live historical) chunk syncing per proximity bin -type OutgoingSwarmSyncer struct { +type SwarmSyncerServer struct { po uint8 - db *DbAccess + db *storage.DBAPI sessionAt uint64 start uint64 } -// NewOutgoingSwarmSyncer is contructor for OutgoingSwarmSyncer -func NewOutgoingSwarmSyncer(live bool, po uint8, db *DbAccess) (*OutgoingSwarmSyncer, error) { - sessionAt := db.currentBucketStorageIndex(po) +// NewSwarmSyncerServer is contructor for SwarmSyncerServer +func NewSwarmSyncerServer(live bool, po uint8, db *storage.DBAPI) (*SwarmSyncerServer, error) { + sessionAt := db.CurrentBucketStorageIndex(po) var start uint64 if live { start = sessionAt } - self := &OutgoingSwarmSyncer{ + return &SwarmSyncerServer{ po: po, db: db, sessionAt: sessionAt, start: start, - } - return self, nil + }, nil } const maxPO = 32 -func RegisterOutgoingSyncer(streamer *Streamer, db *DbAccess) { - streamer.RegisterOutgoingStreamer("SYNC", func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) { +func RegisterSwarmSyncerServer(streamer *Registry, db *storage.DBAPI) { + streamer.RegisterServerFunc("SYNC", func(p *Peer, t []byte) (Server, error) { po := uint8(t[0]) // TODO: make this work for HISTORY too - return NewOutgoingSwarmSyncer(false, po, db) + return NewSwarmSyncerServer(false, po, db) }) - // streamer.RegisterOutgoingStreamer(stream, func(p *StreamerPeer) (OutgoingStreamer, error) { + // streamer.RegisterOutgoingStreamer(stream, func(p *Peer) (OutgoingStreamer, error) { // return NewOutgoingProvableSwarmSyncer(po, db) // }) } // GetSection retrieves the actual chunk from localstore -func (self *OutgoingSwarmSyncer) GetData(key []byte) []byte { - chunk, err := self.db.get(storage.Key(key)) +func (s *SwarmSyncerServer) GetData(key []byte) []byte { + chunk, err := s.db.Get(storage.Key(key)) if err != nil { return nil } @@ -118,23 +82,23 @@ func (self *OutgoingSwarmSyncer) GetData(key []byte) []byte { } // GetBatch retrieves the next batch of hashes from the dbstore -func (self *OutgoingSwarmSyncer) SetNextBatch(from, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) { +func (s *SwarmSyncerServer) SetNextBatch(from, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) { var batch []byte i := 0 if from == 0 { - from = self.start + from = s.start } - if to <= from || from >= self.sessionAt { + if to <= from || from >= s.sessionAt { to = math.MaxUint64 } ticker := time.NewTicker(10 * time.Millisecond) defer ticker.Stop() for range ticker.C { - err := self.db.iterator(from, to, self.po, func(key storage.Key, idx uint64) bool { + err := s.db.Iterator(from, to, s.po, func(key storage.Key, idx uint64) bool { batch = append(batch, key[:]...) i++ to = idx - return i < batchSize + return i < BatchSize }) if err != nil { return nil, 0, 0, nil, err @@ -144,41 +108,40 @@ func (self *OutgoingSwarmSyncer) SetNextBatch(from, to uint64) ([]byte, uint64, } } - log.Debug("Swarm syncer offer batch", "po", self.po, "len", i, "from", from, "to", to, "current store count", self.db.currentBucketStorageIndex(self.po)) + log.Debug("Swarm syncer offer batch", "po", s.po, "len", i, "from", from, "to", to, "current store count", s.db.CurrentBucketStorageIndex(s.po)) return batch, from, to + 1, nil, nil } -// IncomingSwarmSyncer -type IncomingSwarmSyncer struct { +// SwarmSyncerClient +type SwarmSyncerClient struct { sessionAt uint64 nextC chan struct{} sessionRoot storage.Key sessionReader storage.LazySectionReader retrieveC chan *storage.Chunk storeC chan *storage.Chunk - dbAccess *DbAccess + db *storage.DBAPI chunker storage.Chunker currentRoot storage.Key requestFunc func(chunk *storage.Chunk) end, start uint64 } -// NewIncomingSwarmSyncer is a contructor for provable data exchange syncer -func NewIncomingSwarmSyncer(p Peer, dbAccess *DbAccess, chunker storage.Chunker) (*IncomingSwarmSyncer, error) { - self := &IncomingSwarmSyncer{ - dbAccess: dbAccess, - chunker: chunker, - } - return self, nil +// NewSwarmSyncerClient is a contructor for provable data exchange syncer +func NewSwarmSyncerClient(_ *Peer, db *storage.DBAPI, chunker storage.Chunker) (*SwarmSyncerClient, error) { + return &SwarmSyncerClient{ + db: db, + chunker: chunker, + }, nil } // // NewIncomingProvableSwarmSyncer is a contructor for provable data exchange syncer -// func NewIncomingProvableSwarmSyncer(po int, priority int, index uint64, sessionAt uint64, intervals []uint64, sessionRoot storage.Key, chunker *storage.PyramidChunker, store storage.ChunkStore, p Peer) *IncomingSwarmSyncer { +// func NewIncomingProvableSwarmSyncer(po int, priority int, index uint64, sessionAt uint64, intervals []uint64, sessionRoot storage.Key, chunker *storage.PyramidChunker, store storage.ChunkStore, p Peer) *SwarmSyncerClient { // retrieveC := make(storage.Chunk, chunksCap) // RunChunkRequestor(p, retrieveC) // storeC := make(storage.Chunk, chunksCap) // RunChunkStorer(store, storeC) -// self := &IncomingSwarmSyncer{ +// s := &SwarmSyncerClient{ // po: po, // priority: priority, // sessionAt: sessionAt, @@ -191,10 +154,10 @@ func NewIncomingSwarmSyncer(p Peer, dbAccess *DbAccess, chunker storage.Chunker) // retrieveC: retrieveC, // storeC: storeC, // } -// return self +// return s // } -// // StartSyncing is called on the StreamerPeer to start the syncing process +// // StartSyncing is called on the Peer to start the syncing process // // the idea is that it is called only after kademlia is close to healthy // func StartSyncing(s *Streamer, peerId discover.NodeID, po uint8, nn bool) { // lastPO := po @@ -208,15 +171,15 @@ func NewIncomingSwarmSyncer(p Peer, dbAccess *DbAccess, chunker storage.Chunker) // } // } -func RegisterIncomingSyncer(streamer *Streamer, db *DbAccess) { - streamer.RegisterIncomingStreamer("SYNC", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { - return NewIncomingSwarmSyncer(p, db, nil) +func RegisterSwarmSyncerClient(streamer *Registry, db *storage.DBAPI) { + streamer.RegisterClientFunc("SYNC", func(p *Peer, t []byte) (Client, error) { + return NewSwarmSyncerClient(p, db, nil) }) } // NeedData -func (self *IncomingSwarmSyncer) NeedData(key []byte) (wait func()) { - chunk, _ := self.dbAccess.getOrCreateRequest(key) +func (s *SwarmSyncerClient) NeedData(key []byte) (wait func()) { + chunk, _ := s.db.GetOrCreateRequest(key) // TODO: we may want to request from this peer anyway even if the request exists if chunk.ReqC == nil { return nil @@ -226,29 +189,29 @@ func (self *IncomingSwarmSyncer) NeedData(key []byte) (wait func()) { } // BatchDone -func (self *IncomingSwarmSyncer) BatchDone(s string, from uint64, hashes []byte, root []byte) func() (*TakeoverProof, error) { - if self.chunker != nil { - return func() (*TakeoverProof, error) { return self.TakeoverProof(s, from, hashes, root) } +func (s *SwarmSyncerClient) BatchDone(streamName string, from uint64, hashes []byte, root []byte) func() (*TakeoverProof, error) { + if s.chunker != nil { + return func() (*TakeoverProof, error) { return s.TakeoverProof(streamName, from, hashes, root) } } return nil } -func (self *IncomingSwarmSyncer) TakeoverProof(s string, from uint64, hashes []byte, root storage.Key) (*TakeoverProof, error) { +func (s *SwarmSyncerClient) TakeoverProof(streamName string, from uint64, hashes []byte, root storage.Key) (*TakeoverProof, error) { // for provable syncer currentRoot is non-zero length - if self.chunker != nil { - if from > self.sessionAt { // for live syncing currentRoot is always updated - //expRoot, err := self.chunker.Append(self.currentRoot, bytes.NewReader(hashes), self.retrieveC, self.storeC) - expRoot, _, err := self.chunker.Append(self.currentRoot, bytes.NewReader(hashes), self.retrieveC) + if s.chunker != nil { + if from > s.sessionAt { // for live syncing currentRoot is always updated + //expRoot, err := s.chunker.Append(s.currentRoot, bytes.NewReader(hashes), s.retrieveC, s.storeC) + expRoot, _, err := s.chunker.Append(s.currentRoot, bytes.NewReader(hashes), s.retrieveC) if err != nil { return nil, err } if !bytes.Equal(root, expRoot) { return nil, fmt.Errorf("HandoverProof mismatch") } - self.currentRoot = root + s.currentRoot = root } else { expHashes := make([]byte, len(hashes)) - _, err := self.sessionReader.ReadAt(expHashes, int64(self.end*HashSize)) + _, err := s.sessionReader.ReadAt(expHashes, int64(s.end*HashSize)) if err != nil && err != io.EOF { return nil, err } @@ -258,12 +221,12 @@ func (self *IncomingSwarmSyncer) TakeoverProof(s string, from uint64, hashes []b } return nil, nil } - self.end += uint64(len(hashes)) / HashSize + s.end += uint64(len(hashes)) / HashSize takeover := &Takeover{ - Stream: s, - // Key: self.Key, - Start: self.start, - End: self.end, + Stream: streamName, + // Key: s.Key, + Start: s.start, + End: s.end, Root: root, } // serialise and sign diff --git a/swarm/network/syncer_test.go b/swarm/network/stream/syncer_test.go similarity index 85% rename from swarm/network/syncer_test.go rename to swarm/network/stream/syncer_test.go index 893be367f5..c1a9bd6fac 100644 --- a/swarm/network/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 The go-ethereum Authors +// Copyright 2018 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package network +package stream import ( "context" @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/protocols" "github.com/ethereum/go-ethereum/p2p/simulations" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/storage" ) @@ -74,10 +75,10 @@ func testSyncBetweenNodes(nodes, conns, size int, skipCheck bool, po uint8) func } check := func(net *simulations.Network, dpa *storage.DPA) func(ctx context.Context, id discover.NodeID) (bool, error) { - dbAccesses := make([]*DbAccess, nodes) + dbs := make([]*storage.DBAPI, nodes) for i := 0; i < nodes; i++ { - dbAccesses[i] = NewDbAccess(localStores[i].(*storage.LocalStore)) + dbs[i] = NewDbAccess(localStores[i].(*storage.LocalStore)) } return func(ctx context.Context, id discover.NodeID) (bool, error) { if id != net.Nodes[0].ID() { @@ -91,8 +92,8 @@ func testSyncBetweenNodes(nodes, conns, size int, skipCheck bool, po uint8) func var found, total int for i := 1; i < nodes; i++ { - dbAccesses[i].iterator(0, math.MaxUint64, po, func(key storage.Key, index uint64) bool { - _, err := dbAccesses[0].get(key) + dbs[i].iterator(0, math.MaxUint64, po, func(key storage.Key, index uint64) bool { + _, err := dbs[0].get(key) if err == nil { found++ } @@ -105,7 +106,7 @@ func testSyncBetweenNodes(nodes, conns, size int, skipCheck bool, po uint8) func } } toAddr := func(id discover.NodeID) *BzzAddr { - addr := NewAddrFromNodeID(id) + addr := network.NewAddrFromNodeID(id) addr.OAddr[0] = byte(0) return addr } @@ -123,15 +124,15 @@ func testSyncBetweenNodes(nodes, conns, size int, skipCheck bool, po uint8) func func newSyncerService(ctx *adapters.ServiceContext) (node.Service, error) { id := ctx.Config.ID - addr := NewAddrFromNodeID(id) + addr := network.NewAddrFromNodeID(id) // for the test we make all peers share 8 bits so that syncing full bins make sense addr.OAddr[0] = byte(0) kad := NewKademlia(addr.Over(), NewKadParams()) localStore := localStores[nodeCount] - dbAccess := NewDbAccess(localStore.(*storage.LocalStore)) - streamer := NewStreamer(NewDelivery(kad, dbAccess)) - RegisterIncomingSyncer(streamer, dbAccess) - RegisterOutgoingSyncer(streamer, dbAccess) + db := NewDbAccess(localStore.(*storage.LocalStore)) + streamer := NewRegistry(NewDelivery(kad, db)) + RegisterIncomingSyncer(streamer, db) + RegisterOutgoingSyncer(streamer, db) self := &testStreamerService{ index: nodeCount, @@ -144,15 +145,15 @@ func newSyncerService(ctx *adapters.ServiceContext) (node.Service, error) { } func (b *testStreamerService) runSyncer(p *p2p.Peer, rw p2p.MsgReadWriter) error { - addr := NewAddrFromNodeID(p.ID()) + addr := network.NewAddrFromNodeID(p.ID()) addr.OAddr[0] = byte(0) - bzzPeer := &bzzPeer{ - Peer: protocols.NewPeer(p, rw, StreamerSpec), + BzzPeer := &BzzPeer{ + Peer: protocols.NewPeer(p, rw, Spec), localAddr: b.addr, BzzAddr: addr, } - b.streamer.delivery.overlay.On(bzzPeer) - defer b.streamer.delivery.overlay.Off(bzzPeer) + b.streamer.delivery.overlay.On(BzzPeer) + defer b.streamer.delivery.overlay.Off(BzzPeer) // if len(addr) > b.index+1 && bytes.Equal(addrs[b.index+1], addr) { go func() { // each node Subscribes to each other's retrieveRequestStream @@ -164,5 +165,5 @@ func (b *testStreamerService) runSyncer(p *p2p.Peer, rw p2p.MsgReadWriter) error } }() // } - return b.streamer.Run(bzzPeer) + return b.streamer.Run(BzzPeer) } diff --git a/swarm/network/streamer_common_test.go b/swarm/network/stream/testing/testing.go similarity index 57% rename from swarm/network/streamer_common_test.go rename to swarm/network/stream/testing/testing.go index 7690ed75c8..e0da4d0336 100644 --- a/swarm/network/streamer_common_test.go +++ b/swarm/network/stream/testing/testing.go @@ -1,4 +1,4 @@ -// Copyright 2016 The go-ethereum Authors +// Copyright 2018 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -14,14 +14,12 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package network +package testing import ( "context" "errors" - "flag" "fmt" - "io" "io/ioutil" "math/rand" "os" @@ -33,44 +31,23 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" - "github.com/ethereum/go-ethereum/p2p/protocols" "github.com/ethereum/go-ethereum/p2p/simulations" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" - p2ptest "github.com/ethereum/go-ethereum/p2p/testing" "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/swarm/network" + "github.com/ethereum/go-ethereum/swarm/network/stream" "github.com/ethereum/go-ethereum/swarm/storage" ) -var services = adapters.Services{ - "delivery": newDeliveryService, - "syncer": newSyncerService, -} - var ( - adapter = flag.String("adapter", "sim", "type of simulation: sim|socket|exec|docker") - loglevel = flag.Int("loglevel", 2, "verbosity of logs") + LocalStores []storage.ChunkStore + Addrs []network.Addr + NodeCount int ) -func init() { - flag.Parse() - // register the Delivery service which will run as a devp2p - // protocol when using the exec adapter - adapters.RegisterServices(services) - - log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(false)))) -} - -var ( - delivery *Delivery - localStores []storage.ChunkStore - addrs []Addr - fileHash storage.Key - nodeCount int -) - -func setLocalStores(addrs ...Addr) (func(), error) { +func setLocalStores(addrs ...network.Addr) (func(), error) { var datadirs []string - localStores = make([]storage.ChunkStore, len(addrs)) + LocalStores = make([]storage.ChunkStore, len(addrs)) var err error for i, addr := range addrs { // TODO: remove temp datadir after test @@ -85,7 +62,7 @@ func setLocalStores(addrs ...Addr) (func(), error) { break } datadirs = append(datadirs, datadir) - localStores[i] = localStore + LocalStores[i] = localStore } teardown := func() { for _, datadir := range datadirs { @@ -95,27 +72,12 @@ func setLocalStores(addrs ...Addr) (func(), error) { return teardown, err } -func mustReadAll(dpa *storage.DPA, hash storage.Key) (int, error) { - r := dpa.Retrieve(fileHash) - buf := make([]byte, 1024) - var n, total int - var err error - for (total == 0 || n > 0) && err == nil { - n, err = r.ReadAt(buf, int64(total)) - total += n - } - if err != nil && err != io.EOF { - return total, err - } - return total, nil -} - -func testSimulation(t *testing.T, simf func(adapters.NodeAdapter) (*simulations.StepResult, error)) { +func testSimulation(t *testing.T, services adapters.Services, adapter string, simf func(adapters.NodeAdapter) (*simulations.StepResult, error)) { var err error var result *simulations.StepResult startedAt := time.Now() - switch *adapter { + switch adapter { case "sim": t.Logf("simadapter") result, err = simf(adapters.NewSimAdapter(services)) @@ -158,7 +120,7 @@ func testSimulation(t *testing.T, simf func(adapters.NodeAdapter) (*simulations. t.Logf("Setup: %s, shutdown: %s", result.StartedAt.Sub(startedAt), finishedAt.Sub(result.FinishedAt)) } -func runSimulation(nodes, conns int, serviceName string, toAddr func(discover.NodeID) *BzzAddr, action func(*simulations.Network) func(context.Context) error, trigger func(*simulations.Network) chan discover.NodeID, check func(*simulations.Network, *storage.DPA) func(context.Context, discover.NodeID) (bool, error), adapter adapters.NodeAdapter) (*simulations.StepResult, error) { +func runSimulation(nodes, conns int, serviceName string, toAddr func(discover.NodeID) *network.BzzAddr, action func(*simulations.Network) func(context.Context) error, trigger func(*simulations.Network) chan discover.NodeID, check func(*simulations.Network, *storage.DPA) func(context.Context, discover.NodeID) (bool, error), adapter adapters.NodeAdapter) (*simulations.StepResult, error) { // create network net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{ ID: "0", @@ -166,8 +128,8 @@ func runSimulation(nodes, conns int, serviceName string, toAddr func(discover.No }) defer net.Shutdown() ids := make([]discover.NodeID, nodes) - nodeCount = 0 - addrs = make([]Addr, nodes) + NodeCount = 0 + Addrs = make([]network.Addr, nodes) // start nodes for i := 0; i < nodes; i++ { node, err := net.NewNode() @@ -175,10 +137,10 @@ func runSimulation(nodes, conns int, serviceName string, toAddr func(discover.No return nil, fmt.Errorf("error creating node: %s", err) } ids[i] = node.ID() - addrs[i] = toAddr(ids[i]) + Addrs[i] = toAddr(ids[i]) } // set nodes number of localstores globally available - teardown, err := setLocalStores(addrs...) + teardown, err := setLocalStores(Addrs...) defer teardown() if err != nil { return nil, err @@ -212,11 +174,11 @@ func runSimulation(nodes, conns int, serviceName string, toAddr func(discover.No } wg.Wait() - log.Debug(fmt.Sprintf("nodes: %v", len(addrs))) + log.Debug(fmt.Sprintf("nodes: %v", len(Addrs))) // create an only locally retrieving dpa for the pivot node to test // if retriee requests have arrived - dpa := storage.NewDPA(localStores[0], storage.NewChunkerParams()) + dpa := storage.NewDPA(LocalStores[0], storage.NewChunkerParams()) dpa.Start() defer dpa.Stop() timeout := 300 * time.Second @@ -233,47 +195,6 @@ func runSimulation(nodes, conns int, serviceName string, toAddr func(discover.No return result, nil } -func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Streamer, *storage.LocalStore, func(), error) { - // setup - addr := RandomAddr() // tested peers peer address - to := NewKademlia(addr.OAddr, NewKadParams()) - - // temp datadir - datadir, err := ioutil.TempDir("", "streamer") - if err != nil { - return nil, nil, nil, func() {}, err - } - teardown := func() { - os.RemoveAll(datadir) - } - - localStore, err := storage.NewTestLocalStoreForAddr(datadir, addr.Over()) - if err != nil { - return nil, nil, nil, teardown, err - } - - dbAccess := NewDbAccess(localStore) - delivery := NewDelivery(to, dbAccess) - streamer := NewStreamer(delivery) - run := func(p *p2p.Peer, rw p2p.MsgReadWriter) error { - bzzPeer := &bzzPeer{ - Peer: protocols.NewPeer(p, rw, StreamerSpec), - localAddr: addr, - BzzAddr: NewAddrFromNodeID(p.ID()), - } - to.On(bzzPeer) - return streamer.Run(bzzPeer) - } - protocolTester := p2ptest.NewProtocolTester(t, NewNodeIDFromAddr(addr), 1, run) - - err = waitForPeers(streamer, 1*time.Second) - if err != nil { - return nil, nil, nil, nil, errors.New("timeout: peer is not created") - } - - return protocolTester, streamer, localStore, teardown, nil -} - type roundRobinStore struct { index uint32 stores []storage.ChunkStore @@ -301,34 +222,24 @@ func (rrs *roundRobinStore) Close() { } } -func waitForPeers(streamer *Streamer, timeout time.Duration) error { - ticker := time.NewTicker(10 * time.Millisecond) - timeoutTimer := time.NewTimer(timeout) - for { - select { - case <-ticker.C: - if len(streamer.peers) > 0 { - return nil - } - case <-timeoutTimer.C: - return errors.New("timeout") - } - } -} - -type testStreamerService struct { +type TestStreamerService struct { index int - addr *BzzAddr - streamer *Streamer - run func(p *p2p.Peer, rw p2p.MsgReadWriter) error + addr *network.BzzAddr + streamer *stream.Registry + run func(s *TestStreamerService, p *p2p.Peer, rw p2p.MsgReadWriter) error } -func (tds *testStreamerService) Protocols() []p2p.Protocol { +func NewTestStreamerService(run func(s *TestStreamerService, p *p2p.Peer, rw p2p.MsgReadWriter) error) TestStreamerService { + t := &TestStreamerService{} + t.run = run +} + +func (tds *TestStreamerService) Protocols() []p2p.Protocol { return []p2p.Protocol{ { - Name: StreamerSpec.Name, - Version: StreamerSpec.Version, - Length: StreamerSpec.Length(), + Name: stream.Spec.Name, + Version: stream.Spec.Version, + Length: stream.Spec.Length(), Run: tds.run, // NodeInfo: , // PeerInfo: , @@ -336,14 +247,14 @@ func (tds *testStreamerService) Protocols() []p2p.Protocol { } } -func (b *testStreamerService) APIs() []rpc.API { +func (b *TestStreamerService) APIs() []rpc.API { return []rpc.API{} } -func (b *testStreamerService) Start(server *p2p.Server) error { +func (b *TestStreamerService) Start(server *p2p.Server) error { return nil } -func (b *testStreamerService) Stop() error { +func (b *TestStreamerService) Stop() error { return nil } diff --git a/swarm/network/streamer.go b/swarm/network/streamer.go deleted file mode 100644 index b0e8b9eb3b..0000000000 --- a/swarm/network/streamer.go +++ /dev/null @@ -1,630 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package network - -import ( - "context" - "errors" - "fmt" - "math" - "sync" - - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/p2p/discover" - "github.com/ethereum/go-ethereum/p2p/protocols" - bv "github.com/ethereum/go-ethereum/swarm/network/bitvector" - pq "github.com/ethereum/go-ethereum/swarm/network/priorityqueue" - "github.com/ethereum/go-ethereum/swarm/storage" -) - -const ( - Low uint8 = iota - Mid - High - Top - PriorityQueue // number of queues - PriorityQueueCap = 3 // queue capacity - HashSize = 32 -) - -// Handover represents a statement that the upstream peer hands over the stream section -type Handover struct { - Stream string // name of stream - Start, End uint64 // index of hashes - Root []byte // Root hash for indexed segment inclusion proofs -} - -// HandoverProof represents a signed statement that the upstream peer handed over the stream section -type HandoverProof struct { - Sig []byte // Sign(Hash(Serialisation(Handover))) - *Handover -} - -// Takeover represents a statement that downstream peer took over (stored all data) -// handed over -type Takeover Handover - -// TakeoverProof represents a signed statement that the downstream peer took over -// the stream section -type TakeoverProof struct { - Sig []byte // Sign(Hash(Serialisation(Takeover))) - *Takeover -} - -// TakeoverProofMsg is the protocol msg sent by downstream peer -type TakeoverProofMsg TakeoverProof - -// String pretty prints TakeoverProofMsg -func (self TakeoverProofMsg) String() string { - return fmt.Sprintf("Stream: '%v' [%v-%v], Root: %x, Sig: %x", self.Stream, self.Start, self.End, self.Root, self.Sig) -} - -// SubcribeMsg is the protocol msg for requesting a stream(section) -type SubscribeMsg struct { - Stream string - Key []byte - From, To uint64 - Priority uint8 // delivered on priority channel -} - -// OfferedHashesMsg is the protocol msg for offering to hand over a -// stream section -type OfferedHashesMsg struct { - Stream string // name of Stream - Key []byte // subtype or key - From, To uint64 // peer and db-specific entry count - Hashes []byte // stream of hashes (128) - *HandoverProof // HandoverProof -} - -// String pretty prints OfferedHashesMsg -func (self OfferedHashesMsg) String() string { - return fmt.Sprintf("Stream '%v' [%v-%v] (%v)", self.Stream, self.From, self.To, len(self.Hashes)/HashSize) -} - -// WantedHashesMsg is the protocol msg data for signaling which hashes -// offered in OfferedHashesMsg downstream peer actually wants sent over -type WantedHashesMsg struct { - Stream string // name of stream - Key []byte // subtype or key - Want []byte // bitvector indicating which keys of the batch needed - From, To uint64 // next interval offset - empty if not to be continued -} - -// String pretty prints WantedHashesMsg -func (self WantedHashesMsg) String() string { - return fmt.Sprintf("Stream '%v', Want: %x, Next: [%v-%v]", self.Stream, self.Want, self.From, self.To) -} - -func keyToString(key []byte) string { - l := len(key) - if l == 0 { - return "" - } - return fmt.Sprintf("%s-%d", string(key[:l-1]), uint8(key[l-1])) -} - -// Streamer registry for outgoing and incoming streamer constructors -type Streamer struct { - incomingLock sync.RWMutex - outgoingLock sync.RWMutex - peersLock sync.RWMutex - outgoing map[string]func(*StreamerPeer, []byte) (OutgoingStreamer, error) - incoming map[string]func(*StreamerPeer, []byte) (IncomingStreamer, error) - peers map[discover.NodeID]*StreamerPeer - delivery *Delivery -} - -// NewStreamer is Streamer constructor -func NewStreamer(delivery *Delivery) *Streamer { - streamer := &Streamer{ - outgoing: make(map[string]func(*StreamerPeer, []byte) (OutgoingStreamer, error)), - incoming: make(map[string]func(*StreamerPeer, []byte) (IncomingStreamer, error)), - peers: make(map[discover.NodeID]*StreamerPeer), - delivery: delivery, - } - delivery.getPeer = streamer.getPeer - streamer.RegisterOutgoingStreamer(retrieveRequestStream, func(_ *StreamerPeer, t []byte) (OutgoingStreamer, error) { - return NewRetrieveRequestStreamer(delivery.dbAccess), nil - }) - streamer.RegisterIncomingStreamer(retrieveRequestStream, func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { - return NewIncomingSwarmSyncer(p, delivery.dbAccess, nil) - }) - return streamer -} - -func (self *Streamer) Retrieve(chunk *storage.Chunk) error { - return self.delivery.RequestFromPeers(chunk.Key[:], false) -} - -// RegisterIncomingStreamer registers an incoming streamer constructor -func (self *Streamer) RegisterIncomingStreamer(stream string, f func(*StreamerPeer, []byte) (IncomingStreamer, error)) { - self.incomingLock.Lock() - defer self.incomingLock.Unlock() - self.incoming[stream] = f -} - -// RegisterOutgoingStreamer registers an outgoing streamer constructor -func (self *Streamer) RegisterOutgoingStreamer(stream string, f func(*StreamerPeer, []byte) (OutgoingStreamer, error)) { - self.outgoingLock.Lock() - defer self.outgoingLock.Unlock() - self.outgoing[stream] = f -} - -// GetIncomingStreamer accessor for incoming streamer constructors -func (self *Streamer) GetIncomingStreamer(stream string) (func(*StreamerPeer, []byte) (IncomingStreamer, error), error) { - self.incomingLock.RLock() - defer self.incomingLock.RUnlock() - f := self.incoming[stream] - if f == nil { - return nil, fmt.Errorf("stream %v not registered", stream) - } - return f, nil -} - -// GetOutgoingStreamer accessor for incoming streamer constructors -func (self *Streamer) GetOutgoingStreamer(stream string) (func(*StreamerPeer, []byte) (OutgoingStreamer, error), error) { - self.outgoingLock.RLock() - defer self.outgoingLock.RUnlock() - f := self.outgoing[stream] - if f == nil { - return nil, fmt.Errorf("stream %v not registered", stream) - } - return f, nil -} - -func (self *Streamer) NodeInfo() interface{} { - return nil -} - -func (self *Streamer) PeerInfo(id discover.NodeID) interface{} { - return nil -} - -type outgoingStreamer struct { - OutgoingStreamer - priority uint8 - currentBatch []byte - stream string - key []byte -} - -// OutgoingStreamer interface for outgoing peer Streamer -type OutgoingStreamer interface { - SetNextBatch(uint64, uint64) (hashes []byte, from uint64, to uint64, proof *HandoverProof, err error) - GetData([]byte) []byte -} - -type incomingStreamer struct { - IncomingStreamer - priority uint8 - sessionAt uint64 - live bool - stream string - key []byte - quit chan struct{} - next chan struct{} -} - -// IncomingStreamer interface for incoming peer Streamer -type IncomingStreamer interface { - NeedData([]byte) func() - BatchDone(string, uint64, []byte, []byte) func() (*TakeoverProof, error) -} - -// StreamerPeer is the Peer extention for the streaming protocol -type StreamerPeer struct { - Peer - streamer *Streamer - pq *pq.PriorityQueue - //netStore storage.ChunkStore - outgoingLock sync.RWMutex - incomingLock sync.RWMutex - outgoing map[string]*outgoingStreamer - incoming map[string]*incomingStreamer - quit chan struct{} -} - -// NewStreamerPeer is the constructor for StreamerPeer -func NewStreamerPeer(p Peer, streamer *Streamer) *StreamerPeer { - self := &StreamerPeer{ - Peer: p, - pq: pq.New(int(PriorityQueue), PriorityQueueCap), - streamer: streamer, - outgoing: make(map[string]*outgoingStreamer), - incoming: make(map[string]*incomingStreamer), - quit: make(chan struct{}), - } - ctx, cancel := context.WithCancel(context.Background()) - go self.pq.Run(ctx, func(i interface{}) { p.Send(i) }) - go func() { - <-self.quit - cancel() - }() - return self -} - -func (self *Streamer) getPeer(peerId discover.NodeID) *StreamerPeer { - self.peersLock.RLock() - defer self.peersLock.RUnlock() - return self.peers[peerId] -} - -func (self *Streamer) setPeer(peer *StreamerPeer) { - self.peersLock.Lock() - self.peers[peer.ID()] = peer - self.peersLock.Unlock() -} - -func (self *Streamer) deletePeer(peer *StreamerPeer) { - self.peersLock.Lock() - delete(self.peers, peer.ID()) - self.peersLock.Unlock() -} - -func (self *StreamerPeer) getOutgoingStreamer(s string) (*outgoingStreamer, error) { - self.outgoingLock.RLock() - defer self.outgoingLock.RUnlock() - streamer := self.outgoing[s] - if streamer == nil { - return nil, fmt.Errorf("outgoing stream '%v' not provided to peer %v", s, self.ID()) - } - return streamer, nil -} - -func (self *StreamerPeer) getIncomingStreamer(s string) (*incomingStreamer, error) { - self.incomingLock.RLock() - defer self.incomingLock.RUnlock() - streamer := self.incoming[s] - if streamer == nil { - return nil, fmt.Errorf("incoming stream '%v' not provided to peer %v", s, self.ID()) - } - return streamer, nil -} - -func (self *StreamerPeer) setOutgoingStreamer(s string, key []byte, o OutgoingStreamer, priority uint8) (*outgoingStreamer, error) { - self.outgoingLock.Lock() - defer self.outgoingLock.Unlock() - sk := s + keyToString(key) - if self.outgoing[sk] != nil { - return nil, fmt.Errorf("stream %v already registered", sk) - } - os := &outgoingStreamer{ - OutgoingStreamer: o, - priority: priority, - stream: s, - key: key, - } - self.outgoing[sk] = os - return os, nil -} - -func (self *StreamerPeer) setIncomingStreamer(s string, key []byte, i IncomingStreamer, priority uint8, live bool) error { - self.incomingLock.Lock() - defer self.incomingLock.Unlock() - - sk := s + keyToString(key) - if self.incoming[sk] != nil { - return fmt.Errorf("stream %v already registered", sk) - } - next := make(chan struct{}, 1) - // var intervals *Intervals - // if !live { - // key := s + self.ID().String() - // intervals = NewIntervals(key, self.streamer) - // } - self.incoming[sk] = &incomingStreamer{ - IncomingStreamer: i, - // intervals: intervals, - live: live, - priority: priority, - next: next, - stream: s, - key: key, - } - next <- struct{}{} // this is to allow wantedKeysMsg before first batch arrives - return nil -} - -// NextBatch adjusts the indexes by inspecting the intervals -func (self *incomingStreamer) nextBatch(from uint64) (nextFrom uint64, nextTo uint64) { - var intervals []uint64 - if self.live { - if len(intervals) == 0 { - intervals = []uint64{self.sessionAt, from} - } else { - intervals[1] = from - } - nextFrom = from - } else if from >= self.sessionAt { // history sync complete - intervals = nil - nextFrom = from - nextTo = math.MaxUint64 - } else if len(intervals) > 2 && from >= intervals[2] { // filled a gap in the intervals - intervals = append(intervals[:1], intervals[3:]...) - nextFrom = intervals[1] - if len(intervals) > 2 { - nextTo = intervals[2] - } else { - nextTo = self.sessionAt - } - } else { - nextFrom = from - intervals[1] = from - nextTo = self.sessionAt - } - // self.intervals.set(intervals) - return nextFrom, nextTo -} - -// Subscribe initiates the streamer -func (self *Streamer) Subscribe(peerId discover.NodeID, s string, t []byte, from, to uint64, priority uint8, live bool) error { - f, err := self.GetIncomingStreamer(s) - if err != nil { - return err - } - - peer := self.getPeer(peerId) - if peer == nil { - return fmt.Errorf("peer not found %v", peerId) - } - - is, err := f(peer, t) - if err != nil { - return err - } - err = peer.setIncomingStreamer(s, t, is, priority, live) - if err != nil { - return err - } - - msg := &SubscribeMsg{ - Stream: s, - Key: t, - // Live: live, - From: from, - To: to, - Priority: priority, - } - log.Debug("Subscribe ", "peer", peerId, "stream", s, "key", t, "from", from, "to", to) - - peer.SendPriority(msg, priority) - return nil -} - -func (self *StreamerPeer) handleSubscribeMsg(req *SubscribeMsg) error { - f, err := self.streamer.GetOutgoingStreamer(req.Stream) - if err != nil { - return err - } - s, err := f(self, req.Key) - if err != nil { - return err - } - os, err := self.setOutgoingStreamer(req.Stream, req.Key, s, req.Priority) - if err != nil { - return nil - } - log.Debug("received subscription", "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To) - go self.SendOfferedHashes(os, req.From, req.To) - return nil -} - -// handleOfferedHashesMsg protocol msg handler calls the incoming streamer interface -// Filter method -func (self *StreamerPeer) handleOfferedHashesMsg(req *OfferedHashesMsg) error { - sk := req.Stream - sk += keyToString(req.Key) - s, err := self.getIncomingStreamer(sk) - if err != nil { - return err - } - hashes := req.Hashes - want, err := bv.New(len(hashes) / HashSize) - if err != nil { - return fmt.Errorf("error initiaising bitvector of length %v: %v", len(hashes)/HashSize, err) - } - wg := sync.WaitGroup{} - for i := 0; i < len(hashes); i += HashSize { - hash := hashes[i : i+HashSize] - if wait := s.NeedData(hash); wait != nil { - want.Set(i/HashSize, true) - wg.Add(1) - // create request and wait until the chunk data arrives and is stored - go func(w func()) { - w() - wg.Done() - }(wait) - } - } - go func() { - wg.Wait() - if tf := s.BatchDone(req.Stream, req.From, hashes, req.Root); tf != nil { - tp, err := tf() - if err != nil { - return - } - self.SendPriority(tp, s.priority) - } - s.next <- struct{}{} - }() - // only send wantedKeysMsg if all missing chunks of the previous batch arrived - // except - if s.live { - s.sessionAt = req.From - } - from, to := s.nextBatch(req.To) - log.Debug("received batch", "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To) - if from == to { - return nil - } - - msg := &WantedHashesMsg{ - Stream: req.Stream, - Key: req.Key, - Want: want.Bytes(), - From: from, - To: to, - } - go func() { - select { - case <-s.next: - case <-s.quit: - return - } - log.Debug("want batch", "stream", msg.Stream, "Key", msg.Key, "from", msg.From, "to", msg.To) - self.SendPriority(msg, s.priority) - }() - return nil -} - -// handleWantedHashesMsg protocol msg handler -// * sends the next batch of unsynced keys -// * sends the actual data chunks as per WantedHashesMsg -func (self *StreamerPeer) handleWantedHashesMsg(req *WantedHashesMsg) error { - log.Debug("received wanted batch", "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To) - s, err := self.getOutgoingStreamer(req.Stream + keyToString(req.Key)) - if err != nil { - log.Debug(err.Error()) - return err - } - hashes := s.currentBatch - // launch in go routine since GetBatch blocks until new hashes arrive - go self.SendOfferedHashes(s, req.From, req.To) - l := len(hashes) / HashSize - want, err := bv.NewFromBytes(req.Want, l) - if err != nil { - return fmt.Errorf("error initiaising bitvector of length %v: %v", l, err) - } - for i := 0; i < l; i++ { - if want.Get(i) { - hash := hashes[i*HashSize : (i+1)*HashSize] - data := s.GetData(hash) - if data == nil { - return errors.New("not found") - } - chunk := storage.NewChunk(hash, nil) - chunk.SData = data - if err := self.Deliver(chunk, s.priority); err != nil { - return err - } - } - } - return nil -} - -func (self *StreamerPeer) handleTakeoverProofMsg(req *TakeoverProofMsg) error { - _, err := self.getOutgoingStreamer(req.Stream) - if err != nil { - return err - } - // store the strongest takeoverproof for the stream in streamer - return nil -} - -// Deliver sends a storeRequestMsg protocol message to the peer -func (self *StreamerPeer) Deliver(chunk *storage.Chunk, priority uint8) error { - msg := &ChunkDeliveryMsg{ - Key: chunk.Key, - SData: chunk.SData, - } - return self.pq.Push(nil, msg, int(priority)) -} - -// Deliver sends a storeRequestMsg protocol message to the peer -func (self *StreamerPeer) SendPriority(msg interface{}, priority uint8) error { - return self.pq.Push(nil, msg, int(priority)) -} - -// SendOfferedHashes sends OfferedHashesMsg protocol msg -func (self *StreamerPeer) SendOfferedHashes(s *outgoingStreamer, f, t uint64) error { - hashes, from, to, proof, err := s.SetNextBatch(f, t) - if err != nil { - return err - } - if proof == nil { - proof = &HandoverProof{ - Handover: &Handover{}, - } - } - s.currentBatch = hashes - msg := &OfferedHashesMsg{ - HandoverProof: proof, - Hashes: hashes, - From: from, - To: to, - Stream: s.stream, - Key: s.key, - } - log.Debug("Swarm syncer offer batch", "stream", s.stream, "key", s.key, "len", len(hashes), "from", from, "to", to) - return self.SendPriority(msg, s.priority) -} - -// StreamerSpec is the spec of the streamer protocol. -var StreamerSpec = &protocols.Spec{ - Name: "stream", - Version: 1, - MaxMsgSize: 10 * 1024 * 1024, - Messages: []interface{}{ - HandshakeMsg{}, - OfferedHashesMsg{}, - WantedHashesMsg{}, - TakeoverProofMsg{}, - SubscribeMsg{}, - RetrieveRequestMsg{}, - ChunkDeliveryMsg{}, - }, -} - -// Run protocol run function -func (s *Streamer) Run(p *bzzPeer) error { - sp := NewStreamerPeer(p, s) - // load saved intervals - - s.setPeer(sp) - - defer s.deletePeer(sp) - defer close(sp.quit) - return sp.Run(sp.HandleMsg) -} - -// HandleMsg is the message handler that delegates incoming messages -func (self *StreamerPeer) HandleMsg(msg interface{}) error { - switch msg := msg.(type) { - - case *SubscribeMsg: - return self.handleSubscribeMsg(msg) - - case *OfferedHashesMsg: - return self.handleOfferedHashesMsg(msg) - - case *TakeoverProofMsg: - return self.handleTakeoverProofMsg(msg) - - case *WantedHashesMsg: - return self.handleWantedHashesMsg(msg) - - case *ChunkDeliveryMsg: - return self.streamer.delivery.handleChunkDeliveryMsg(msg) - - case *RetrieveRequestMsg: - return self.streamer.delivery.handleRetrieveRequestMsg(self, msg) - - default: - return fmt.Errorf("unknown message type: %T", msg) - } -} diff --git a/swarm/storage/dbaccess.go b/swarm/storage/dbaccess.go new file mode 100644 index 0000000000..69a659564a --- /dev/null +++ b/swarm/storage/dbaccess.go @@ -0,0 +1,52 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package storage + +// wrapper of db-s to provide mockable custom local chunk store access to syncer +type DBAPI struct { + db *DbStore + loc *LocalStore +} + +func NewDBAPI(loc *LocalStore) *DBAPI { + return &DBAPI{loc.DbStore.(*DbStore), loc} +} + +// to obtain the chunks from key or request db entry only +func (self *DBAPI) Get(key Key) (*Chunk, error) { + return self.loc.Get(key) +} + +// current storage counter of chunk db +func (self *DBAPI) CurrentBucketStorageIndex(po uint8) uint64 { + return self.db.CurrentBucketStorageIndex(po) +} + +// iteration storage counter and proximity order +func (self *DBAPI) Iterator(from uint64, to uint64, po uint8, f func(Key, uint64) bool) error { + return self.db.SyncIterator(from, to, po, f) +} + +// to obtain the chunks from key or request db entry only +func (self *DBAPI) GetOrCreateRequest(key Key) (*Chunk, bool) { + return self.loc.GetOrCreateRequest(key) +} + +// to obtain the chunks from key or request db entry only +func (self *DBAPI) Put(chunk *Chunk) { + self.loc.Put(chunk) +} diff --git a/swarm/swarm.go b/swarm/swarm.go index bc6533875b..6280d0fce7 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -39,6 +39,7 @@ import ( httpapi "github.com/ethereum/go-ethereum/swarm/api/http" "github.com/ethereum/go-ethereum/swarm/fuse" "github.com/ethereum/go-ethereum/swarm/network" + "github.com/ethereum/go-ethereum/swarm/network/stream" "github.com/ethereum/go-ethereum/swarm/pss" "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/storage/mock" @@ -53,7 +54,7 @@ type Swarm struct { //storage storage.ChunkStore // internal access to storage, common interface to cloud storage backends dpa *storage.DPA // distributed preimage archive, the local API to the storage with document level storage/retrieval support //depo network.StorageHandler // remote request handler, interface between bzz protocol and the storage - streamer *network.Streamer + streamer *stream.Registry //cloud storage.CloudStore // procurement, cloud storage backend (can multi-cloud) bzz *network.Bzz // the logistic manager backend chequebook.Backend // simple blockchain Backend @@ -129,13 +130,13 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e HiveParams: config.HiveParams, } - dbAccess := network.NewDbAccess(self.lstore) - delivery := network.NewDelivery(to, dbAccess) - self.streamer = network.NewStreamer(delivery) - network.RegisterOutgoingSyncer(self.streamer, dbAccess) - network.RegisterIncomingSyncer(self.streamer, dbAccess) + db := storage.NewDBAPI(self.lstore) + delivery := stream.NewDelivery(to, db) + self.streamer = stream.NewRegistry(delivery) + stream.RegisterSwarmSyncerServer(self.streamer, db) + stream.RegisterSwarmSyncerClient(self.streamer, db) - self.bzz = network.NewBzz(bzzconfig, to, nil, self.streamer) + self.bzz = network.NewBzz(bzzconfig, to, nil) // set up DPA, the cloud storage local access layer dpaChunkStore := storage.NewNetStore(self.lstore, self.streamer.Retrieve) @@ -271,6 +272,11 @@ func (self *Swarm) Protocols() (protos []p2p.Protocol) { protos = append(protos, p) } } + if self.streamer != nil { + for _, p := range self.streamer.Protocols() { + protos = append(protos, p) + } + } return } From f0f62218a35cfa350f5552ccab800337750c9f7c Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Thu, 18 Jan 2018 18:47:43 +0100 Subject: [PATCH 065/210] swarm/network, swarm/storage: Further refactor fixes --- swarm/network/stream/common_test.go | 6 +-- swarm/network/stream/delivery_test.go | 56 +++++++++++------------ swarm/network/stream/syncer_test.go | 60 ++++++++++++------------- swarm/network/stream/testing/testing.go | 25 ++++++----- swarm/storage/{dbaccess.go => dbapi.go} | 0 5 files changed, 72 insertions(+), 75 deletions(-) rename swarm/storage/{dbaccess.go => dbapi.go} (100%) diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go index c46cc47144..4b9bc86d8a 100644 --- a/swarm/network/stream/common_test.go +++ b/swarm/network/stream/common_test.go @@ -96,13 +96,13 @@ func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Registry, *stora delivery := NewDelivery(to, db) streamer := NewRegistry(delivery) run := func(p *p2p.Peer, rw p2p.MsgReadWriter) error { - BzzPeer := &BzzPeer{ + bzzPeer := &network.BzzPeer{ Peer: protocols.NewPeer(p, rw, Spec), localAddr: addr, BzzAddr: network.NewAddrFromNodeID(p.ID()), } - to.On(BzzPeer) - return streamer.Run(BzzPeer) + to.On(bzzPeer) + return streamer.Run(bzzPeer) } protocolTester := p2ptest.NewProtocolTester(t, network.NewNodeIDFromAddr(addr), 1, run) diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index 317132be61..f19cbb9a4d 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/simulations" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" p2ptest "github.com/ethereum/go-ethereum/p2p/testing" + "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/storage" ) @@ -323,10 +324,10 @@ func testDeliveryFromNodes(nodes, conns, size int, skipCheck bool) func(adapter action := func(net *simulations.Network) func(context.Context) error { // here we distribute chunks of a random file into localstores of nodes 1 to nodes - rrdpa := storage.NewDPA(newRoundRobinStore(localStores[1:]...), storage.NewChunkerParams()) + rrdpa := storage.NewDPA(newRoundRobinStore(testing.LocalStores[1:]...), storage.NewChunkerParams()) rrdpa.Start() // create a retriever dpa for the pivot node - dpacs := storage.NewNetStore(localStores[0].(*storage.LocalStore), func(chunk *storage.Chunk) error { return delivery.RequestFromPeers(chunk.Key[:], skipCheck) }) + dpacs := storage.NewNetStore(testing.LocalStores[0].(*storage.LocalStore), func(chunk *storage.Chunk) error { return delivery.RequestFromPeers(chunk.Key[:], skipCheck) }) dpa := storage.NewDPA(dpacs, storage.NewChunkerParams()) dpa.Start() return func(context.Context) error { @@ -404,41 +405,38 @@ func newDeliveryService(ctx *adapters.ServiceContext) (node.Service, error) { id := ctx.Config.ID addr := NewAddrFromNodeID(id) kad := NewKademlia(addr.Over(), NewKadParams()) - localStore := localStores[nodeCount] + localStore := testing.LocalStores[testing.NodeCount] db := NewDBAPI(localStore.(*storage.LocalStore)) streamer := NewStreamerRegistry(NewDelivery(kad, db)) - if nodeCount == 0 { + if testing.NodeCount == 0 { // the delivery service for the pivot node is assigned globally // so that the simulation action call can use it for the // swarm enabled dpa delivery = streamer.delivery } - self := &testStreamerService{ - addr: addr, - streamer: streamer, - } - self.run = self.runDelivery - nodeCount++ - return self, nil + testing.NodeCount++ + return testing.NewTestStreamerService(Spec, makeRunFunc(addr, streamer)), nil } -func (b *testStreamerService) runDelivery(p *p2p.Peer, rw p2p.MsgReadWriter) error { - BzzPeer := &BzzPeer{ - Peer: protocols.NewPeer(p, rw, StreamerSpec), - localAddr: b.addr, - BzzAddr: NewAddrFromNodeID(p.ID()), - } - b.streamer.delivery.overlay.On(BzzPeer) - defer b.streamer.delivery.overlay.Off(BzzPeer) - go func() { - // each node Subscribes to each other's retrieveRequestStream - // need to wait till an aynchronous process registers the peers in streamer.peers - // that is used by Subscribe - time.Sleep(1 * time.Second) - err := b.streamer.Subscribe(p.ID(), retrieveRequestStream, nil, 0, 0, Top, true) - if err != nil { - log.Warn("error in subscribe", "err", err) +func makeRunFunc(addr network.Addr, streamer *Registry) (func(p *p2p.Peer, rw p2p.MsgReadWriter), error) { + return func(p *p2p.Peer, rw p2p.MsgReadWriter) error { + bzzPeer := &network.BzzPeer{ + Peer: protocols.NewPeer(p, rw, Spec), + localAddr: addr, + BzzAddr: NewAddrFromNodeID(p.ID()), } - }() - return b.streamer.Run(BzzPeer) + streamer.delivery.overlay.On(bzzPeer) + defer streamer.delivery.overlay.Off(bzzPeer) + go func() { + // each node Subscribes to each other's retrieveRequestStream + // need to wait till an aynchronous process registers the peers in streamer.peers + // that is used by Subscribe + time.Sleep(1 * time.Second) + err := streamer.Subscribe(p.ID(), retrieveRequestStream, nil, 0, 0, Top, true) + if err != nil { + log.Warn("error in subscribe", "err", err) + } + }() + return streamer.Run(bzzPeer) + } } diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index c1a9bd6fac..a6e0ab6b1a 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -58,7 +58,7 @@ func testSyncBetweenNodes(nodes, conns, size int, skipCheck bool, po uint8) func action := func(net *simulations.Network) func(context.Context) error { // here we distribute chunks of a random file into localstores of nodes 1 to nodes - rrdpa := storage.NewDPA(newRoundRobinStore(localStores[1:]...), storage.NewChunkerParams()) + rrdpa := storage.NewDPA(newRoundRobinStore(testing.LocalStores[1:]...), storage.NewChunkerParams()) rrdpa.Start() // create a retriever dpa for the pivot node return func(context.Context) error { @@ -78,7 +78,7 @@ func testSyncBetweenNodes(nodes, conns, size int, skipCheck bool, po uint8) func dbs := make([]*storage.DBAPI, nodes) for i := 0; i < nodes; i++ { - dbs[i] = NewDbAccess(localStores[i].(*storage.LocalStore)) + dbs[i] = NewDbAccess(testing.LocalStores[i].(*storage.LocalStore)) } return func(ctx context.Context, id discover.NodeID) (bool, error) { if id != net.Nodes[0].ID() { @@ -128,42 +128,38 @@ func newSyncerService(ctx *adapters.ServiceContext) (node.Service, error) { // for the test we make all peers share 8 bits so that syncing full bins make sense addr.OAddr[0] = byte(0) kad := NewKademlia(addr.Over(), NewKadParams()) - localStore := localStores[nodeCount] + localStore := testing.LocalStores[testing.NodeCount] db := NewDbAccess(localStore.(*storage.LocalStore)) streamer := NewRegistry(NewDelivery(kad, db)) RegisterIncomingSyncer(streamer, db) RegisterOutgoingSyncer(streamer, db) - self := &testStreamerService{ - index: nodeCount, - addr: addr, - streamer: streamer, - } - self.run = self.runSyncer - nodeCount++ - return self, nil + testing.NodeCount++ + return testing.NewTestStreamerService(Spec, makeRunFunc(addr, streamer)), nil } -func (b *testStreamerService) runSyncer(p *p2p.Peer, rw p2p.MsgReadWriter) error { - addr := network.NewAddrFromNodeID(p.ID()) - addr.OAddr[0] = byte(0) - BzzPeer := &BzzPeer{ - Peer: protocols.NewPeer(p, rw, Spec), - localAddr: b.addr, - BzzAddr: addr, - } - b.streamer.delivery.overlay.On(BzzPeer) - defer b.streamer.delivery.overlay.Off(BzzPeer) - // if len(addr) > b.index+1 && bytes.Equal(addrs[b.index+1], addr) { - go func() { - // each node Subscribes to each other's retrieveRequestStream - // need to wait till an aynchronous process registers the peers in streamer.peers - // that is used by Subscribe - time.Sleep(1 * time.Second) - if err := b.streamer.Subscribe(p.ID(), "SYNC", []byte{uint8(1)}, 0, 0, Top, false); err != nil { - log.Warn("error in subscribe", "err", err) +func makeRunFunc(localAddr network.Addr, streamer *Registry) (func(p *p2p.Peer, rw p2p.MsgReadWriter), error) { + return func(p *p2p.Peer, rw p2p.MsgReadWriter) error { + remoteAddr := network.NewAddrFromNodeID(p.ID()) + remoteAddr.OAddr[0] = byte(0) + bzzPeer := &network.BzzPeer{ + Peer: protocols.NewPeer(p, rw, Spec), + localAddr: localAddr, + BzzAddr: remoteAddr, } - }() - // } - return b.streamer.Run(BzzPeer) + streamer.delivery.overlay.On(bzzPeer) + defer streamer.delivery.overlay.Off(bzzPeer) + // if len(addr) > b.index+1 && bytes.Equal(testing.Addrs[b.index+1], addr) { + go func() { + // each node Subscribes to each other's retrieveRequestStream + // need to wait till an aynchronous process registers the peers in streamer.peers + // that is used by Subscribe + time.Sleep(1 * time.Second) + if err := streamer.Subscribe(p.ID(), "SYNC", []byte{uint8(1)}, 0, 0, Top, false); err != nil { + log.Warn("error in subscribe", "err", err) + } + }() + // } + return streamer.Run(bzzPeer) + } } diff --git a/swarm/network/stream/testing/testing.go b/swarm/network/stream/testing/testing.go index e0da4d0336..d3a78ecde2 100644 --- a/swarm/network/stream/testing/testing.go +++ b/swarm/network/stream/testing/testing.go @@ -31,11 +31,11 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/protocols" "github.com/ethereum/go-ethereum/p2p/simulations" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm/network" - "github.com/ethereum/go-ethereum/swarm/network/stream" "github.com/ethereum/go-ethereum/swarm/storage" ) @@ -223,23 +223,26 @@ func (rrs *roundRobinStore) Close() { } type TestStreamerService struct { - index int - addr *network.BzzAddr - streamer *stream.Registry - run func(s *TestStreamerService, p *p2p.Peer, rw p2p.MsgReadWriter) error + // index int + // addr *network.BzzAddr + // // streamer *stream.Registry + run func(p *p2p.Peer, rw p2p.MsgReadWriter) error + spec *protocols.Spec } -func NewTestStreamerService(run func(s *TestStreamerService, p *p2p.Peer, rw p2p.MsgReadWriter) error) TestStreamerService { - t := &TestStreamerService{} - t.run = run +func NewTestStreamerService(spec *protocols.Spec, run func(p *p2p.Peer, rw p2p.MsgReadWriter) error) *TestStreamerService { + return &TestStreamerService{ + run: run, + spec: spec, + } } func (tds *TestStreamerService) Protocols() []p2p.Protocol { return []p2p.Protocol{ { - Name: stream.Spec.Name, - Version: stream.Spec.Version, - Length: stream.Spec.Length(), + Name: tds.spec.Name, + Version: tds.spec.Version, + Length: tds.spec.Length(), Run: tds.run, // NodeInfo: , // PeerInfo: , diff --git a/swarm/storage/dbaccess.go b/swarm/storage/dbapi.go similarity index 100% rename from swarm/storage/dbaccess.go rename to swarm/storage/dbapi.go From 4e032aeb108afb6ba33f7e84403a4c96b90ef0b6 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Fri, 19 Jan 2018 11:17:43 +0100 Subject: [PATCH 066/210] swarm/storage: Fix logging of number of written chunks Fixes issue https://github.com/ethersphere/go-ethereum/issues/201 --- swarm/storage/dbstore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/storage/dbstore.go b/swarm/storage/dbstore.go index bed07bdd9b..16c4ecdf50 100644 --- a/swarm/storage/dbstore.go +++ b/swarm/storage/dbstore.go @@ -615,7 +615,7 @@ func (s *DbStore) writeBatch(b *leveldb.Batch, entryCnt, dataIdx, accessCnt uint b.Put(keyEntryCnt, U64ToBytes(entryCnt)) b.Put(keyDataIdx, U64ToBytes(dataIdx)) b.Put(keyAccessCnt, U64ToBytes(accessCnt)) - l := s.batch.Len() + l := b.Len() if err := s.db.Write(b); err != nil { log.Error(fmt.Sprintf("unable to write batch: %v", err)) } From 02bdde67fd2314170ebc56db6c38a27cc52936e6 Mon Sep 17 00:00:00 2001 From: zelig Date: Fri, 19 Jan 2018 13:14:59 +0100 Subject: [PATCH 067/210] swarm/network: simplify stream test code, continue refactor --- swarm/network/protocol.go | 8 + swarm/network/stream/common_test.go | 76 ++++--- swarm/network/stream/delivery_test.go | 264 +++++++++++------------- swarm/network/stream/stream.go | 87 +++++++- swarm/network/stream/streamer_test.go | 24 +-- swarm/network/stream/syncer_test.go | 254 ++++++++++++----------- swarm/network/stream/testing/testing.go | 213 +++++++------------ swarm/swarm.go | 4 +- 8 files changed, 474 insertions(+), 456 deletions(-) diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index 53776e5ba6..9afa69c3a9 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -253,6 +253,14 @@ type BzzPeer struct { lastActive time.Time // time is updated whenever mutexes are releasing } +func NewBzzTestPeer(p *protocols.Peer, addr *BzzAddr) *BzzPeer { + return &BzzPeer{ + Peer: p, + localAddr: addr, + BzzAddr: NewAddrFromNodeID(p.ID()), + } +} + // Off returns the overlay peer record for offline persistance func (p *BzzPeer) Off() OverlayAddr { return p.BzzAddr diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go index 4b9bc86d8a..f0468be935 100644 --- a/swarm/network/stream/common_test.go +++ b/swarm/network/stream/common_test.go @@ -19,15 +19,14 @@ package stream import ( "errors" "flag" - "io" "io/ioutil" "os" + "sync/atomic" "testing" "time" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/protocols" + "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" p2ptest "github.com/ethereum/go-ethereum/p2p/testing" "github.com/ethereum/go-ethereum/swarm/network" @@ -40,8 +39,7 @@ var ( ) var services = adapters.Services{ - "delivery": newDeliveryService, - "syncer": newSyncerService, + "streamer": NewStreamerService, } func init() { @@ -53,24 +51,16 @@ func init() { log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(false)))) } -var ( - delivery *Delivery - fileHash storage.Key -) - -func mustReadAll(dpa *storage.DPA, hash storage.Key) (int, error) { - r := dpa.Retrieve(fileHash) - buf := make([]byte, 1024) - var n, total int - var err error - for (total == 0 || n > 0) && err == nil { - n, err = r.ReadAt(buf, int64(total)) - total += n - } - if err != nil && err != io.EOF { - return total, err - } - return total, nil +// newService +func NewStreamerService(ctx *adapters.ServiceContext) (node.Service, error) { + id := ctx.Config.ID + addr := toAddr(id) + kad := network.NewKademlia(addr.Over(), network.NewKadParams()) + store := stores[id] + db := storage.NewDBAPI(store.(*storage.LocalStore)) + delivery := NewDelivery(kad, db) + deliveries[id] = delivery + return NewRegistry(addr, delivery, store), nil } func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Registry, *storage.LocalStore, func(), error) { @@ -94,17 +84,8 @@ func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Registry, *stora db := storage.NewDBAPI(localStore) delivery := NewDelivery(to, db) - streamer := NewRegistry(delivery) - run := func(p *p2p.Peer, rw p2p.MsgReadWriter) error { - bzzPeer := &network.BzzPeer{ - Peer: protocols.NewPeer(p, rw, Spec), - localAddr: addr, - BzzAddr: network.NewAddrFromNodeID(p.ID()), - } - to.On(bzzPeer) - return streamer.Run(bzzPeer) - } - protocolTester := p2ptest.NewProtocolTester(t, network.NewNodeIDFromAddr(addr), 1, run) + streamer := NewRegistry(addr, delivery, localStore) + protocolTester := p2ptest.NewProtocolTester(t, network.NewNodeIDFromAddr(addr), 1, streamer.runProtocol) err = waitForPeers(streamer, 1*time.Second) if err != nil { @@ -128,3 +109,30 @@ func waitForPeers(streamer *Registry, timeout time.Duration) error { } } } + +type roundRobinStore struct { + index uint32 + stores []storage.ChunkStore +} + +func newRoundRobinStore(stores ...storage.ChunkStore) *roundRobinStore { + return &roundRobinStore{ + stores: stores, + } +} + +func (rrs *roundRobinStore) Get(key storage.Key) (*storage.Chunk, error) { + return nil, errors.New("get not well defined on round robin store") +} + +func (rrs *roundRobinStore) Put(chunk *storage.Chunk) { + i := atomic.AddUint32(&rrs.index, 1) + idx := int(i) % len(rrs.stores) + rrs.stores[idx].Put(chunk) +} + +func (rrs *roundRobinStore) Close() { + for _, store := range rrs.stores { + store.Close() + } +} diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index f19cbb9a4d..a37651a148 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -26,17 +26,20 @@ import ( "time" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" - "github.com/ethereum/go-ethereum/p2p/protocols" "github.com/ethereum/go-ethereum/p2p/simulations" - "github.com/ethereum/go-ethereum/p2p/simulations/adapters" p2ptest "github.com/ethereum/go-ethereum/p2p/testing" "github.com/ethereum/go-ethereum/swarm/network" + streamTesting "github.com/ethereum/go-ethereum/swarm/network/stream/testing" "github.com/ethereum/go-ethereum/swarm/storage" ) +var ( + deliveries map[discover.NodeID]*Delivery + stores map[discover.NodeID]storage.ChunkStore + toAddr func(discover.NodeID) *network.BzzAddr +) + func TestStreamerRetrieveRequest(t *testing.T) { tester, streamer, _, teardown, err := newStreamerTester(t) defer teardown() @@ -81,7 +84,7 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { peer := streamer.getPeer(peerID) peer.handleSubscribeMsg(&SubscribeMsg{ - Stream: retrieveRequestStream, + Stream: swarmChunkServerStreamName, Key: nil, From: 0, To: 0, @@ -132,7 +135,7 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { peer := streamer.getPeer(peerID) peer.handleSubscribeMsg(&SubscribeMsg{ - Stream: retrieveRequestStream, + Stream: swarmChunkServerStreamName, Key: nil, From: 0, To: 0, @@ -168,7 +171,7 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { // TODO: why is this 32??? To: 32, Key: []byte{}, - Stream: retrieveRequestStream, + Stream: swarmChunkServerStreamName, }, Peer: peerID, }, @@ -221,8 +224,8 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { t.Fatal(err) } - streamer.RegisterIncomingStreamer("foo", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { - return &testIncomingStreamer{ + streamer.RegisterClientFunc("foo", func(p *Peer, t []byte) (Client, error) { + return &testClient{ t: t, }, nil }) @@ -301,142 +304,127 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { } func TestDeliveryFromNodes(t *testing.T) { - testSimulation(t, testDeliveryFromNodes(2, 1, 8100, true)) - testSimulation(t, testDeliveryFromNodes(2, 1, 8100, false)) - testSimulation(t, testDeliveryFromNodes(3, 1, 8100, true)) - testSimulation(t, testDeliveryFromNodes(3, 1, 8100, false)) + testDeliveryFromNodes(t, 2, 1, 8100, true) + testDeliveryFromNodes(t, 2, 1, 8100, false) + testDeliveryFromNodes(t, 3, 1, 8100, true) + testDeliveryFromNodes(t, 3, 1, 8100, false) } -func testDeliveryFromNodes(nodes, conns, size int, skipCheck bool) func(adapter adapters.NodeAdapter) (*simulations.StepResult, error) { - return func(adapter adapters.NodeAdapter) (*simulations.StepResult, error) { - trigger := func(net *simulations.Network) chan discover.NodeID { - triggerC := make(chan discover.NodeID) - ticker := time.NewTicker(500 * time.Millisecond) - go func() { - defer ticker.Stop() - // we are only testing the pivot node (net.Nodes[0]) - for range ticker.C { - triggerC <- net.Nodes[0].ID() - } - }() - return triggerC - } - - action := func(net *simulations.Network) func(context.Context) error { - // here we distribute chunks of a random file into localstores of nodes 1 to nodes - rrdpa := storage.NewDPA(newRoundRobinStore(testing.LocalStores[1:]...), storage.NewChunkerParams()) - rrdpa.Start() - // create a retriever dpa for the pivot node - dpacs := storage.NewNetStore(testing.LocalStores[0].(*storage.LocalStore), func(chunk *storage.Chunk) error { return delivery.RequestFromPeers(chunk.Key[:], skipCheck) }) - dpa := storage.NewDPA(dpacs, storage.NewChunkerParams()) - dpa.Start() - return func(context.Context) error { - defer rrdpa.Stop() - // upload an actual random file of size size - hash, wait, err := rrdpa.Store(io.LimitReader(crand.Reader, int64(size)), int64(size)) - if err != nil { - return err - } - // wait until all chunks stored - // TODO: is wait() necessary? - wait() - // assign the fileHash to a global so that it is available for the check function - fileHash = hash - go func() { - defer dpa.Stop() - log.Debug(fmt.Sprintf("retrieve %v", fileHash)) - // start the retrieval on the pivot node - this will spawn retrieve requests for missing chunks - // we must wait for the peer connections to have started before requesting - time.Sleep(2 * time.Second) - n, err := mustReadAll(dpa, fileHash) - log.Debug(fmt.Sprintf("retrieved %v", fileHash), "read", n, "err", err) - }() - return nil - } - } - - check := func(net *simulations.Network, dpa *storage.DPA) func(ctx context.Context, id discover.NodeID) (bool, error) { - return func(ctx context.Context, id discover.NodeID) (bool, error) { - if id != net.Nodes[0].ID() { - return true, nil - } - select { - case <-ctx.Done(): - return false, ctx.Err() - default: - } - // try to locally retrieve the file to check if retrieve requests have been successful - total, err := mustReadAll(dpa, fileHash) - log.Debug(fmt.Sprintf("check if %08x is available locally: number of bytes read %v/%v (error: %v)", fileHash, total, size, err)) - if err != nil || total != size { - return false, nil - } - return true, nil - // node := net.GetNode(id) - // if node == nil { - // return false, fmt.Errorf("unknown node: %s", id) - // } - // client, err := node.Client() - // if err != nil { - // return false, fmt.Errorf("error getting node client: %s", err) - // } - // var response int - // if err := client.Call(&response, "test_haslocal", hash); err != nil { - // return false, fmt.Errorf("error getting bzz_has response: %s", err) - // } - // log.Debug(fmt.Sprintf("node has: %v\n%v", id, response)) - // return response == 0, nil - } - } - - result, err := runSimulation(nodes, conns, "delivery", NewAddrFromNodeID, action, trigger, check, adapter) - if err != nil { - return nil, fmt.Errorf("Setting up simulation failed: %v", err) - } - if result.Error != nil { - return nil, fmt.Errorf("Simulation failed: %s", result.Error) - } - return result, err +func testDeliveryFromNodes(t *testing.T, nodes, conns, size int, skipCheck bool) { + toAddr = network.NewAddrFromNodeID + conf := &streamTesting.RunConfig{ + Adapter: *adapter, + NodeCount: nodes, + ConnLevel: conns, + ToAddr: toAddr, + Services: services, } -} -// newDeliveryService -func newDeliveryService(ctx *adapters.ServiceContext) (node.Service, error) { - id := ctx.Config.ID - addr := NewAddrFromNodeID(id) - kad := NewKademlia(addr.Over(), NewKadParams()) - localStore := testing.LocalStores[testing.NodeCount] - db := NewDBAPI(localStore.(*storage.LocalStore)) - streamer := NewStreamerRegistry(NewDelivery(kad, db)) - if testing.NodeCount == 0 { - // the delivery service for the pivot node is assigned globally - // so that the simulation action call can use it for the - // swarm enabled dpa - delivery = streamer.delivery + sim, teardown, err := streamTesting.NewSimulation(conf) + defer teardown() + if err != nil { + t.Fatal(err.Error()) + } + stores = make(map[discover.NodeID]storage.ChunkStore) + deliveries = make(map[discover.NodeID]*Delivery) + for i, id := range sim.IDs { + stores[id] = sim.Stores[i] } - testing.NodeCount++ - return testing.NewTestStreamerService(Spec, makeRunFunc(addr, streamer)), nil -} -func makeRunFunc(addr network.Addr, streamer *Registry) (func(p *p2p.Peer, rw p2p.MsgReadWriter), error) { - return func(p *p2p.Peer, rw p2p.MsgReadWriter) error { - bzzPeer := &network.BzzPeer{ - Peer: protocols.NewPeer(p, rw, Spec), - localAddr: addr, - BzzAddr: NewAddrFromNodeID(p.ID()), - } - streamer.delivery.overlay.On(bzzPeer) - defer streamer.delivery.overlay.Off(bzzPeer) + // here we distribute chunks of a random file into Stores of nodes 1 to nodes + rrdpa := storage.NewDPA(newRoundRobinStore(sim.Stores[1:]...), storage.NewChunkerParams()) + rrdpa.Start() + fileHash, wait, err := rrdpa.Store(io.LimitReader(crand.Reader, int64(size)), int64(size)) + defer rrdpa.Stop() + if err != nil { + t.Fatal(err.Error()) + } + // wait until all chunks stored + // TODO: is wait() necessary? + wait() + // each node Subscribes to each other's swarmChunkServerStreamName + // need to wait till an aynchronous process registers the peers in streamer.peers + // that is used by Subscribe + // time.Sleep(1 * time.Second) + // err := streamer.Subscribe(p.ID(), swarmChunkServerStreamName, nil, 0, 0, Top, true) + if err != nil { + t.Fatal(err.Error()) + } + // create a retriever dpa for the pivot node + delivery := deliveries[sim.IDs[0]] + dpacs := storage.NewNetStore(sim.Stores[0].(*storage.LocalStore), func(chunk *storage.Chunk) error { return delivery.RequestFromPeers(chunk.Key[:], skipCheck) }) + dpa := storage.NewDPA(dpacs, storage.NewChunkerParams()) + dpa.Start() + action := func(context.Context) error { + dpa := storage.NewDPA(sim.Stores[0], storage.NewChunkerParams()) + dpa.Start() + // defer dpa.Stop() + go func() { - // each node Subscribes to each other's retrieveRequestStream - // need to wait till an aynchronous process registers the peers in streamer.peers - // that is used by Subscribe - time.Sleep(1 * time.Second) - err := streamer.Subscribe(p.ID(), retrieveRequestStream, nil, 0, 0, Top, true) - if err != nil { - log.Warn("error in subscribe", "err", err) - } + defer dpa.Stop() + log.Debug(fmt.Sprintf("retrieve %v", fileHash)) + // start the retrieval on the pivot node - this will spawn retrieve requests for missing chunks + // we must wait for the peer connections to have started before requesting + time.Sleep(2 * time.Second) + n, err := mustReadAll(dpa, fileHash) + log.Debug(fmt.Sprintf("retrieved %v", fileHash), "read", n, "err", err) }() - return streamer.Run(bzzPeer) + return nil } + + check := func(ctx context.Context, id discover.NodeID) (bool, error) { + select { + case <-ctx.Done(): + return false, ctx.Err() + default: + } + // try to locally retrieve the file to check if retrieve requests have been successful + node := sim.Net.GetNode(id) + if node == nil { + return false, fmt.Errorf("unknown node: %s", id) + } + client, err := node.Client() + if err != nil { + return false, fmt.Errorf("error getting node client: %s", err) + } + var total int64 + if err := client.Call(&total, "stream_readAll", fileHash); err != nil { + return false, fmt.Errorf("error reading all: %s (read %v)", err, total) + } + // total, err := mustReadAll(dpa, fileHash) + log.Debug(fmt.Sprintf("check if %08x is available locally: number of bytes read %v/%v (error: %v)", fileHash, total, size, err)) + if err != nil || total != int64(size) { + return false, nil + } + return true, nil + } + + trigger := make(chan discover.NodeID) + ticker := time.NewTicker(500 * time.Millisecond) + go func() { + defer ticker.Stop() + // we are only testing the pivot node (net.Nodes[0]) + for range ticker.C { + trigger <- sim.Net.Nodes[0].ID() + } + }() + + conf.Step = &simulations.Step{ + Action: action, + Trigger: trigger, + Expect: &simulations.Expectation{ + Nodes: sim.IDs[0:1], + Check: check, + }, + } + startedAt := time.Now() + result, err := sim.Run(conf) + finishedAt := time.Now() + if err != nil { + t.Fatalf("Setting up simulation failed: %v", err) + } + if result.Error != nil { + t.Fatalf("Simulation failed: %s", result.Error) + } + streamTesting.CheckResult(t, result, startedAt, finishedAt) } diff --git a/swarm/network/stream/stream.go b/swarm/network/stream/stream.go index 469ed9f126..7565785bee 100644 --- a/swarm/network/stream/stream.go +++ b/swarm/network/stream/stream.go @@ -18,14 +18,17 @@ package stream import ( "fmt" + "io" "math" "sync" "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/protocols" + "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/storage" ) @@ -41,6 +44,7 @@ const ( // Registry registry for outgoing and incoming streamer constructors type Registry struct { + addr *network.BzzAddr clientMu sync.RWMutex serverMu sync.RWMutex peersMu sync.RWMutex @@ -48,11 +52,14 @@ type Registry struct { clientFuncs map[string]func(*Peer, []byte) (Client, error) peers map[discover.NodeID]*Peer delivery *Delivery + store storage.ChunkStore } // NewRegistry is Streamer constructor -func NewRegistry(delivery *Delivery) *Registry { +func NewRegistry(addr *network.BzzAddr, delivery *Delivery, store storage.ChunkStore) *Registry { streamer := &Registry{ + addr: addr, + store: store, serverFuncs: make(map[string]func(*Peer, []byte) (Server, error)), clientFuncs: make(map[string]func(*Peer, []byte) (Client, error)), peers: make(map[discover.NodeID]*Peer), @@ -175,17 +182,22 @@ func (r *Registry) deletePeer(peer *Peer) { } // Run protocol run function -func (r *Registry) Run(p *protocols.Peer) error { +func (r *Registry) run(p *protocols.Peer) error { sp := NewPeer(p, r) - // load saved intervals - r.setPeer(sp) - defer r.deletePeer(sp) defer close(sp.quit) return sp.Run(sp.HandleMsg) } +func (r *Registry) runProtocol(p *p2p.Peer, rw p2p.MsgReadWriter) error { + peer := protocols.NewPeer(p, rw, Spec) + bzzPeer := network.NewBzzTestPeer(peer, r.addr) + r.delivery.overlay.On(bzzPeer) + defer r.delivery.overlay.Off(bzzPeer) + return r.run(peer) +} + // HandleMsg is the message handler that delegates incoming messages func (p *Peer) HandleMsg(msg interface{}) error { switch msg := msg.(type) { @@ -305,12 +317,65 @@ func (r *Registry) Protocols() []p2p.Protocol { Name: Spec.Name, Version: Spec.Version, Length: Spec.Length(), - Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error { - peer := protocols.NewPeer(p, rw, Spec) - return r.Run(peer) - }, - NodeInfo: r.NodeInfo, - PeerInfo: r.PeerInfo, + Run: r.runProtocol, + // NodeInfo: , + // PeerInfo: , }, } } + +func (r *Registry) APIs() []rpc.API { + return []rpc.API{ + { + Namespace: "stream", + Version: "0.1", + Service: NewAPI(r, r.store), + Public: true, + }, + } +} + +func (r *Registry) Start(server *p2p.Server) error { + return nil +} + +func (r *Registry) Stop() error { + return nil +} + +type API struct { + streamer *Registry + dpa *storage.DPA +} + +func NewAPI(r *Registry, store storage.ChunkStore) *API { + dpa := storage.NewDPA(store, storage.NewChunkerParams()) + return &API{ + streamer: r, + dpa: dpa, + } +} + +func mustReadAll(dpa *storage.DPA, hash []byte) (int64, error) { + r := dpa.Retrieve(hash) + buf := make([]byte, 1024) + var n int + var total int64 + var err error + for (total == 0 || n > 0) && err == nil { + n, err = r.ReadAt(buf, total) + total += int64(n) + } + if err != nil && err != io.EOF { + return total, err + } + return total, nil +} + +func (api *API) ReadAll(hash []byte) (int64, error) { + return mustReadAll(api.dpa, hash) +} + +func (api *API) Subscribe(peerId discover.NodeID, s string, t []byte, from, to uint64, priority uint8, live bool) error { + return api.streamer.Subscribe(peerId, s, t, from, to, priority, live) +} diff --git a/swarm/network/stream/streamer_test.go b/swarm/network/stream/streamer_test.go index 250573ade4..a905f4c963 100644 --- a/swarm/network/stream/streamer_test.go +++ b/swarm/network/stream/streamer_test.go @@ -50,15 +50,15 @@ var ( batchDone = make(chan bool) ) -type testIncomingStreamer struct { +type testClient struct { t []byte } -type testOutgoingStreamer struct { +type testServer struct { t []byte } -func (self *testIncomingStreamer) NeedData(hash []byte) func() { +func (self *testClient) NeedData(hash []byte) func() { receivedHashes[string(hash)] = hash if bytes.Equal(hash, hash0[:]) { return func() { @@ -72,16 +72,16 @@ func (self *testIncomingStreamer) NeedData(hash []byte) func() { return nil } -func (self *testIncomingStreamer) BatchDone(string, uint64, []byte, []byte) func() (*TakeoverProof, error) { +func (self *testClient) BatchDone(string, uint64, []byte, []byte) func() (*TakeoverProof, error) { close(batchDone) return nil } -func (self *testOutgoingStreamer) SetNextBatch(from uint64, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) { +func (self *testServer) SetNextBatch(from uint64, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) { return make([]byte, HashSize), from + 1, to + 1, nil, nil } -func (self *testOutgoingStreamer) GetData([]byte) []byte { +func (self *testServer) GetData([]byte) []byte { return nil } @@ -92,8 +92,8 @@ func TestStreamerDownstreamSubscribeMsgExchange(t *testing.T) { t.Fatal(err) } - streamer.RegisterClientFunc("foo", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { - return &testIncomingStreamer{ + streamer.RegisterClientFunc("foo", func(p *Peer, t []byte) (Client, error) { + return &testClient{ t: t, }, nil }) @@ -134,8 +134,8 @@ func TestStreamerUpstreamSubscribeMsgExchange(t *testing.T) { t.Fatal(err) } - streamer.RegisterServerFunc("foo", func(p *StreamerPeer, t []byte) (OutgoingStreamer, error) { - return &testOutgoingStreamer{ + streamer.RegisterServerFunc("foo", func(p *Peer, t []byte) (Server, error) { + return &testServer{ t: t, }, nil }) @@ -188,8 +188,8 @@ func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) { t.Fatal(err) } - streamer.RegisterClientFunc("foo", func(p *StreamerPeer, t []byte) (IncomingStreamer, error) { - return &testIncomingStreamer{ + streamer.RegisterClientFunc("foo", func(p *Peer, t []byte) (Client, error) { + return &testClient{ t: t, }, nil }) diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index a6e0ab6b1a..cbe2b380b1 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -26,140 +26,144 @@ import ( "time" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" - "github.com/ethereum/go-ethereum/p2p/protocols" "github.com/ethereum/go-ethereum/p2p/simulations" - "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/swarm/network" + streamTesting "github.com/ethereum/go-ethereum/swarm/network/stream/testing" "github.com/ethereum/go-ethereum/swarm/storage" ) func TestSyncerSimulation(t *testing.T) { - testSimulation(t, testSyncBetweenNodes(2, 1, 81000, true, 1)) - testSimulation(t, testSyncBetweenNodes(3, 1, 81000, true, 1)) + testSyncBetweenNodes(t, 2, 1, 81000, true, 1) + testSyncBetweenNodes(t, 3, 1, 81000, true, 1) } -func testSyncBetweenNodes(nodes, conns, size int, skipCheck bool, po uint8) func(adapter adapters.NodeAdapter) (*simulations.StepResult, error) { - return func(adapter adapters.NodeAdapter) (*simulations.StepResult, error) { - trigger := func(net *simulations.Network) chan discover.NodeID { - triggerC := make(chan discover.NodeID) - ticker := time.NewTicker(500 * time.Millisecond) - go func() { - defer ticker.Stop() - // we are only testing the pivot node (net.Nodes[0]) - for range ticker.C { - triggerC <- net.Nodes[0].ID() - } - }() - return triggerC - } - - action := func(net *simulations.Network) func(context.Context) error { - // here we distribute chunks of a random file into localstores of nodes 1 to nodes - rrdpa := storage.NewDPA(newRoundRobinStore(testing.LocalStores[1:]...), storage.NewChunkerParams()) - rrdpa.Start() - // create a retriever dpa for the pivot node - return func(context.Context) error { - defer rrdpa.Stop() - // upload an actual random file of size size - _, wait, err := rrdpa.Store(io.LimitReader(crand.Reader, int64(size)), int64(size)) - if err != nil { - return err - } - // wait until all chunks stored - wait() - return nil - } - } - - check := func(net *simulations.Network, dpa *storage.DPA) func(ctx context.Context, id discover.NodeID) (bool, error) { - dbs := make([]*storage.DBAPI, nodes) - - for i := 0; i < nodes; i++ { - dbs[i] = NewDbAccess(testing.LocalStores[i].(*storage.LocalStore)) - } - return func(ctx context.Context, id discover.NodeID) (bool, error) { - if id != net.Nodes[0].ID() { - return true, nil - } - select { - case <-ctx.Done(): - return false, ctx.Err() - default: - } - - var found, total int - for i := 1; i < nodes; i++ { - dbs[i].iterator(0, math.MaxUint64, po, func(key storage.Key, index uint64) bool { - _, err := dbs[0].get(key) - if err == nil { - found++ - } - total++ - return true - }) - } - log.Debug("sync check", "bin", po, "found", found, "total", total) - return found == total, nil - } - } - toAddr := func(id discover.NodeID) *BzzAddr { - addr := network.NewAddrFromNodeID(id) - addr.OAddr[0] = byte(0) - return addr - } - - result, err := runSimulation(nodes, conns, "syncer", toAddr, action, trigger, check, adapter) - if err != nil { - return nil, fmt.Errorf("Setting up simulation failed: %v", err) - } - if result.Error != nil { - return nil, fmt.Errorf("Simulation failed: %s", result.Error) - } - return result, err +func testSyncBetweenNodes(t *testing.T, nodes, conns, size int, skipCheck bool, po uint8) { + toAddr = func(id discover.NodeID) *network.BzzAddr { + addr := network.NewAddrFromNodeID(id) + addr.OAddr[0] = byte(0) + return addr } -} - -func newSyncerService(ctx *adapters.ServiceContext) (node.Service, error) { - id := ctx.Config.ID - addr := network.NewAddrFromNodeID(id) - // for the test we make all peers share 8 bits so that syncing full bins make sense - addr.OAddr[0] = byte(0) - kad := NewKademlia(addr.Over(), NewKadParams()) - localStore := testing.LocalStores[testing.NodeCount] - db := NewDbAccess(localStore.(*storage.LocalStore)) - streamer := NewRegistry(NewDelivery(kad, db)) - RegisterIncomingSyncer(streamer, db) - RegisterOutgoingSyncer(streamer, db) - - testing.NodeCount++ - return testing.NewTestStreamerService(Spec, makeRunFunc(addr, streamer)), nil -} - -func makeRunFunc(localAddr network.Addr, streamer *Registry) (func(p *p2p.Peer, rw p2p.MsgReadWriter), error) { - return func(p *p2p.Peer, rw p2p.MsgReadWriter) error { - remoteAddr := network.NewAddrFromNodeID(p.ID()) - remoteAddr.OAddr[0] = byte(0) - bzzPeer := &network.BzzPeer{ - Peer: protocols.NewPeer(p, rw, Spec), - localAddr: localAddr, - BzzAddr: remoteAddr, - } - streamer.delivery.overlay.On(bzzPeer) - defer streamer.delivery.overlay.Off(bzzPeer) - // if len(addr) > b.index+1 && bytes.Equal(testing.Addrs[b.index+1], addr) { - go func() { - // each node Subscribes to each other's retrieveRequestStream - // need to wait till an aynchronous process registers the peers in streamer.peers - // that is used by Subscribe - time.Sleep(1 * time.Second) - if err := streamer.Subscribe(p.ID(), "SYNC", []byte{uint8(1)}, 0, 0, Top, false); err != nil { - log.Warn("error in subscribe", "err", err) - } - }() - // } - return streamer.Run(bzzPeer) + conf := &streamTesting.RunConfig{ + Adapter: *adapter, + NodeCount: nodes, + ConnLevel: conns, + ToAddr: toAddr, + Services: services, } + + sim, teardown, err := streamTesting.NewSimulation(conf) + defer teardown() + if err != nil { + t.Fatal(err.Error()) + } + stores = make(map[discover.NodeID]storage.ChunkStore) + deliveries = make(map[discover.NodeID]*Delivery) + log.Warn("Stores", "len", len(sim.Stores)) + for i, id := range sim.IDs { + stores[id] = sim.Stores[i] + } + + // here we distribute chunks of a random file into Stores of nodes 1 to nodes + rrdpa := storage.NewDPA(newRoundRobinStore(sim.Stores[1:]...), storage.NewChunkerParams()) + rrdpa.Start() + _, wait, err := rrdpa.Store(io.LimitReader(crand.Reader, int64(size)), int64(size)) + defer rrdpa.Stop() + if err != nil { + t.Fatal(err.Error()) + } + // wait until all chunks stored + // TODO: is wait() necessary? + wait() + // each node Subscribes to each other's swarmChunkServerStreamName + // need to wait till an aynchronous process registers the peers in streamer.peers + // that is used by Subscribe + // time.Sleep(1 * time.Second) + // err := streamer.Subscribe(p.ID(), swarmChunkServerStreamName, nil, 0, 0, Top, true) + if err != nil { + t.Fatal(err.Error()) + } + // create a retriever dpa for the pivot node + action := func(context.Context) error { + for i := 0; i < len(sim.IDs)-1; i++ { + id := sim.IDs[i] + // if err := streamer.Subscribe(p.ID(), "SYNC", []byte{uint8(1)}, 0, 0, Top, false); err != nil { + // log.Warn("error in subscribe", "err", err) + // } + node := sim.Net.GetNode(id) + if node == nil { + return fmt.Errorf("unknown node: %s", id) + } + client, err := node.Client() + if err != nil { + return fmt.Errorf("error getting node client: %s", err) + } + var n int64 + sid := sim.IDs[i+1] + if err := client.Call(&n, "stream_subscribe", sid, "SYNC", []byte{uint8(1)}, 0, 0, Top, false); err != nil { + return fmt.Errorf("error subscribing: %s", err) + } + } + return nil + } + + dbs := make([]*storage.DBAPI, nodes) + for i := 0; i < nodes; i++ { + dbs[i] = storage.NewDBAPI(sim.Stores[i].(*storage.LocalStore)) + } + + check := func(ctx context.Context, id discover.NodeID) (bool, error) { + if id != sim.Net.Nodes[0].ID() { + return true, nil + } + select { + case <-ctx.Done(): + return false, ctx.Err() + default: + } + + var found, total int + for i := 1; i < nodes; i++ { + + dbs[i].Iterator(0, math.MaxUint64, po, func(key storage.Key, index uint64) bool { + _, err := dbs[0].Get(key) + if err == nil { + found++ + } + total++ + return true + }) + } + log.Debug("sync check", "bin", po, "found", found, "total", total) + return found == total, nil + } + + trigger := make(chan discover.NodeID) + ticker := time.NewTicker(500 * time.Millisecond) + go func() { + defer ticker.Stop() + // we are only testing the pivot node (net.Nodes[0]) + for range ticker.C { + trigger <- sim.Net.Nodes[0].ID() + } + }() + + conf.Step = &simulations.Step{ + Action: action, + Trigger: trigger, + Expect: &simulations.Expectation{ + Nodes: sim.IDs[0:1], + Check: check, + }, + } + startedAt := time.Now() + result, err := sim.Run(conf) + finishedAt := time.Now() + if err != nil { + t.Fatalf("Setting up simulation failed: %v", err) + } + if result.Error != nil { + t.Fatalf("Simulation failed: %s", result.Error) + } + streamTesting.CheckResult(t, result, startedAt, finishedAt) } diff --git a/swarm/network/stream/testing/testing.go b/swarm/network/stream/testing/testing.go index d3a78ecde2..b427efd532 100644 --- a/swarm/network/stream/testing/testing.go +++ b/swarm/network/stream/testing/testing.go @@ -24,84 +24,76 @@ import ( "math/rand" "os" "sync" - "sync/atomic" "testing" "time" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" - "github.com/ethereum/go-ethereum/p2p/protocols" "github.com/ethereum/go-ethereum/p2p/simulations" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" - "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/storage" ) -var ( - LocalStores []storage.ChunkStore - Addrs []network.Addr - NodeCount int -) +type Simulation struct { + Net *simulations.Network + Stores []storage.ChunkStore + Addrs []network.Addr + IDs []discover.NodeID +} -func setLocalStores(addrs ...network.Addr) (func(), error) { +func SetStores(addrs ...network.Addr) ([]storage.ChunkStore, func(), error) { var datadirs []string - LocalStores = make([]storage.ChunkStore, len(addrs)) + stores := make([]storage.ChunkStore, len(addrs)) var err error for i, addr := range addrs { - // TODO: remove temp datadir after test var datadir string datadir, err = ioutil.TempDir("", "streamer") if err != nil { break } - var localStore *storage.LocalStore - localStore, err = storage.NewTestLocalStoreForAddr(datadir, addr.Over()) + var store storage.ChunkStore + store, err = storage.NewTestLocalStoreForAddr(datadir, addr.Over()) if err != nil { break } datadirs = append(datadirs, datadir) - LocalStores[i] = localStore + stores[i] = store } teardown := func() { for _, datadir := range datadirs { os.RemoveAll(datadir) } } - return teardown, err + return stores, teardown, err } -func testSimulation(t *testing.T, services adapters.Services, adapter string, simf func(adapters.NodeAdapter) (*simulations.StepResult, error)) { - var err error - var result *simulations.StepResult - startedAt := time.Now() - - switch adapter { +func NewAdapter(adapterType string, services adapters.Services) (adapter adapters.NodeAdapter, teardown func(), err error) { + teardown = func() {} + switch adapterType { case "sim": - t.Logf("simadapter") - result, err = simf(adapters.NewSimAdapter(services)) + adapter = adapters.NewSimAdapter(services) case "socket": - result, err = simf(adapters.NewSocketAdapter(services)) + adapter = adapters.NewSocketAdapter(services) case "exec": baseDir, err0 := ioutil.TempDir("", "swarm-test") if err0 != nil { - t.Fatal(err0) + return nil, teardown, err0 } - defer os.RemoveAll(baseDir) - result, err = simf(adapters.NewExecAdapter(baseDir)) + teardown = func() { os.RemoveAll(baseDir) } + adapter = adapters.NewExecAdapter(baseDir) case "docker": - adapter, err0 := adapters.NewDockerAdapter() - if err0 != nil { - t.Fatal(err0) + adapter, err = adapters.NewDockerAdapter() + if err != nil { + return nil, teardown, err } - result, err = simf(adapter) default: - t.Fatal("adapter needs to be one of sim, socket, exec, docker") - } - if err != nil { - t.Fatal(err) + return nil, teardown, errors.New("adapter needs to be one of sim, socket, exec, docker") } + return adapter, teardown, nil +} + +func CheckResult(t *testing.T, result *simulations.StepResult, startedAt, finishedAt time.Time) { t.Logf("Simulation with %d nodes passed in %s", len(result.Passes), result.FinishedAt.Sub(result.StartedAt)) var min, max time.Duration var sum int @@ -116,148 +108,101 @@ func testSimulation(t *testing.T, services adapters.Services, adapter string, si sum += int(duration.Nanoseconds()) } t.Logf("Min: %s, Max: %s, Average: %s", min, max, time.Duration(sum/len(result.Passes))*time.Nanosecond) - finishedAt := time.Now() - t.Logf("Setup: %s, shutdown: %s", result.StartedAt.Sub(startedAt), finishedAt.Sub(result.FinishedAt)) + t.Logf("Setup: %s, Shutdown: %s", result.StartedAt.Sub(startedAt), finishedAt.Sub(result.FinishedAt)) } -func runSimulation(nodes, conns int, serviceName string, toAddr func(discover.NodeID) *network.BzzAddr, action func(*simulations.Network) func(context.Context) error, trigger func(*simulations.Network) chan discover.NodeID, check func(*simulations.Network, *storage.DPA) func(context.Context, discover.NodeID) (bool, error), adapter adapters.NodeAdapter) (*simulations.StepResult, error) { +type RunConfig struct { + Adapter string + Step *simulations.Step + NodeCount int + ConnLevel int + ToAddr func(discover.NodeID) *network.BzzAddr + Services adapters.Services +} + +func NewSimulation(conf *RunConfig) (*Simulation, func(), error) { // create network + nodes := conf.NodeCount + adapter, adapterTeardown, err := NewAdapter(conf.Adapter, conf.Services) + if err != nil { + return nil, adapterTeardown, err + } net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{ ID: "0", - DefaultService: serviceName, + DefaultService: "streamer", }) - defer net.Shutdown() + teardown := func() { + adapterTeardown() + net.Shutdown() + } ids := make([]discover.NodeID, nodes) - NodeCount = 0 - Addrs = make([]network.Addr, nodes) + addrs := make([]network.Addr, nodes) // start nodes for i := 0; i < nodes; i++ { node, err := net.NewNode() if err != nil { - return nil, fmt.Errorf("error creating node: %s", err) + return nil, teardown, fmt.Errorf("error creating node: %s", err) } ids[i] = node.ID() - Addrs[i] = toAddr(ids[i]) + addrs[i] = conf.ToAddr(ids[i]) + } + // set nodes number of Stores available + stores, storeTeardown, err := SetStores(addrs...) + teardown = func() { + storeTeardown() + adapterTeardown() + net.Shutdown() } - // set nodes number of localstores globally available - teardown, err := setLocalStores(Addrs...) - defer teardown() if err != nil { - return nil, err + return nil, teardown, err } + s := &Simulation{ + Net: net, + Stores: stores, + IDs: ids, + Addrs: addrs, + } + return s, teardown, nil +} +func (s *Simulation) Run(conf *RunConfig) (*simulations.StepResult, error) { + // bring up nodes, launch the servive + nodes := conf.NodeCount + conns := conf.ConnLevel for i := 0; i < nodes; i++ { - if err := net.Start(ids[i]); err != nil { - return nil, fmt.Errorf("error starting node %s: %s", ids[i].TerminalString(), err) + if err := s.Net.Start(s.IDs[i]); err != nil { + return nil, fmt.Errorf("error starting node %s: %s", s.IDs[i].TerminalString(), err) } } - // run a simulation which connects the 10 nodes in a chain wg := sync.WaitGroup{} - for i := range ids { + for i := range s.IDs { // collect the overlay addresses, to for j := 0; j < conns; j++ { var k int if j == 0 { k = i - 1 } else { - k = rand.Intn(len(ids)) + k = rand.Intn(len(s.IDs)) } if i > 0 { wg.Add(1) go func(i, k int) { defer wg.Done() - net.Connect(ids[i], ids[k]) + s.Net.Connect(s.IDs[i], s.IDs[k]) }(i, k) } } } wg.Wait() - log.Debug(fmt.Sprintf("nodes: %v", len(Addrs))) + log.Debug(fmt.Sprintf("nodes: %v", len(s.Addrs))) // create an only locally retrieving dpa for the pivot node to test // if retriee requests have arrived - dpa := storage.NewDPA(LocalStores[0], storage.NewChunkerParams()) - dpa.Start() - defer dpa.Stop() timeout := 300 * time.Second ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - result := simulations.NewSimulation(net).Run(ctx, &simulations.Step{ - Action: action(net), - Trigger: trigger(net), - Expect: &simulations.Expectation{ - Nodes: ids[0:1], - Check: check(net, dpa), - }, - }) + result := simulations.NewSimulation(s.Net).Run(ctx, conf.Step) return result, nil } - -type roundRobinStore struct { - index uint32 - stores []storage.ChunkStore -} - -func newRoundRobinStore(stores ...storage.ChunkStore) *roundRobinStore { - return &roundRobinStore{ - stores: stores, - } -} - -func (rrs *roundRobinStore) Get(key storage.Key) (*storage.Chunk, error) { - return nil, errors.New("get not well defined on round robin store") -} - -func (rrs *roundRobinStore) Put(chunk *storage.Chunk) { - i := atomic.AddUint32(&rrs.index, 1) - idx := int(i) % len(rrs.stores) - rrs.stores[idx].Put(chunk) -} - -func (rrs *roundRobinStore) Close() { - for _, store := range rrs.stores { - store.Close() - } -} - -type TestStreamerService struct { - // index int - // addr *network.BzzAddr - // // streamer *stream.Registry - run func(p *p2p.Peer, rw p2p.MsgReadWriter) error - spec *protocols.Spec -} - -func NewTestStreamerService(spec *protocols.Spec, run func(p *p2p.Peer, rw p2p.MsgReadWriter) error) *TestStreamerService { - return &TestStreamerService{ - run: run, - spec: spec, - } -} - -func (tds *TestStreamerService) Protocols() []p2p.Protocol { - return []p2p.Protocol{ - { - Name: tds.spec.Name, - Version: tds.spec.Version, - Length: tds.spec.Length(), - Run: tds.run, - // NodeInfo: , - // PeerInfo: , - }, - } -} - -func (b *TestStreamerService) APIs() []rpc.API { - return []rpc.API{} -} - -func (b *TestStreamerService) Start(server *p2p.Server) error { - return nil -} - -func (b *TestStreamerService) Stop() error { - return nil -} diff --git a/swarm/swarm.go b/swarm/swarm.go index 6280d0fce7..b97390e369 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -132,7 +132,7 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e db := storage.NewDBAPI(self.lstore) delivery := stream.NewDelivery(to, db) - self.streamer = stream.NewRegistry(delivery) + self.streamer = stream.NewRegistry(addr, delivery) stream.RegisterSwarmSyncerServer(self.streamer, db) stream.RegisterSwarmSyncerClient(self.streamer, db) @@ -289,7 +289,7 @@ func (self *Swarm) RegisterPssProtocol(spec *protocols.Spec, targetprotocol *p2p } // implements node.Service -// Apis returns the RPC Api descriptors the Swarm implementation offers +// APIs returns the RPC Api descriptors the Swarm implementation offers func (self *Swarm) APIs() []rpc.API { apis := []rpc.API{ From 4c9d0deb690711a8204434eb8bfc6bb4bbd6c5f3 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Fri, 19 Jan 2018 17:41:35 +0100 Subject: [PATCH 068/210] swarm/network/stream: fix syncer tests --- swarm/network/stream/common_test.go | 20 ++++++++--- swarm/network/stream/delivery_test.go | 49 +++++++++++++++++++++------ swarm/network/stream/stream.go | 24 +++++++++++-- swarm/network/stream/syncer_test.go | 16 +++++++-- 4 files changed, 90 insertions(+), 19 deletions(-) diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go index f0468be935..10946b0664 100644 --- a/swarm/network/stream/common_test.go +++ b/swarm/network/stream/common_test.go @@ -38,6 +38,10 @@ var ( loglevel = flag.Int("loglevel", 2, "verbosity of logs") ) +var ( + waitPeerErrC chan error +) + var services = adapters.Services{ "streamer": NewStreamerService, } @@ -49,6 +53,7 @@ func init() { adapters.RegisterServices(services) log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(false)))) + } // newService @@ -60,7 +65,14 @@ func NewStreamerService(ctx *adapters.ServiceContext) (node.Service, error) { db := storage.NewDBAPI(store.(*storage.LocalStore)) delivery := NewDelivery(kad, db) deliveries[id] = delivery - return NewRegistry(addr, delivery, store), nil + //netStore := storage.NewNetStore(store.(*storage.LocalStore), func(*storage.Chunk) error { return errors.New("not retrieved yet") }) + r := NewRegistry(addr, delivery, store) + RegisterSwarmSyncerServer(r, db) + RegisterSwarmSyncerClient(r, db) + go func() { + waitPeerErrC <- waitForPeers(r, 1*time.Second, 1) + }() + return r, nil } func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Registry, *storage.LocalStore, func(), error) { @@ -87,7 +99,7 @@ func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Registry, *stora streamer := NewRegistry(addr, delivery, localStore) protocolTester := p2ptest.NewProtocolTester(t, network.NewNodeIDFromAddr(addr), 1, streamer.runProtocol) - err = waitForPeers(streamer, 1*time.Second) + err = waitForPeers(streamer, 1*time.Second, 1) if err != nil { return nil, nil, nil, nil, errors.New("timeout: peer is not created") } @@ -95,13 +107,13 @@ func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Registry, *stora return protocolTester, streamer, localStore, teardown, nil } -func waitForPeers(streamer *Registry, timeout time.Duration) error { +func waitForPeers(streamer *Registry, timeout time.Duration, expectedPeers int) error { ticker := time.NewTicker(10 * time.Millisecond) timeoutTimer := time.NewTimer(timeout) for { select { case <-ticker.C: - if len(streamer.peers) > 0 { + if streamer.peersCount() >= expectedPeers { return nil } case <-timeoutTimer.C: diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index a37651a148..364e7ddfaa 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -350,22 +350,49 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, size int, skipCheck bool) if err != nil { t.Fatal(err.Error()) } - // create a retriever dpa for the pivot node - delivery := deliveries[sim.IDs[0]] - dpacs := storage.NewNetStore(sim.Stores[0].(*storage.LocalStore), func(chunk *storage.Chunk) error { return delivery.RequestFromPeers(chunk.Key[:], skipCheck) }) - dpa := storage.NewDPA(dpacs, storage.NewChunkerParams()) - dpa.Start() + + waitPeerErrC = make(chan error) + action := func(context.Context) error { - dpa := storage.NewDPA(sim.Stores[0], storage.NewChunkerParams()) + + i := 0 + for err := range waitPeerErrC { + if err != nil { + return fmt.Errorf("error waiting for peers: %s", err) + } + i++ + if i == nodes { + break + } + } + + for i := 0; i < len(sim.IDs)-1; i++ { + id := sim.IDs[i] + node := sim.Net.GetNode(id) + if node == nil { + return fmt.Errorf("unknown node: %s", id) + } + client, err := node.Client() + if err != nil { + return fmt.Errorf("error getting node client: %s", err) + } + sid := sim.IDs[i+1] + if err := client.Call(nil, "stream_subscribeStream", sid, swarmChunkServerStreamName, nil, 0, 0, Top, false); err != nil { + return fmt.Errorf("error subscribing: %s", err) + } + } + + // create a retriever dpa for the pivot node + delivery := deliveries[sim.IDs[0]] + dpacs := storage.NewNetStore(sim.Stores[0].(*storage.LocalStore), func(chunk *storage.Chunk) error { return delivery.RequestFromPeers(chunk.Key[:], skipCheck) }) + dpa := storage.NewDPA(dpacs, storage.NewChunkerParams()) dpa.Start() - // defer dpa.Stop() go func() { defer dpa.Stop() log.Debug(fmt.Sprintf("retrieve %v", fileHash)) // start the retrieval on the pivot node - this will spawn retrieve requests for missing chunks // we must wait for the peer connections to have started before requesting - time.Sleep(2 * time.Second) n, err := mustReadAll(dpa, fileHash) log.Debug(fmt.Sprintf("retrieved %v", fileHash), "read", n, "err", err) }() @@ -388,9 +415,9 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, size int, skipCheck bool) return false, fmt.Errorf("error getting node client: %s", err) } var total int64 - if err := client.Call(&total, "stream_readAll", fileHash); err != nil { - return false, fmt.Errorf("error reading all: %s (read %v)", err, total) - } + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + err = client.CallContext(ctx, &total, "stream_readAll", fileHash) // total, err := mustReadAll(dpa, fileHash) log.Debug(fmt.Sprintf("check if %08x is available locally: number of bytes read %v/%v (error: %v)", fileHash, total, size, err)) if err != nil || total != int64(size) { diff --git a/swarm/network/stream/stream.go b/swarm/network/stream/stream.go index 7565785bee..7cd54fede9 100644 --- a/swarm/network/stream/stream.go +++ b/swarm/network/stream/stream.go @@ -181,6 +181,13 @@ func (r *Registry) deletePeer(peer *Peer) { r.peersMu.Unlock() } +func (r *Registry) peersCount() (c int) { + r.peersMu.Lock() + c = len(r.peers) + r.peersMu.Unlock() + return +} + // Run protocol run function func (r *Registry) run(p *protocols.Peer) error { sp := NewPeer(p, r) @@ -373,9 +380,22 @@ func mustReadAll(dpa *storage.DPA, hash []byte) (int64, error) { } func (api *API) ReadAll(hash []byte) (int64, error) { - return mustReadAll(api.dpa, hash) + r := api.dpa.Retrieve(hash) + buf := make([]byte, 1024) + var n int + var total int64 + var err error + for (total == 0 || n > 0) && err == nil { + n, err = r.ReadAt(buf, total) + total += int64(n) + } + if err != nil && err != io.EOF { + return total, err + } + return total, nil + //return mustReadAll(api.dpa, hash) } -func (api *API) Subscribe(peerId discover.NodeID, s string, t []byte, from, to uint64, priority uint8, live bool) error { +func (api *API) SubscribeStream(peerId discover.NodeID, s string, t []byte, from, to uint64, priority uint8, live bool) error { return api.streamer.Subscribe(peerId, s, t, from, to, priority, live) } diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index cbe2b380b1..0b90658ec4 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -83,8 +83,21 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, size int, skipCheck bool, if err != nil { t.Fatal(err.Error()) } + waitPeerErrC = make(chan error) // create a retriever dpa for the pivot node action := func(context.Context) error { + + i := 0 + for err := range waitPeerErrC { + if err != nil { + return fmt.Errorf("error waiting for peers: %s", err) + } + i++ + if i == nodes { + break + } + } + for i := 0; i < len(sim.IDs)-1; i++ { id := sim.IDs[i] // if err := streamer.Subscribe(p.ID(), "SYNC", []byte{uint8(1)}, 0, 0, Top, false); err != nil { @@ -98,9 +111,8 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, size int, skipCheck bool, if err != nil { return fmt.Errorf("error getting node client: %s", err) } - var n int64 sid := sim.IDs[i+1] - if err := client.Call(&n, "stream_subscribe", sid, "SYNC", []byte{uint8(1)}, 0, 0, Top, false); err != nil { + if err := client.Call(nil, "stream_subscribeStream", sid, "SYNC", []byte{uint8(1)}, 0, 0, Top, false); err != nil { return fmt.Errorf("error subscribing: %s", err) } } From fa602ad9240851cf098f4c0c55b9556c7c057220 Mon Sep 17 00:00:00 2001 From: zelig Date: Sat, 20 Jan 2018 04:05:33 +0100 Subject: [PATCH 069/210] swarm/network/stream: - NewStreamerService now uses netStore - processReceivedChunks fix negation error; chunks found and no Req should be ignored - processReceivedChunks fix logic: no error but ReqC doesnt block - rename mustReadAll to ReadAll - use common.Hash in RPC hash - on streamer api field - streamer API ReadAll function takes hexencoded string, byte slice aint cut it - streamer service Start/Stop start and stop api.dpa. fixes nil chunk channel issue --- swarm/network/stream/common_test.go | 4 +-- swarm/network/stream/delivery.go | 14 +++------- swarm/network/stream/delivery_test.go | 38 ++++++++++++++------------- swarm/network/stream/stream.go | 30 ++++++++------------- swarm/network/stream/syncer_test.go | 4 +-- swarm/storage/chunker.go | 4 +-- swarm/storage/netstore.go | 5 ---- 7 files changed, 39 insertions(+), 60 deletions(-) diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go index 10946b0664..0387abab09 100644 --- a/swarm/network/stream/common_test.go +++ b/swarm/network/stream/common_test.go @@ -65,8 +65,8 @@ func NewStreamerService(ctx *adapters.ServiceContext) (node.Service, error) { db := storage.NewDBAPI(store.(*storage.LocalStore)) delivery := NewDelivery(kad, db) deliveries[id] = delivery - //netStore := storage.NewNetStore(store.(*storage.LocalStore), func(*storage.Chunk) error { return errors.New("not retrieved yet") }) - r := NewRegistry(addr, delivery, store) + netStore := storage.NewNetStore(store.(*storage.LocalStore), func(*storage.Chunk) error { return nil }) + r := NewRegistry(addr, delivery, netStore) RegisterSwarmSyncerServer(r, db) RegisterSwarmSyncerClient(r, db) go func() { diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go index d242dd01f3..9d485eb7ec 100644 --- a/swarm/network/stream/delivery.go +++ b/swarm/network/stream/delivery.go @@ -18,10 +18,8 @@ package stream import ( "errors" - "fmt" "time" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/storage" @@ -151,27 +149,21 @@ type ChunkDeliveryMsg struct { } func (d *Delivery) handleChunkDeliveryMsg(req *ChunkDeliveryMsg) error { - chunk, err := d.db.Get(req.Key) - if err != nil { - return err - } - d.receiveC <- req - - log.Trace(fmt.Sprintf("delivery of %v from %v", chunk, d)) return nil } func (d *Delivery) processReceivedChunks() { for req := range d.receiveC { + // this should be has locally chunk, err := d.db.Get(req.Key) - if err != nil { + if err == nil && chunk.ReqC == nil { continue } - chunk.SData = req.SData select { case <-chunk.ReqC: default: + chunk.SData = req.SData d.db.Put(chunk) close(chunk.ReqC) } diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index 364e7ddfaa..5a8ed5ec8e 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -25,6 +25,7 @@ import ( "testing" "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/simulations" @@ -335,26 +336,19 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, size int, skipCheck bool) rrdpa := storage.NewDPA(newRoundRobinStore(sim.Stores[1:]...), storage.NewChunkerParams()) rrdpa.Start() fileHash, wait, err := rrdpa.Store(io.LimitReader(crand.Reader, int64(size)), int64(size)) - defer rrdpa.Stop() - if err != nil { - t.Fatal(err.Error()) - } // wait until all chunks stored - // TODO: is wait() necessary? wait() - // each node Subscribes to each other's swarmChunkServerStreamName - // need to wait till an aynchronous process registers the peers in streamer.peers - // that is used by Subscribe - // time.Sleep(1 * time.Second) - // err := streamer.Subscribe(p.ID(), swarmChunkServerStreamName, nil, 0, 0, Top, true) + rrdpa.Stop() if err != nil { t.Fatal(err.Error()) } - waitPeerErrC = make(chan error) - action := func(context.Context) error { - + // each node Subscribes to each other's swarmChunkServerStreamName + // need to wait till an aynchronous process registers the peers in streamer.peers + // that is used by Subscribe + // using a global err channel to share betweem action and node service + waitPeerErrC = make(chan error) i := 0 for err := range waitPeerErrC { if err != nil { @@ -366,6 +360,8 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, size int, skipCheck bool) } } + // each node subscribes to the upstream swarm chunk server stream + // which responds to chunk retrieve requests all but the last node in the chain does not for i := 0; i < len(sim.IDs)-1; i++ { id := sim.IDs[i] node := sim.Net.GetNode(id) @@ -376,6 +372,9 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, size int, skipCheck bool) if err != nil { return fmt.Errorf("error getting node client: %s", err) } + // rpc call to streamer API subscribing to chunk Server to their + // unique upstream except for the last node in the chain + // Note in this test we only test one direction sid := sim.IDs[i+1] if err := client.Call(nil, "stream_subscribeStream", sid, swarmChunkServerStreamName, nil, 0, 0, Top, false); err != nil { return fmt.Errorf("error subscribing: %s", err) @@ -384,16 +383,18 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, size int, skipCheck bool) // create a retriever dpa for the pivot node delivery := deliveries[sim.IDs[0]] - dpacs := storage.NewNetStore(sim.Stores[0].(*storage.LocalStore), func(chunk *storage.Chunk) error { return delivery.RequestFromPeers(chunk.Key[:], skipCheck) }) + retrieveFunc := func(chunk *storage.Chunk) error { + return delivery.RequestFromPeers(chunk.Key[:], skipCheck) + } + dpacs := storage.NewNetStore(sim.Stores[0].(*storage.LocalStore), retrieveFunc) dpa := storage.NewDPA(dpacs, storage.NewChunkerParams()) dpa.Start() go func() { defer dpa.Stop() - log.Debug(fmt.Sprintf("retrieve %v", fileHash)) // start the retrieval on the pivot node - this will spawn retrieve requests for missing chunks // we must wait for the peer connections to have started before requesting - n, err := mustReadAll(dpa, fileHash) + n, err := readAll(dpa, fileHash) log.Debug(fmt.Sprintf("retrieved %v", fileHash), "read", n, "err", err) }() return nil @@ -417,8 +418,8 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, size int, skipCheck bool) var total int64 ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() - err = client.CallContext(ctx, &total, "stream_readAll", fileHash) - // total, err := mustReadAll(dpa, fileHash) + // call RPC method to streamer API readAll method to check local availability + err = client.CallContext(ctx, &total, "stream_readAll", common.BytesToHash(fileHash)) log.Debug(fmt.Sprintf("check if %08x is available locally: number of bytes read %v/%v (error: %v)", fileHash, total, size, err)) if err != nil || total != int64(size) { return false, nil @@ -439,6 +440,7 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, size int, skipCheck bool) conf.Step = &simulations.Step{ Action: action, Trigger: trigger, + // we are only testing the pivot node (net.Nodes[0]) Expect: &simulations.Expectation{ Nodes: sim.IDs[0:1], Check: check, diff --git a/swarm/network/stream/stream.go b/swarm/network/stream/stream.go index 7cd54fede9..8e3e3ea223 100644 --- a/swarm/network/stream/stream.go +++ b/swarm/network/stream/stream.go @@ -22,6 +22,7 @@ import ( "math" "sync" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rpc" @@ -44,6 +45,7 @@ const ( // Registry registry for outgoing and incoming streamer constructors type Registry struct { + api *API addr *network.BzzAddr clientMu sync.RWMutex serverMu sync.RWMutex @@ -65,6 +67,7 @@ func NewRegistry(addr *network.BzzAddr, delivery *Delivery, store storage.ChunkS peers: make(map[discover.NodeID]*Peer), delivery: delivery, } + streamer.api = NewAPI(streamer, streamer.store) delivery.getPeer = streamer.getPeer streamer.RegisterServerFunc(swarmChunkServerStreamName, func(_ *Peer, t []byte) (Server, error) { return NewSwarmChunkServer(delivery.db), nil @@ -271,7 +274,7 @@ type Client interface { BatchDone(string, uint64, []byte, []byte) func() (*TakeoverProof, error) } -// NextBatch adjusts the indexes by inspecting the intervals +// nextBatch adjusts the indexes by inspecting the intervals func (c *client) nextBatch(from uint64) (nextFrom uint64, nextTo uint64) { var intervals []uint64 if c.live { @@ -302,7 +305,7 @@ func (c *client) nextBatch(from uint64) (nextFrom uint64, nextTo uint64) { return nextFrom, nextTo } -// Spec is the spec of the streamer protocol. +// Spec is the spec of the streamer protocol var Spec = &protocols.Spec{ Name: "stream", Version: 1, @@ -336,17 +339,19 @@ func (r *Registry) APIs() []rpc.API { { Namespace: "stream", Version: "0.1", - Service: NewAPI(r, r.store), + Service: r.api, Public: true, }, } } func (r *Registry) Start(server *p2p.Server) error { + r.api.dpa.Start() return nil } func (r *Registry) Stop() error { + r.api.dpa.Stop() return nil } @@ -363,7 +368,7 @@ func NewAPI(r *Registry, store storage.ChunkStore) *API { } } -func mustReadAll(dpa *storage.DPA, hash []byte) (int64, error) { +func readAll(dpa *storage.DPA, hash []byte) (int64, error) { r := dpa.Retrieve(hash) buf := make([]byte, 1024) var n int @@ -379,21 +384,8 @@ func mustReadAll(dpa *storage.DPA, hash []byte) (int64, error) { return total, nil } -func (api *API) ReadAll(hash []byte) (int64, error) { - r := api.dpa.Retrieve(hash) - buf := make([]byte, 1024) - var n int - var total int64 - var err error - for (total == 0 || n > 0) && err == nil { - n, err = r.ReadAt(buf, total) - total += int64(n) - } - if err != nil && err != io.EOF { - return total, err - } - return total, nil - //return mustReadAll(api.dpa, hash) +func (api *API) ReadAll(hash common.Hash) (int64, error) { + return readAll(api.dpa, hash[:]) } func (api *API) SubscribeStream(peerId discover.NodeID, s string, t []byte, from, to uint64, priority uint8, live bool) error { diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index 0b90658ec4..9dde5bda72 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -35,7 +35,9 @@ import ( func TestSyncerSimulation(t *testing.T) { testSyncBetweenNodes(t, 2, 1, 81000, true, 1) + testSyncBetweenNodes(t, 2, 1, 81000, false, 1) testSyncBetweenNodes(t, 3, 1, 81000, true, 1) + testSyncBetweenNodes(t, 3, 1, 81000, false, 1) } func testSyncBetweenNodes(t *testing.T, nodes, conns, size int, skipCheck bool, po uint8) { @@ -59,7 +61,6 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, size int, skipCheck bool, } stores = make(map[discover.NodeID]storage.ChunkStore) deliveries = make(map[discover.NodeID]*Delivery) - log.Warn("Stores", "len", len(sim.Stores)) for i, id := range sim.IDs { stores[id] = sim.Stores[i] } @@ -136,7 +137,6 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, size int, skipCheck bool, var found, total int for i := 1; i < nodes; i++ { - dbs[i].Iterator(0, math.MaxUint64, po, func(key storage.Key, index uint64) bool { _, err := dbs[0].Get(key) if err == nil { diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index 2ea81403bf..9ba6f5c1e0 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -13,7 +13,6 @@ // // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . - package storage import ( @@ -463,8 +462,7 @@ func retrieve(key Key, chunkC chan *Chunk, quitC chan bool) *Chunk { case <-chunk.C: // bells are ringing, data have been delivered } if len(chunk.SData) == 0 { - return nil // chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) - + return nil } return chunk } diff --git a/swarm/storage/netstore.go b/swarm/storage/netstore.go index f01ffe4a69..334cf3635a 100644 --- a/swarm/storage/netstore.go +++ b/swarm/storage/netstore.go @@ -18,10 +18,7 @@ package storage import ( "encoding/binary" - "fmt" "time" - - "github.com/ethereum/go-ethereum/log" ) // NetStore implements the ChunkStore interface, @@ -43,7 +40,6 @@ func (self *NetStore) Get(key Key) (chunk *Chunk, err error) { var created bool chunk, created = self.localStore.GetOrCreateRequest(key) if chunk.ReqC == nil { - log.Trace(fmt.Sprintf("DPA.Get: %v found locally, %d bytes", key.Log(), len(chunk.SData))) return } @@ -57,7 +53,6 @@ func (self *NetStore) Get(key Key) (chunk *Chunk, err error) { select { case <-t.C: - log.Trace(fmt.Sprintf("DPA.Get: %v request time out ", key.Log())) return nil, notFound case <-chunk.ReqC: } From 98ba78c5218889f9159d8db1c5d5c29167cf7fc4 Mon Sep 17 00:00:00 2001 From: zelig Date: Sun, 21 Jan 2018 20:30:02 +0100 Subject: [PATCH 070/210] swarm/network, swarm/storage, p2p/similations: fix stream tests add delivery benchmarks - memstore garbage collect does not delete open requests - dbstore batch write errors handling somewhat impoved but should be checkable after dbStored fires - inproc adapter does not allow message events in p2p server - network pkg allows loglevel flag - streamer registry gets a defaultSkipCheck option which the RequestFromPeers uses - waitForPeers gets a parameter how many peers it should wait for - number of peers to wait for is given by the global peerCount NodeID -> int function - make delivery and received chunk channels buffered with const deliveryCap - change all sendpriority calls to use context to avoid disconnects due to buffer contention (but potential memory leak!) - check error for all sends - abstract out trigger function for pivot - abstract out client calls for IDs - introduce channel to control check function is only called if previous one finished even if triggered (dubious) - implement benchmarks for delivery through a chain of requests through multiple hops - abstract out batchDone function on client - rename peer locks to client/serverMu - testing CheckResult only gives averages if there are more than one node to passed - testing implememts WatchDisconnections which aborts the simulation - testing implements PivorTrigger and ClientCall --- p2p/simulations/adapters/inproc.go | 2 +- swarm/network/protocol_test.go | 13 + swarm/network/stream/common_test.go | 13 +- swarm/network/stream/delivery.go | 54 ++-- swarm/network/stream/delivery_test.go | 315 ++++++++++++++++++++---- swarm/network/stream/messages.go | 35 +-- swarm/network/stream/peer.go | 47 ++-- swarm/network/stream/stream.go | 23 +- swarm/network/stream/syncer.go | 12 +- swarm/network/stream/syncer_test.go | 130 +++++----- swarm/network/stream/testing/testing.go | 90 +++++-- swarm/storage/dbstore.go | 10 +- swarm/storage/memstore.go | 6 +- 13 files changed, 541 insertions(+), 209 deletions(-) diff --git a/p2p/simulations/adapters/inproc.go b/p2p/simulations/adapters/inproc.go index 0d22b4f56f..6ecacd87a7 100644 --- a/p2p/simulations/adapters/inproc.go +++ b/p2p/simulations/adapters/inproc.go @@ -112,7 +112,7 @@ func (s *SimAdapter) NewNode(config *NodeConfig) (Node, error) { MaxPeers: math.MaxInt32, NoDiscovery: true, Dialer: s, - EnableMsgEvents: true, + EnableMsgEvents: false, }, NoUSB: true, Logger: log.New("node.id", id.String()), diff --git a/swarm/network/protocol_test.go b/swarm/network/protocol_test.go index fdabddb1c3..c603da7e8e 100644 --- a/swarm/network/protocol_test.go +++ b/swarm/network/protocol_test.go @@ -17,16 +17,29 @@ package network import ( + "flag" "fmt" + "os" "sync" "testing" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/protocols" p2ptest "github.com/ethereum/go-ethereum/p2p/testing" ) +var ( + adapter = flag.String("adapter", "sim", "type of simulation: sim|socket|exec|docker") + loglevel = flag.Int("loglevel", 2, "verbosity of logs") +) + +func init() { + flag.Parse() + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) +} + type testStore struct { sync.Mutex diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go index 0387abab09..7997e977e9 100644 --- a/swarm/network/stream/common_test.go +++ b/swarm/network/stream/common_test.go @@ -39,7 +39,9 @@ var ( ) var ( - waitPeerErrC chan error + defaultSkipCheck bool + waitPeerErrC chan error + chunkSize = 4096 ) var services = adapters.Services{ @@ -56,7 +58,7 @@ func init() { } -// newService +// NewStreamerService func NewStreamerService(ctx *adapters.ServiceContext) (node.Service, error) { id := ctx.Config.ID addr := toAddr(id) @@ -65,12 +67,11 @@ func NewStreamerService(ctx *adapters.ServiceContext) (node.Service, error) { db := storage.NewDBAPI(store.(*storage.LocalStore)) delivery := NewDelivery(kad, db) deliveries[id] = delivery - netStore := storage.NewNetStore(store.(*storage.LocalStore), func(*storage.Chunk) error { return nil }) - r := NewRegistry(addr, delivery, netStore) + r := NewRegistry(addr, delivery, store, defaultSkipCheck) RegisterSwarmSyncerServer(r, db) RegisterSwarmSyncerClient(r, db) go func() { - waitPeerErrC <- waitForPeers(r, 1*time.Second, 1) + waitPeerErrC <- waitForPeers(r, 1*time.Second, peerCount(id)) }() return r, nil } @@ -96,7 +97,7 @@ func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Registry, *stora db := storage.NewDBAPI(localStore) delivery := NewDelivery(to, db) - streamer := NewRegistry(addr, delivery, localStore) + streamer := NewRegistry(addr, delivery, localStore, defaultSkipCheck) protocolTester := p2ptest.NewProtocolTester(t, network.NewNodeIDFromAddr(addr), 1, streamer.runProtocol) err = waitForPeers(streamer, 1*time.Second, 1) diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go index 9d485eb7ec..67181c3f22 100644 --- a/swarm/network/stream/delivery.go +++ b/swarm/network/stream/delivery.go @@ -20,12 +20,16 @@ import ( "errors" "time" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/storage" ) -const swarmChunkServerStreamName = "RETRIEVE_REQUEST" +const ( + swarmChunkServerStreamName = "RETRIEVE_REQUEST" + deliveryCap = 32 +) type Delivery struct { db *storage.DBAPI @@ -39,7 +43,7 @@ func NewDelivery(overlay network.Overlay, db *storage.DBAPI) *Delivery { d := &Delivery{ db: db, overlay: overlay, - receiveC: make(chan *ChunkDeliveryMsg, 10), + receiveC: make(chan *ChunkDeliveryMsg, deliveryCap), } go d.processReceivedChunks() @@ -57,7 +61,7 @@ type SwarmChunkServer struct { // NewSwarmChunkServer is SwarmChunkServer constructor func NewSwarmChunkServer(db *storage.DBAPI) *SwarmChunkServer { s := &SwarmChunkServer{ - deliveryC: make(chan []byte), + deliveryC: make(chan []byte, deliveryCap), batchC: make(chan []byte), db: db, } @@ -103,6 +107,7 @@ type RetrieveRequestMsg struct { } func (d *Delivery) handleRetrieveRequestMsg(sp *Peer, req *RetrieveRequestMsg) error { + log.Debug("received request", "peer", sp.ID(), "hash", req.Key) s, err := sp.getServer(swarmChunkServerStreamName) if err != nil { return err @@ -112,6 +117,7 @@ func (d *Delivery) handleRetrieveRequestMsg(sp *Peer, req *RetrieveRequestMsg) e if chunk.ReqC != nil { if created { if err := d.RequestFromPeers(chunk.Key[:], false, sp.ID()); err != nil { + log.Warn("unable to forward chunk request", "peer", sp.ID(), "key", chunk.Key, "err", err) return nil } } @@ -128,8 +134,10 @@ func (d *Delivery) handleRetrieveRequestMsg(sp *Peer, req *RetrieveRequestMsg) e } if req.SkipCheck { - sp.Deliver(chunk, s.priority) - return + err := sp.Deliver(chunk, s.priority) + if err != nil { + sp.Drop(err) + } } streamer.deliveryC <- chunk.Key[:] }() @@ -137,6 +145,7 @@ func (d *Delivery) handleRetrieveRequestMsg(sp *Peer, req *RetrieveRequestMsg) e } // TODO: call the retrieve function of the outgoing syncer if req.SkipCheck { + log.Trace("deliver", "peer", sp.ID(), "hash", chunk.Key) return sp.Deliver(chunk, s.priority) } streamer.deliveryC <- chunk.Key[:] @@ -154,45 +163,60 @@ func (d *Delivery) handleChunkDeliveryMsg(req *ChunkDeliveryMsg) error { } func (d *Delivery) processReceivedChunks() { +R: for req := range d.receiveC { // this should be has locally chunk, err := d.db.Get(req.Key) - if err == nil && chunk.ReqC == nil { - continue + if err != nil { + log.Error("not in db? ", "key", req.Key, "chunk", chunk) + continue R + } + if chunk.ReqC == nil { + continue R } select { case <-chunk.ReqC: + continue R default: - chunk.SData = req.SData - d.db.Put(chunk) - close(chunk.ReqC) } + chunk.SData = req.SData + d.db.Put(chunk) + log.Warn("reecived delivery", "hash", chunk.Key) + chunk.WaitToStore() + log.Warn("received delivery stored", "hash", chunk.Key) + close(chunk.ReqC) + log.Warn("received delivery requesters notified", "hash", chunk.Key) } } // RequestFromPeers sends a chunk retrieve request to func (d *Delivery) RequestFromPeers(hash []byte, skipCheck bool, peersToSkip ...discover.NodeID) error { var success bool + var err error + log.Warn("request", "hash", hash) d.overlay.EachConn(hash, 255, func(p network.OverlayConn, po int, nn bool) bool { spId := p.(*network.BzzPeer).ID() for _, p := range peersToSkip { if p == spId { + log.Warn("skip peer", "peer", spId) return true } } sp := d.getPeer(spId) + if sp == nil { + log.Warn("peer not found", "id", spId) + return true + } // TODO: skip light nodes that do not accept retrieve requests - err := sp.SendPriority(&RetrieveRequestMsg{ + err = sp.SendPriority(&RetrieveRequestMsg{ Key: hash, SkipCheck: skipCheck, }, Top) - if err == nil { - success = true - } + success = true return false }) if success { - return nil + return err } return errors.New("no peer found") } diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index 5a8ed5ec8e..1ca89ea5e6 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/simulations" p2ptest "github.com/ethereum/go-ethereum/p2p/testing" + "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm/network" streamTesting "github.com/ethereum/go-ethereum/swarm/network/stream/testing" "github.com/ethereum/go-ethereum/swarm/storage" @@ -39,6 +40,7 @@ var ( deliveries map[discover.NodeID]*Delivery stores map[discover.NodeID]storage.ChunkStore toAddr func(discover.NodeID) *network.BzzAddr + peerCount func(discover.NodeID) int ) func TestStreamerRetrieveRequest(t *testing.T) { @@ -305,13 +307,18 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { } func TestDeliveryFromNodes(t *testing.T) { - testDeliveryFromNodes(t, 2, 1, 8100, true) - testDeliveryFromNodes(t, 2, 1, 8100, false) - testDeliveryFromNodes(t, 3, 1, 8100, true) - testDeliveryFromNodes(t, 3, 1, 8100, false) + testDeliveryFromNodes(t, 2, 1, dataChunkCount, true) + testDeliveryFromNodes(t, 2, 1, dataChunkCount, false) + testDeliveryFromNodes(t, 4, 1, dataChunkCount, true) + testDeliveryFromNodes(t, 4, 1, dataChunkCount, false) + testDeliveryFromNodes(t, 8, 1, dataChunkCount, true) + testDeliveryFromNodes(t, 8, 1, dataChunkCount, false) + testDeliveryFromNodes(t, 16, 1, dataChunkCount, true) + testDeliveryFromNodes(t, 16, 1, dataChunkCount, false) } -func testDeliveryFromNodes(t *testing.T, nodes, conns, size int, skipCheck bool) { +func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck bool) { + defaultSkipCheck = skipCheck toAddr = network.NewAddrFromNodeID conf := &streamTesting.RunConfig{ Adapter: *adapter, @@ -331,24 +338,33 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, size int, skipCheck bool) for i, id := range sim.IDs { stores[id] = sim.Stores[i] } + peerCount = func(id discover.NodeID) int { + if sim.IDs[0] == id || sim.IDs[nodes-1] == id { + return 1 + } + return 2 + } // here we distribute chunks of a random file into Stores of nodes 1 to nodes rrdpa := storage.NewDPA(newRoundRobinStore(sim.Stores[1:]...), storage.NewChunkerParams()) rrdpa.Start() + size := chunkCount * chunkSize fileHash, wait, err := rrdpa.Store(io.LimitReader(crand.Reader, int64(size)), int64(size)) // wait until all chunks stored wait() - rrdpa.Stop() + defer rrdpa.Stop() if err != nil { t.Fatal(err.Error()) } + errc := make(chan error, 1) + waitPeerErrC = make(chan error) + quitC := make(chan struct{}) - action := func(context.Context) error { + action := func(ctx context.Context) error { // each node Subscribes to each other's swarmChunkServerStreamName // need to wait till an aynchronous process registers the peers in streamer.peers // that is used by Subscribe // using a global err channel to share betweem action and node service - waitPeerErrC = make(chan error) i := 0 for err := range waitPeerErrC { if err != nil { @@ -362,23 +378,20 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, size int, skipCheck bool) // each node subscribes to the upstream swarm chunk server stream // which responds to chunk retrieve requests all but the last node in the chain does not - for i := 0; i < len(sim.IDs)-1; i++ { - id := sim.IDs[i] - node := sim.Net.GetNode(id) - if node == nil { - return fmt.Errorf("unknown node: %s", id) - } - client, err := node.Client() + var j int + err := sim.CallClient(func(client *rpc.Client) error { + err := streamTesting.WatchDisconnections(sim.IDs[j], client, errc, quitC) if err != nil { - return fmt.Errorf("error getting node client: %s", err) - } - // rpc call to streamer API subscribing to chunk Server to their - // unique upstream except for the last node in the chain - // Note in this test we only test one direction - sid := sim.IDs[i+1] - if err := client.Call(nil, "stream_subscribeStream", sid, swarmChunkServerStreamName, nil, 0, 0, Top, false); err != nil { - return fmt.Errorf("error subscribing: %s", err) + return err } + ctx, cancel := context.WithTimeout(ctx, 1*time.Second) + defer cancel() + j++ + sid := sim.IDs[j] + return client.CallContext(ctx, nil, "stream_subscribeStream", sid, swarmChunkServerStreamName, nil, 0, 0, Top, false) + }, sim.IDs[0:nodes-1]...) + if err != nil { + return err } // create a retriever dpa for the pivot node @@ -386,8 +399,8 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, size int, skipCheck bool) retrieveFunc := func(chunk *storage.Chunk) error { return delivery.RequestFromPeers(chunk.Key[:], skipCheck) } - dpacs := storage.NewNetStore(sim.Stores[0].(*storage.LocalStore), retrieveFunc) - dpa := storage.NewDPA(dpacs, storage.NewChunkerParams()) + netStore := storage.NewNetStore(sim.Stores[0].(*storage.LocalStore), retrieveFunc) + dpa := storage.NewDPA(netStore, storage.NewChunkerParams()) dpa.Start() go func() { @@ -395,51 +408,40 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, size int, skipCheck bool) // start the retrieval on the pivot node - this will spawn retrieve requests for missing chunks // we must wait for the peer connections to have started before requesting n, err := readAll(dpa, fileHash) - log.Debug(fmt.Sprintf("retrieved %v", fileHash), "read", n, "err", err) + log.Info(fmt.Sprintf("retrieved %v", fileHash), "read", n, "err", err) + if err != nil { + errc <- fmt.Errorf("requesting chunks action error: %v", err) + } }() return nil } - + checkC := make(chan struct{}) check := func(ctx context.Context, id discover.NodeID) (bool, error) { + defer func() { checkC <- struct{}{} }() select { + case err := <-errc: + return false, err case <-ctx.Done(): return false, ctx.Err() default: } - // try to locally retrieve the file to check if retrieve requests have been successful - node := sim.Net.GetNode(id) - if node == nil { - return false, fmt.Errorf("unknown node: %s", id) - } - client, err := node.Client() - if err != nil { - return false, fmt.Errorf("error getting node client: %s", err) - } var total int64 - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - // call RPC method to streamer API readAll method to check local availability - err = client.CallContext(ctx, &total, "stream_readAll", common.BytesToHash(fileHash)) - log.Debug(fmt.Sprintf("check if %08x is available locally: number of bytes read %v/%v (error: %v)", fileHash, total, size, err)) + err := sim.CallClient(func(client *rpc.Client) error { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + return client.CallContext(ctx, &total, "stream_readAll", common.BytesToHash(fileHash)) + }, id) + log.Info(fmt.Sprintf("check if %08x is available locally: number of bytes read %v/%v (error: %v)", fileHash, total, size, err)) if err != nil || total != int64(size) { return false, nil } + close(quitC) return true, nil } - trigger := make(chan discover.NodeID) - ticker := time.NewTicker(500 * time.Millisecond) - go func() { - defer ticker.Stop() - // we are only testing the pivot node (net.Nodes[0]) - for range ticker.C { - trigger <- sim.Net.Nodes[0].ID() - } - }() - conf.Step = &simulations.Step{ Action: action, - Trigger: trigger, + Trigger: streamTesting.PivotTrigger(10*time.Millisecond, checkC, sim.IDs[0]), // we are only testing the pivot node (net.Nodes[0]) Expect: &simulations.Expectation{ Nodes: sim.IDs[0:1], @@ -457,3 +459,212 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, size int, skipCheck bool) } streamTesting.CheckResult(t, result, startedAt, finishedAt) } + +func BenchmarkDeliveryFromNodesWithoutCheck(b *testing.B) { + for chunks := 32; chunks <= 128; chunks *= 2 { + for i := 2; i < 32; i *= 2 { + b.Run( + fmt.Sprintf("nodes=%v,chunks=%v", i, chunks), + func(b *testing.B) { + benchmarkDeliveryFromNodes(b, i, 1, chunks, true) + }, + ) + } + } +} + +func BenchmarkDeliveryFromNodesWithCheck(b *testing.B) { + for chunks := 32; chunks <= 128; chunks *= 2 { + for i := 2; i < 32; i *= 2 { + b.Run( + fmt.Sprintf("nodes=%v,chunks=%v", i, chunks), + func(b *testing.B) { + benchmarkDeliveryFromNodes(b, i, 1, chunks, false) + }, + ) + } + } +} + +func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skipCheck bool) { + toAddr = network.NewAddrFromNodeID + conf := &streamTesting.RunConfig{ + Adapter: *adapter, + NodeCount: nodes, + ConnLevel: conns, + ToAddr: toAddr, + Services: services, + } + defaultSkipCheck = skipCheck + sim, teardown, err := streamTesting.NewSimulation(conf) + defer teardown() + if err != nil { + b.Fatal(err.Error()) + } + stores = make(map[discover.NodeID]storage.ChunkStore) + deliveries = make(map[discover.NodeID]*Delivery) + for i, id := range sim.IDs { + stores[id] = sim.Stores[i] + } + peerCount = func(id discover.NodeID) int { + if sim.IDs[0] == id || sim.IDs[nodes-1] == id { + return 1 + } + return 2 + } + // create a dpa for the last node in the chain which we are gonna write to + remoteDpa := storage.NewDPA(sim.Stores[nodes-1], storage.NewChunkerParams()) + remoteDpa.Start() + defer remoteDpa.Stop() + + // wait channel for all nodes all peer connections to set up + waitPeerErrC = make(chan error) + // channel to signal simulation initialisation with action call complete + // or node disconnections + simErrC := make(chan error) + quitC := make(chan struct{}) + + action := func(ctx context.Context) error { + // each node Subscribes to each other's swarmChunkServerStreamName + // need to wait till an aynchronous process registers the peers in streamer.peers + // that is used by Subscribe + // waitPeerErrC using a global err channel to share betweem action and node service + i := 0 + for err := range waitPeerErrC { + if err != nil { + return fmt.Errorf("error waiting for peers: %s", err) + } + i++ + if i == nodes { + break + } + } + + // each node except the last one subscribes to the upstream swarm chunk server stream + // which responds to chunk retrieve requests + var j int + simErrC <- sim.CallClient(func(client *rpc.Client) error { + err := streamTesting.WatchDisconnections(sim.IDs[j], client, simErrC, quitC) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, 1*time.Second) + defer cancel() + j++ + sid := sim.IDs[j] // the upstream peer's id + return client.CallContext(ctx, nil, "stream_subscribeStream", sid, swarmChunkServerStreamName, nil, 0, 0, Top, false) + }, sim.IDs[0:nodes-1]...) + // signal to the benchmark that setup is complete + return err + } + + // the check function is only triggered when the benchmark finishes + checkC := make(chan error) + trigger := make(chan discover.NodeID) + check := func(ctx context.Context, id discover.NodeID) (_ bool, err error) { + select { + case <-ctx.Done(): + err = ctx.Err() + case err = <-checkC: + } + if err != nil { + return false, err + } + return true, nil + } + + conf.Step = &simulations.Step{ + Action: action, + Trigger: trigger, + // we are only testing the pivot node (net.Nodes[0]) + Expect: &simulations.Expectation{ + Nodes: sim.IDs[0:1], + Check: check, + }, + } + + // run the simulation in the background + errc := make(chan error) + go func() { + _, err := sim.Run(conf) + errc <- err + }() + + // wait for simulation action to complete stream subscriptions + err = <-simErrC + if err != nil { + b.Fatalf("simulation failed to initialise. expected no error. got %v", err) + } + go func() { + for { + var err error + select { + case err = <-simErrC: + case <-quitC: + } + trigger <- sim.IDs[0] + checkC <- err + } + }() + + // create a retriever dpa for the pivot node + // by now deliveries are set for each node by the streamer service + delivery := deliveries[sim.IDs[0]] + retrieveFunc := func(chunk *storage.Chunk) error { + return delivery.RequestFromPeers(chunk.Key[:], skipCheck) + } + netStore := storage.NewNetStore(sim.Stores[0].(*storage.LocalStore), retrieveFunc) + + // benchmark loop + b.ResetTimer() + b.StopTimer() + for i := 0; i < b.N; i++ { + // uploading chunkCount random chunks to the last node + hashes := make([]storage.Key, chunkCount) + for i := 0; i < chunkCount; i++ { + // create actual size real chunks + hash, wait, err := remoteDpa.Store(io.LimitReader(crand.Reader, int64(chunkSize)), int64(chunkSize)) + // wait until all chunks stored + wait() + if err != nil { + b.Fatalf("expected no error. got %v", err) + } + // collect the hashes + hashes[i] = hash + } + // now benchmark the actual retrieval + // netstore.Get is called for each hash in a go routine and errors are collected + b.StartTimer() + errs := make(chan error) + for _, hash := range hashes { + go func(h storage.Key) { + _, err := netStore.Get(h) + log.Warn("test check netstore get", "hash", h, "err", err) + errs <- err + }(hash) + } + // count and report retrieval errors + // if there are misses then chunk timeout is too low for the distance and volume (?) + var total, misses int + for err := range errs { + if err != nil { + log.Warn(err.Error()) + misses++ + } + total++ + if total == chunkCount { + break + } + } + b.StopTimer() + if misses > 0 { + simErrC <- fmt.Errorf("%v chunk not found out of %v", misses, total) + } + } + // benchmark over, trigger the check function to conclude the simulation + close(quitC) + err = <-errc + if err != nil { + b.Fatalf("expected no error. got %v", err) + } +} diff --git a/swarm/network/stream/messages.go b/swarm/network/stream/messages.go index 1e5e281b91..a575b915a6 100644 --- a/swarm/network/stream/messages.go +++ b/swarm/network/stream/messages.go @@ -79,8 +79,12 @@ func (p *Peer) handleSubscribeMsg(req *SubscribeMsg) error { if err != nil { return nil } - log.Debug("received subscription", "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To) - go p.SendOfferedHashes(os, req.From, req.To) + log.Debug("received subscription", "peer", p.ID(), "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To) + go func() { + if err := p.SendOfferedHashes(os, req.From, req.To); err != nil { + p.Drop(err) + } + }() return nil } @@ -128,14 +132,7 @@ func (p *Peer) handleOfferedHashesMsg(req *OfferedHashesMsg) error { } go func() { wg.Wait() - if tf := s.BatchDone(req.Stream, req.From, hashes, req.Root); tf != nil { - tp, err := tf() - if err != nil { - return - } - p.SendPriority(tp, s.priority) - } - s.next <- struct{}{} + s.next <- s.batchDone(p, req, hashes) }() // only send wantedKeysMsg if all missing chunks of the previous batch arrived // except @@ -143,7 +140,7 @@ func (p *Peer) handleOfferedHashesMsg(req *OfferedHashesMsg) error { s.sessionAt = req.From } from, to := s.nextBatch(req.To) - log.Debug("received batch", "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To) + log.Trace("received offered batch", "peer", p.ID(), "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To) if from == to { return nil } @@ -157,12 +154,19 @@ func (p *Peer) handleOfferedHashesMsg(req *OfferedHashesMsg) error { } go func() { select { - case <-s.next: + case err := <-s.next: + if err != nil { + p.Drop(err) + return + } case <-s.quit: return } - log.Debug("want batch", "stream", msg.Stream, "Key", msg.Key, "from", msg.From, "to", msg.To) - p.SendPriority(msg, s.priority) + log.Trace("sending want batch", "peer", p.ID(), "stream", msg.Stream, "Key", msg.Key, "from", msg.From, "to", msg.To) + err := p.SendPriority(msg, s.priority) + if err != nil { + p.Drop(err) + } }() return nil } @@ -185,10 +189,9 @@ func (m WantedHashesMsg) String() string { // * sends the next batch of unsynced keys // * sends the actual data chunks as per WantedHashesMsg func (p *Peer) handleWantedHashesMsg(req *WantedHashesMsg) error { - log.Debug("received wanted batch", "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To) + log.Trace("received wanted batch", "peer", p.ID(), "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To) s, err := p.getServer(req.Stream + keyToString(req.Key)) if err != nil { - log.Debug(err.Error()) return err } hashes := s.currentBatch diff --git a/swarm/network/stream/peer.go b/swarm/network/stream/peer.go index 5d2461a3c6..708a3fbcc7 100644 --- a/swarm/network/stream/peer.go +++ b/swarm/network/stream/peer.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "sync" + "time" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/protocols" @@ -27,16 +28,18 @@ import ( "github.com/ethereum/go-ethereum/swarm/storage" ) +var sendTimeout = 5 * time.Second + // Peer is the Peer extention for the streaming protocol type Peer struct { *protocols.Peer - streamer *Registry - pq *pq.PriorityQueue - outgoingMu sync.RWMutex - incomingMu sync.RWMutex - servers map[string]*server - clients map[string]*client - quit chan struct{} + streamer *Registry + pq *pq.PriorityQueue + serverMu sync.RWMutex + clientMu sync.RWMutex + servers map[string]*server + clients map[string]*client + quit chan struct{} } // NewPeer is the constructor for Peer @@ -64,12 +67,14 @@ func (p *Peer) Deliver(chunk *storage.Chunk, priority uint8) error { Key: chunk.Key, SData: chunk.SData, } - return p.pq.Push(nil, msg, int(priority)) + return p.SendPriority(msg, priority) } -// Deliver sends a storeRequestMsg protocol message to the peer +// SendPriority sends message to the peer using the outgoing priority queue func (p *Peer) SendPriority(msg interface{}, priority uint8) error { - return p.pq.Push(nil, msg, int(priority)) + ctx, cancel := context.WithTimeout(context.Background(), sendTimeout) + defer cancel() + return p.pq.Push(ctx, msg, int(priority)) } // SendOfferedHashes sends OfferedHashesMsg protocol msg @@ -92,13 +97,13 @@ func (p *Peer) SendOfferedHashes(s *server, f, t uint64) error { Stream: s.stream, Key: s.key, } - log.Debug("Swarm syncer offer batch", "stream", s.stream, "key", s.key, "len", len(hashes), "from", from, "to", to) + log.Warn("Swarm syncer offer batch", "peer", p.ID(), "stream", s.stream, "key", s.key, "len", len(hashes), "from", from, "to", to) return p.SendPriority(msg, s.priority) } func (p *Peer) getServer(s string) (*server, error) { - p.outgoingMu.RLock() - defer p.outgoingMu.RUnlock() + p.serverMu.RLock() + defer p.serverMu.RUnlock() server := p.servers[s] if server == nil { @@ -108,8 +113,8 @@ func (p *Peer) getServer(s string) (*server, error) { } func (p *Peer) getClient(s string) (*client, error) { - p.incomingMu.RLock() - defer p.incomingMu.RUnlock() + p.clientMu.RLock() + defer p.clientMu.RUnlock() client := p.clients[s] if client == nil { @@ -119,8 +124,8 @@ func (p *Peer) getClient(s string) (*client, error) { } func (p *Peer) setServer(s string, key []byte, o Server, priority uint8) (*server, error) { - p.outgoingMu.Lock() - defer p.outgoingMu.Unlock() + p.serverMu.Lock() + defer p.serverMu.Unlock() sk := s + keyToString(key) if p.servers[sk] != nil { @@ -137,14 +142,14 @@ func (p *Peer) setServer(s string, key []byte, o Server, priority uint8) (*serve } func (p *Peer) setClient(s string, key []byte, i Client, priority uint8, live bool) error { - p.incomingMu.Lock() - defer p.incomingMu.Unlock() + p.clientMu.Lock() + defer p.clientMu.Unlock() sk := s + keyToString(key) if p.clients[sk] != nil { return fmt.Errorf("client %v already registered", sk) } - next := make(chan struct{}, 1) + next := make(chan error, 1) // var intervals *Intervals // if !live { // key := s + p.ID().String() @@ -159,6 +164,6 @@ func (p *Peer) setClient(s string, key []byte, i Client, priority uint8, live bo stream: s, key: key, } - next <- struct{}{} // this is to allow wantedKeysMsg before first batch arrives + next <- nil // this is to allow wantedKeysMsg before first batch arrives return nil } diff --git a/swarm/network/stream/stream.go b/swarm/network/stream/stream.go index 8e3e3ea223..87a56483c6 100644 --- a/swarm/network/stream/stream.go +++ b/swarm/network/stream/stream.go @@ -38,8 +38,8 @@ const ( Mid High Top - PriorityQueue // number of queues - PriorityQueueCap = 3 // queue capacity + PriorityQueue // number of queues + PriorityQueueCap = 32 // queue capacity HashSize = 32 ) @@ -47,6 +47,7 @@ const ( type Registry struct { api *API addr *network.BzzAddr + skipCheck bool clientMu sync.RWMutex serverMu sync.RWMutex peersMu sync.RWMutex @@ -58,9 +59,10 @@ type Registry struct { } // NewRegistry is Streamer constructor -func NewRegistry(addr *network.BzzAddr, delivery *Delivery, store storage.ChunkStore) *Registry { +func NewRegistry(addr *network.BzzAddr, delivery *Delivery, store storage.ChunkStore, skipCheck bool) *Registry { streamer := &Registry{ addr: addr, + skipCheck: skipCheck, store: store, serverFuncs: make(map[string]func(*Peer, []byte) (Server, error)), clientFuncs: make(map[string]func(*Peer, []byte) (Client, error)), @@ -154,7 +156,7 @@ func (r *Registry) Subscribe(peerId discover.NodeID, s string, t []byte, from, t } func (r *Registry) Retrieve(chunk *storage.Chunk) error { - return r.delivery.RequestFromPeers(chunk.Key[:], false) + return r.delivery.RequestFromPeers(chunk.Key[:], r.skipCheck) } func (r *Registry) NodeInfo() interface{} { @@ -265,7 +267,7 @@ type client struct { stream string key []byte quit chan struct{} - next chan struct{} + next chan error } // Client interface for incoming peer Streamer @@ -305,6 +307,17 @@ func (c *client) nextBatch(from uint64) (nextFrom uint64, nextTo uint64) { return nextFrom, nextTo } +func (c *client) batchDone(p *Peer, req *OfferedHashesMsg, hashes []byte) error { + if tf := c.BatchDone(req.Stream, req.From, hashes, req.Root); tf != nil { + tp, err := tf() + if err != nil { + return err + } + return p.SendPriority(tp, c.priority) + } + return nil +} + // Spec is the spec of the streamer protocol var Spec = &protocols.Spec{ Name: "stream", diff --git a/swarm/network/stream/syncer.go b/swarm/network/stream/syncer.go index 9523e4e440..99436e7ecf 100644 --- a/swarm/network/stream/syncer.go +++ b/swarm/network/stream/syncer.go @@ -29,8 +29,8 @@ import ( ) const ( - BatchSize = 2 - // BatchSize = 128 + // BatchSize = 2 + BatchSize = 128 ) // SwarmSyncerServer implements an OutgoingStreamer for history syncing on bins @@ -171,6 +171,8 @@ func NewSwarmSyncerClient(_ *Peer, db *storage.DBAPI, chunker storage.Chunker) ( // } // } +// RegisterSwarmSyncerClient registers the client constructor function for +// to handle incoming sync streams func RegisterSwarmSyncerClient(streamer *Registry, db *storage.DBAPI) { streamer.RegisterClientFunc("SYNC", func(p *Peer, t []byte) (Client, error) { return NewSwarmSyncerClient(p, db, nil) @@ -180,12 +182,16 @@ func RegisterSwarmSyncerClient(streamer *Registry, db *storage.DBAPI) { // NeedData func (s *SwarmSyncerClient) NeedData(key []byte) (wait func()) { chunk, _ := s.db.GetOrCreateRequest(key) + log.Warn("created request", "key", chunk.Key) // TODO: we may want to request from this peer anyway even if the request exists if chunk.ReqC == nil { return nil } // create request and wait until the chunk data arrives and is stored - return chunk.WaitToStore + return func() { + chunk.WaitToStore() + log.Warn("stored", "key", chunk.Key) + } } // BatchDone diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index 9dde5bda72..11de0bf0c7 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -28,19 +28,27 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/simulations" + "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm/network" streamTesting "github.com/ethereum/go-ethereum/swarm/network/stream/testing" "github.com/ethereum/go-ethereum/swarm/storage" ) +const dataChunkCount = 500 + func TestSyncerSimulation(t *testing.T) { - testSyncBetweenNodes(t, 2, 1, 81000, true, 1) - testSyncBetweenNodes(t, 2, 1, 81000, false, 1) - testSyncBetweenNodes(t, 3, 1, 81000, true, 1) - testSyncBetweenNodes(t, 3, 1, 81000, false, 1) + testSyncBetweenNodes(t, 2, 1, dataChunkCount, true, 1) + // testSyncBetweenNodes(t, 2, 1, dataChunkCount, false, 1) + testSyncBetweenNodes(t, 4, 1, dataChunkCount, true, 1) + // testSyncBetweenNodes(t, 4, 1, dataChunkCount, false, 1) + testSyncBetweenNodes(t, 8, 1, dataChunkCount, true, 1) + // testSyncBetweenNodes(t, 8, 1, dataChunkCount, false, 1) + testSyncBetweenNodes(t, 16, 1, dataChunkCount, true, 1) + // testSyncBetweenNodes(t, 16, 1, dataChunkCount, false, 1) } -func testSyncBetweenNodes(t *testing.T, nodes, conns, size int, skipCheck bool, po uint8) { +func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck bool, po uint8) { + defaultSkipCheck = skipCheck toAddr = func(id discover.NodeID) *network.BzzAddr { addr := network.NewAddrFromNodeID(id) addr.OAddr[0] = byte(0) @@ -64,30 +72,43 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, size int, skipCheck bool, for i, id := range sim.IDs { stores[id] = sim.Stores[i] } - + peerCount = func(id discover.NodeID) int { + if sim.IDs[0] == id || sim.IDs[nodes-1] == id { + return 1 + } + return 2 + } // here we distribute chunks of a random file into Stores of nodes 1 to nodes rrdpa := storage.NewDPA(newRoundRobinStore(sim.Stores[1:]...), storage.NewChunkerParams()) rrdpa.Start() + size := chunkCount * chunkSize _, wait, err := rrdpa.Store(io.LimitReader(crand.Reader, int64(size)), int64(size)) + // need to wait cos we then immediately collect the relevant bin content + wait() defer rrdpa.Stop() if err != nil { t.Fatal(err.Error()) } - // wait until all chunks stored - // TODO: is wait() necessary? - wait() - // each node Subscribes to each other's swarmChunkServerStreamName - // need to wait till an aynchronous process registers the peers in streamer.peers - // that is used by Subscribe - // time.Sleep(1 * time.Second) - // err := streamer.Subscribe(p.ID(), swarmChunkServerStreamName, nil, 0, 0, Top, true) - if err != nil { - t.Fatal(err.Error()) - } - waitPeerErrC = make(chan error) - // create a retriever dpa for the pivot node - action := func(context.Context) error { + // collect hashes in po 1 from all nodes + var hashes []storage.Key + dbs := make([]*storage.DBAPI, nodes) + for i := 0; i < nodes; i++ { + dbs[i] = storage.NewDBAPI(sim.Stores[i].(*storage.LocalStore)) + } + for i := 1; i < nodes; i++ { + dbs[i].Iterator(0, math.MaxUint64, po, func(key storage.Key, index uint64) bool { + hashes = append(hashes, key) + return true + }) + } + + waitPeerErrC = make(chan error) + action := func(ctx context.Context) error { + // need to wait till an aynchronous process registers the peers in streamer.peers + // that is used by Subscribe + // the global peerCount function tells how many connections each node has + // TODO: this is to be reimplemented with peerEvent watcher without global var i := 0 for err := range waitPeerErrC { if err != nil { @@ -98,71 +119,42 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, size int, skipCheck bool, break } } - - for i := 0; i < len(sim.IDs)-1; i++ { - id := sim.IDs[i] - // if err := streamer.Subscribe(p.ID(), "SYNC", []byte{uint8(1)}, 0, 0, Top, false); err != nil { - // log.Warn("error in subscribe", "err", err) - // } - node := sim.Net.GetNode(id) - if node == nil { - return fmt.Errorf("unknown node: %s", id) - } - client, err := node.Client() - if err != nil { - return fmt.Errorf("error getting node client: %s", err) - } - sid := sim.IDs[i+1] - if err := client.Call(nil, "stream_subscribeStream", sid, "SYNC", []byte{uint8(1)}, 0, 0, Top, false); err != nil { - return fmt.Errorf("error subscribing: %s", err) - } - } - return nil - } - - dbs := make([]*storage.DBAPI, nodes) - for i := 0; i < nodes; i++ { - dbs[i] = storage.NewDBAPI(sim.Stores[i].(*storage.LocalStore)) + // each node Subscribes to each other's swarmChunkServerStreamName + j := 0 + return sim.CallClient(func(client *rpc.Client) error { + ctx, cancel := context.WithTimeout(ctx, 1*time.Second) + defer cancel() + j++ + return client.CallContext(ctx, nil, "stream_subscribeStream", sim.IDs[j], "SYNC", []byte{1}, 0, 0, Top, false) + }, sim.IDs[0:nodes-1]...) } + // this makes sure check is not called before the previous call finishes + checkC := make(chan struct{}) check := func(ctx context.Context, id discover.NodeID) (bool, error) { - if id != sim.Net.Nodes[0].ID() { - return true, nil - } + defer func() { checkC <- struct{}{} }() + select { case <-ctx.Done(): return false, ctx.Err() default: } - var found, total int - for i := 1; i < nodes; i++ { - dbs[i].Iterator(0, math.MaxUint64, po, func(key storage.Key, index uint64) bool { - _, err := dbs[0].Get(key) - if err == nil { - found++ - } - total++ - return true - }) + var found int + total := len(hashes) + for _, key := range hashes { + _, err := dbs[0].Get(key) + if err == nil { + found++ + } } log.Debug("sync check", "bin", po, "found", found, "total", total) return found == total, nil } - trigger := make(chan discover.NodeID) - ticker := time.NewTicker(500 * time.Millisecond) - go func() { - defer ticker.Stop() - // we are only testing the pivot node (net.Nodes[0]) - for range ticker.C { - trigger <- sim.Net.Nodes[0].ID() - } - }() - conf.Step = &simulations.Step{ Action: action, - Trigger: trigger, + Trigger: streamTesting.PivotTrigger(10*time.Millisecond, checkC, sim.IDs[0]), Expect: &simulations.Expectation{ Nodes: sim.IDs[0:1], Check: check, diff --git a/swarm/network/stream/testing/testing.go b/swarm/network/stream/testing/testing.go index b427efd532..e92b009f86 100644 --- a/swarm/network/stream/testing/testing.go +++ b/swarm/network/stream/testing/testing.go @@ -28,9 +28,11 @@ import ( "time" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/simulations" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/storage" ) @@ -61,7 +63,8 @@ func SetStores(addrs ...network.Addr) ([]storage.ChunkStore, func(), error) { stores[i] = store } teardown := func() { - for _, datadir := range datadirs { + for i, datadir := range datadirs { + stores[i].Close() os.RemoveAll(datadir) } } @@ -94,20 +97,22 @@ func NewAdapter(adapterType string, services adapters.Services) (adapter adapter } func CheckResult(t *testing.T, result *simulations.StepResult, startedAt, finishedAt time.Time) { - t.Logf("Simulation with %d nodes passed in %s", len(result.Passes), result.FinishedAt.Sub(result.StartedAt)) - var min, max time.Duration - var sum int - for _, pass := range result.Passes { - duration := pass.Sub(result.StartedAt) - if sum == 0 || duration < min { - min = duration + t.Logf("Simulation passed in %s", result.FinishedAt.Sub(result.StartedAt)) + if len(result.Passes) > 1 { + var min, max time.Duration + var sum int + for _, pass := range result.Passes { + duration := pass.Sub(result.StartedAt) + if sum == 0 || duration < min { + min = duration + } + if duration > max { + max = duration + } + sum += int(duration.Nanoseconds()) } - if duration > max { - max = duration - } - sum += int(duration.Nanoseconds()) + t.Logf("Min: %s, Max: %s, Average: %s", min, max, time.Duration(sum/len(result.Passes))*time.Nanosecond) } - t.Logf("Min: %s, Max: %s, Average: %s", min, max, time.Duration(sum/len(result.Passes))*time.Nanosecond) t.Logf("Setup: %s, Shutdown: %s", result.StartedAt.Sub(startedAt), finishedAt.Sub(result.FinishedAt)) } @@ -195,8 +200,7 @@ func (s *Simulation) Run(conf *RunConfig) (*simulations.StepResult, error) { } } wg.Wait() - - log.Debug(fmt.Sprintf("nodes: %v", len(s.Addrs))) + log.Info(fmt.Sprintf("simulation with %v nodes", len(s.Addrs))) // create an only locally retrieving dpa for the pivot node to test // if retriee requests have arrived @@ -206,3 +210,59 @@ func (s *Simulation) Run(conf *RunConfig) (*simulations.StepResult, error) { result := simulations.NewSimulation(s.Net).Run(ctx, conf.Step) return result, nil } + +func WatchDisconnections(id discover.NodeID, client *rpc.Client, errc chan error, quitC chan struct{}) error { + events := make(chan *p2p.PeerEvent) + sub, err := client.Subscribe(context.Background(), "admin", events, "peerEvents") + if err != nil { + return fmt.Errorf("error getting peer events for node %v: %s", id, err) + } + go func() { + defer sub.Unsubscribe() + select { + case <-quitC: + return + case e := <-events: + errc <- fmt.Errorf("peerEvent for node %v: %v", id, e) + case err := <-sub.Err(): + if err != nil { + errc <- fmt.Errorf("error getting peer events for node %v: %v", id, err) + } + } + }() + return nil +} + +func PivotTrigger(d time.Duration, checkC chan struct{}, ids ...discover.NodeID) chan discover.NodeID { + trigger := make(chan discover.NodeID) + go func() { + ticker := time.NewTicker(d) + defer ticker.Stop() + // we are only testing the pivot node (net.Nodes[0]) + for range ticker.C { + for _, id := range ids { + trigger <- id + } + <-checkC + } + }() + return trigger +} + +func (sim *Simulation) CallClient(f func(*rpc.Client) error, ids ...discover.NodeID) error { + for _, id := range ids { + node := sim.Net.GetNode(id) + if node == nil { + return fmt.Errorf("unknown node: %s", id) + } + client, err := node.Client() + if err != nil { + return fmt.Errorf("error getting node client: %s", err) + } + err = f(client) + if err != nil { + return err + } + } + return nil +} diff --git a/swarm/storage/dbstore.go b/swarm/storage/dbstore.go index bed07bdd9b..95e5f286cb 100644 --- a/swarm/storage/dbstore.go +++ b/swarm/storage/dbstore.go @@ -599,8 +599,9 @@ func (s *DbStore) writeBatches() { s.batchC = make(chan bool) s.batch = new(leveldb.Batch) s.lock.Unlock() - log.Trace(fmt.Sprintf("DbStore: spawn batch write (%d chunks) ", b.Len())) - s.writeBatch(b, e, d, a) + err := s.writeBatch(b, e, d, a) + // TODO: set this error on the batch, then tell the chunk + log.Trace(fmt.Sprintf("DbStore: spawn batch write (%d chunks): %v", b.Len(), err)) close(c) if e >= s.capacity { log.Trace(fmt.Sprintf("DbStore: collecting garbage...(%d chunks)", e)) @@ -611,15 +612,16 @@ func (s *DbStore) writeBatches() { } // must be called non concurrently -func (s *DbStore) writeBatch(b *leveldb.Batch, entryCnt, dataIdx, accessCnt uint64) { +func (s *DbStore) writeBatch(b *leveldb.Batch, entryCnt, dataIdx, accessCnt uint64) error { b.Put(keyEntryCnt, U64ToBytes(entryCnt)) b.Put(keyDataIdx, U64ToBytes(dataIdx)) b.Put(keyAccessCnt, U64ToBytes(accessCnt)) l := s.batch.Len() if err := s.db.Write(b); err != nil { - log.Error(fmt.Sprintf("unable to write batch: %v", err)) + return fmt.Errorf("unable to write batch: %v", err) } log.Trace(fmt.Sprintf("DbStore: batch write (%d chunks) complete", l)) + return nil } // newMockEncodeDataFunc returns a function that stores the chunk data diff --git a/swarm/storage/memstore.go b/swarm/storage/memstore.go index e8e393baa9..7eb0aa06c2 100644 --- a/swarm/storage/memstore.go +++ b/swarm/storage/memstore.go @@ -240,7 +240,7 @@ func (s *MemStore) Get(hash Key) (chunk *Chunk, err error) { func (s *MemStore) removeOldest() { node := s.memtree - + log.Warn("purge memstore") for node.entry == nil { aidx := uint(0) @@ -284,9 +284,11 @@ func (s *MemStore) removeOldest() { <-node.entry.dbStored log.Trace(fmt.Sprintf("Memstore Clean: Chunk %v saved to DBStore. Ready to clear from mem.", node.entry.Key.Log())) - if node.entry.SData != nil { + if node.entry.ReqC == nil { node.entry = nil s.entryCnt-- + } else { + return } node.access[0] = 0 From 84be00915416872b6136b06a6f7b12b095585e36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 22 Jan 2018 14:07:47 +0200 Subject: [PATCH 071/210] core: sorted reorg insertion order for proper head header updating --- core/blockchain.go | 13 ++++++----- core/blockchain_test.go | 48 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 737fbe3eee..f886ffe4ed 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -465,7 +465,7 @@ func (bc *BlockChain) insert(block *types.Block) { } bc.currentBlock = block - // If the block is better than out head or is on a different chain, force update heads + // If the block is better than our head or is on a different chain, force update heads if updateHeads { bc.hc.SetCurrentHeader(block.Header()) @@ -1140,18 +1140,17 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { } else { log.Error("Impossible reorg, please file an issue", "oldnum", oldBlock.Number(), "oldhash", oldBlock.Hash(), "newnum", newBlock.Number(), "newhash", newBlock.Hash()) } + // Insert the new chain, taking care of the proper incremental order var addedTxs types.Transactions - // insert blocks. Order does not matter. Last block will be written in ImportChain itself which creates the new head properly - for _, block := range newChain { + for i := len(newChain) - 1; i >= 0; i-- { // insert the block in the canonical way, re-writing history - bc.insert(block) + bc.insert(newChain[i]) // write lookup entries for hash based transaction/receipt searches - if err := WriteTxLookupEntries(bc.chainDb, block); err != nil { + if err := WriteTxLookupEntries(bc.chainDb, newChain[i]); err != nil { return err } - addedTxs = append(addedTxs, block.Transactions()...) + addedTxs = append(addedTxs, newChain[i].Transactions()...) } - // calculate the difference between deleted and added transactions diff := types.TxDifference(deletedTxs, addedTxs) // When transactions get deleted from the database that means the diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 26c816027f..cbde3bcd2d 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -1197,3 +1197,51 @@ func TestEIP161AccountRemoval(t *testing.T) { t.Error("account should not exist") } } + +// This is a regression test (i.e. as weird as it is, don't delete it ever), which +// tests that under weird reorg conditions the blockchain and its internal header- +// chain return the same latest block/header. +// +// https://github.com/ethereum/go-ethereum/pull/15941 +func TestBlockchainHeaderchainReorgConsistency(t *testing.T) { + // Generate a canonical chain to act as the main dataset + engine := ethash.NewFaker() + + db, _ := ethdb.NewMemDatabase() + genesis := new(Genesis).MustCommit(db) + blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) + + // Generate a bunch of fork blocks, each side forking from the canonical chain + forks := make([]*types.Block, len(blocks)) + for i := 0; i < len(forks); i++ { + parent := genesis + if i > 0 { + parent = blocks[i-1] + } + fork, _ := GenerateChain(params.TestChainConfig, parent, engine, db, 1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) }) + forks[i] = fork[0] + } + // Import the canonical and fork chain side by side, verifying the current block + // and current header consistency + diskdb, _ := ethdb.NewMemDatabase() + new(Genesis).MustCommit(diskdb) + + chain, err := NewBlockChain(diskdb, params.TestChainConfig, engine, vm.Config{}) + if err != nil { + t.Fatalf("failed to create tester chain: %v", err) + } + for i := 0; i < len(blocks); i++ { + if _, err := chain.InsertChain(blocks[i : i+1]); err != nil { + t.Fatalf("block %d: failed to insert into chain: %v", i, err) + } + if chain.CurrentBlock().Hash() != chain.CurrentHeader().Hash() { + t.Errorf("block %d: current block/header mismatch: block #%d [%x…], header #%d [%x…]", i, chain.CurrentBlock().Number(), chain.CurrentBlock().Hash().Bytes()[:4], chain.CurrentHeader().Number, chain.CurrentHeader().Hash().Bytes()[:4]) + } + if _, err := chain.InsertChain(forks[i : i+1]); err != nil { + t.Fatalf(" fork %d: failed to insert into chain: %v", i, err) + } + if chain.CurrentBlock().Hash() != chain.CurrentHeader().Hash() { + t.Errorf(" fork %d: current block/header mismatch: block #%d [%x…], header #%d [%x…]", i, chain.CurrentBlock().Number(), chain.CurrentBlock().Hash().Bytes()[:4], chain.CurrentHeader().Number, chain.CurrentHeader().Hash().Bytes()[:4]) + } + } +} From 92580d69d3156e5d2f0788eb5d48664cc3fb0ef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Mon, 22 Jan 2018 13:38:34 +0100 Subject: [PATCH 072/210] p2p, p2p/discover, p2p/discv5: implement UDP port sharing (#15200) This commit affects p2p/discv5 "topic discovery" by running it on the same UDP port where the old discovery works. This is realized by giving an "unhandled" packet channel to the old v4 discovery packet handler where all invalid packets are sent. These packets are then processed by v5. v5 packets are always invalid when interpreted by v4 and vice versa. This is ensured by adding one to the first byte of the packet hash in v5 packets. DiscoveryV5Bootnodes is also changed to point to new bootnodes that are implementing the changed packet format with modified hash. Existing and new v5 bootnodes are both running on different ports ATM. --- cmd/bootnode/main.go | 25 ++++++++++++-- cmd/faucet/faucet.go | 1 - cmd/utils/flags.go | 10 ------ les/backend.go | 5 ++- les/protocol.go | 5 +-- les/server.go | 4 +-- mobile/geth.go | 1 - node/defaults.go | 7 ++-- p2p/discover/udp.go | 44 +++++++++++------------ p2p/discover/udp_test.go | 3 +- p2p/discv5/net.go | 3 +- p2p/discv5/net_test.go | 2 +- p2p/discv5/sim_test.go | 2 +- p2p/discv5/ticket.go | 2 +- p2p/discv5/udp.go | 46 ++++++++++-------------- p2p/server.go | 75 +++++++++++++++++++++++++++++++++++++--- params/bootnodes.go | 6 ++-- 17 files changed, 150 insertions(+), 91 deletions(-) diff --git a/cmd/bootnode/main.go b/cmd/bootnode/main.go index e1734d89ac..ecfc6fc24e 100644 --- a/cmd/bootnode/main.go +++ b/cmd/bootnode/main.go @@ -21,6 +21,7 @@ import ( "crypto/ecdsa" "flag" "fmt" + "net" "os" "github.com/ethereum/go-ethereum/cmd/utils" @@ -96,12 +97,32 @@ func main() { } } + addr, err := net.ResolveUDPAddr("udp", *listenAddr) + if err != nil { + utils.Fatalf("-ResolveUDPAddr: %v", err) + } + conn, err := net.ListenUDP("udp", addr) + if err != nil { + utils.Fatalf("-ListenUDP: %v", err) + } + + realaddr := conn.LocalAddr().(*net.UDPAddr) + if natm != nil { + if !realaddr.IP.IsLoopback() { + go nat.Map(natm, nil, "udp", realaddr.Port, realaddr.Port, "ethereum discovery") + } + // TODO: react to external IP changes over time. + if ext, err := natm.ExternalIP(); err == nil { + realaddr = &net.UDPAddr{IP: ext, Port: realaddr.Port} + } + } + if *runv5 { - if _, err := discv5.ListenUDP(nodeKey, *listenAddr, natm, "", restrictList); err != nil { + if _, err := discv5.ListenUDP(nodeKey, conn, realaddr, "", restrictList); err != nil { utils.Fatalf("%v", err) } } else { - if _, err := discover.ListenUDP(nodeKey, *listenAddr, natm, "", restrictList); err != nil { + if _, err := discover.ListenUDP(nodeKey, conn, realaddr, nil, "", restrictList); err != nil { utils.Fatalf("%v", err) } } diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index e92924fc9c..99527f9d1e 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -223,7 +223,6 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network u NoDiscovery: true, DiscoveryV5: true, ListenAddr: fmt.Sprintf(":%d", port), - DiscoveryV5Addr: fmt.Sprintf(":%d", port+1), MaxPeers: 25, BootstrapNodesV5: enodes, }, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 3766ea4a60..89dcd230c4 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -636,14 +636,6 @@ func setListenAddress(ctx *cli.Context, cfg *p2p.Config) { } } -// setDiscoveryV5Address creates a UDP listening address string from set command -// line flags for the V5 discovery protocol. -func setDiscoveryV5Address(ctx *cli.Context, cfg *p2p.Config) { - if ctx.GlobalIsSet(ListenPortFlag.Name) { - cfg.DiscoveryV5Addr = fmt.Sprintf(":%d", ctx.GlobalInt(ListenPortFlag.Name)+1) - } -} - // setNAT creates a port mapper from command line flags. func setNAT(ctx *cli.Context, cfg *p2p.Config) { if ctx.GlobalIsSet(NATFlag.Name) { @@ -794,7 +786,6 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) { setNodeKey(ctx, cfg) setNAT(ctx, cfg) setListenAddress(ctx, cfg) - setDiscoveryV5Address(ctx, cfg) setBootstrapNodes(ctx, cfg) setBootstrapNodesV5(ctx, cfg) @@ -830,7 +821,6 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) { // --dev mode can't use p2p networking. cfg.MaxPeers = 0 cfg.ListenAddr = ":0" - cfg.DiscoveryV5Addr = ":0" cfg.NoDiscovery = true cfg.DiscoveryV5 = false } diff --git a/les/backend.go b/les/backend.go index 7180b81d76..798e44e85c 100644 --- a/les/backend.go +++ b/les/backend.go @@ -221,9 +221,8 @@ func (s *LightEthereum) Start(srvr *p2p.Server) error { s.startBloomHandlers() log.Warn("Light client mode is an experimental feature") s.netRPCService = ethapi.NewPublicNetAPI(srvr, s.networkId) - // search the topic belonging to the oldest supported protocol because - // servers always advertise all supported protocols - protocolVersion := ClientProtocolVersions[len(ClientProtocolVersions)-1] + // clients are searching for the first advertised protocol in the list + protocolVersion := AdvertiseProtocolVersions[0] s.serverPool.start(srvr, lesTopic(s.blockchain.Genesis().Hash(), protocolVersion)) s.protocolManager.Start() return nil diff --git a/les/protocol.go b/les/protocol.go index 05e6654d6e..6a7354d1c2 100644 --- a/les/protocol.go +++ b/les/protocol.go @@ -41,8 +41,9 @@ const ( // Supported versions of the les protocol (first is primary) var ( - ClientProtocolVersions = []uint{lpv2, lpv1} - ServerProtocolVersions = []uint{lpv2, lpv1} + ClientProtocolVersions = []uint{lpv2, lpv1} + ServerProtocolVersions = []uint{lpv2, lpv1} + AdvertiseProtocolVersions = []uint{lpv2} // clients are searching for the first advertised protocol in the list ) // Number of implemented message corresponding to different protocol versions. diff --git a/les/server.go b/les/server.go index d8f93cd87b..ec2e44fecc 100644 --- a/les/server.go +++ b/les/server.go @@ -56,8 +56,8 @@ func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { return nil, err } - lesTopics := make([]discv5.Topic, len(ServerProtocolVersions)) - for i, pv := range ServerProtocolVersions { + lesTopics := make([]discv5.Topic, len(AdvertiseProtocolVersions)) + for i, pv := range AdvertiseProtocolVersions { lesTopics[i] = lesTopic(eth.BlockChain().Genesis().Hash(), pv) } diff --git a/mobile/geth.go b/mobile/geth.go index 7b39faadec..7e3b8f4915 100644 --- a/mobile/geth.go +++ b/mobile/geth.go @@ -116,7 +116,6 @@ func NewNode(datadir string, config *NodeConfig) (stack *Node, _ error) { P2P: p2p.Config{ NoDiscovery: true, DiscoveryV5: true, - DiscoveryV5Addr: ":0", BootstrapNodesV5: config.BootstrapNodes.nodes, ListenAddr: ":0", NAT: nat.Any(), diff --git a/node/defaults.go b/node/defaults.go index 848f08e05c..d4e1486834 100644 --- a/node/defaults.go +++ b/node/defaults.go @@ -41,10 +41,9 @@ var DefaultConfig = Config{ WSPort: DefaultWSPort, WSModules: []string{"net", "web3"}, P2P: p2p.Config{ - ListenAddr: ":30303", - DiscoveryV5Addr: ":30304", - MaxPeers: 25, - NAT: nat.Any(), + ListenAddr: ":30303", + MaxPeers: 25, + NAT: nat.Any(), }, } diff --git a/p2p/discover/udp.go b/p2p/discover/udp.go index f9eb99ee36..60436952d8 100644 --- a/p2p/discover/udp.go +++ b/p2p/discover/udp.go @@ -210,17 +210,15 @@ type reply struct { matched chan<- bool } +// ReadPacket is sent to the unhandled channel when it could not be processed +type ReadPacket struct { + Data []byte + Addr *net.UDPAddr +} + // ListenUDP returns a new table that listens for UDP packets on laddr. -func ListenUDP(priv *ecdsa.PrivateKey, laddr string, natm nat.Interface, nodeDBPath string, netrestrict *netutil.Netlist) (*Table, error) { - addr, err := net.ResolveUDPAddr("udp", laddr) - if err != nil { - return nil, err - } - conn, err := net.ListenUDP("udp", addr) - if err != nil { - return nil, err - } - tab, _, err := newUDP(priv, conn, natm, nodeDBPath, netrestrict) +func ListenUDP(priv *ecdsa.PrivateKey, conn conn, realaddr *net.UDPAddr, unhandled chan ReadPacket, nodeDBPath string, netrestrict *netutil.Netlist) (*Table, error) { + tab, _, err := newUDP(priv, conn, realaddr, unhandled, nodeDBPath, netrestrict) if err != nil { return nil, err } @@ -228,7 +226,7 @@ func ListenUDP(priv *ecdsa.PrivateKey, laddr string, natm nat.Interface, nodeDBP return tab, nil } -func newUDP(priv *ecdsa.PrivateKey, c conn, natm nat.Interface, nodeDBPath string, netrestrict *netutil.Netlist) (*Table, *udp, error) { +func newUDP(priv *ecdsa.PrivateKey, c conn, realaddr *net.UDPAddr, unhandled chan ReadPacket, nodeDBPath string, netrestrict *netutil.Netlist) (*Table, *udp, error) { udp := &udp{ conn: c, priv: priv, @@ -237,16 +235,6 @@ func newUDP(priv *ecdsa.PrivateKey, c conn, natm nat.Interface, nodeDBPath strin gotreply: make(chan reply), addpending: make(chan *pending), } - realaddr := c.LocalAddr().(*net.UDPAddr) - if natm != nil { - if !realaddr.IP.IsLoopback() { - go nat.Map(natm, udp.closing, "udp", realaddr.Port, realaddr.Port, "ethereum discovery") - } - // TODO: react to external IP changes over time. - if ext, err := natm.ExternalIP(); err == nil { - realaddr = &net.UDPAddr{IP: ext, Port: realaddr.Port} - } - } // TODO: separate TCP port udp.ourEndpoint = makeEndpoint(realaddr, uint16(realaddr.Port)) tab, err := newTable(udp, PubkeyID(&priv.PublicKey), realaddr, nodeDBPath) @@ -256,7 +244,7 @@ func newUDP(priv *ecdsa.PrivateKey, c conn, natm nat.Interface, nodeDBPath strin udp.Table = tab go udp.loop() - go udp.readLoop() + go udp.readLoop(unhandled) return udp.Table, udp, nil } @@ -492,8 +480,11 @@ func encodePacket(priv *ecdsa.PrivateKey, ptype byte, req interface{}) ([]byte, } // readLoop runs in its own goroutine. it handles incoming UDP packets. -func (t *udp) readLoop() { +func (t *udp) readLoop(unhandled chan ReadPacket) { defer t.conn.Close() + if unhandled != nil { + defer close(unhandled) + } // Discovery packets are defined to be no larger than 1280 bytes. // Packets larger than this size will be cut at the end and treated // as invalid because their hash won't match. @@ -509,7 +500,12 @@ func (t *udp) readLoop() { log.Debug("UDP read error", "err", err) return } - t.handlePacket(from, buf[:nbytes]) + if t.handlePacket(from, buf[:nbytes]) != nil && unhandled != nil { + select { + case unhandled <- ReadPacket{buf[:nbytes], from}: + default: + } + } } } diff --git a/p2p/discover/udp_test.go b/p2p/discover/udp_test.go index 21e8b561da..b81caf8392 100644 --- a/p2p/discover/udp_test.go +++ b/p2p/discover/udp_test.go @@ -70,7 +70,8 @@ func newUDPTest(t *testing.T) *udpTest { remotekey: newkey(), remoteaddr: &net.UDPAddr{IP: net.IP{10, 0, 1, 99}, Port: 30303}, } - test.table, test.udp, _ = newUDP(test.localkey, test.pipe, nil, "", nil) + realaddr := test.pipe.LocalAddr().(*net.UDPAddr) + test.table, test.udp, _ = newUDP(test.localkey, test.pipe, realaddr, nil, "", nil) return test } diff --git a/p2p/discv5/net.go b/p2p/discv5/net.go index cd9981584d..f9baf126f1 100644 --- a/p2p/discv5/net.go +++ b/p2p/discv5/net.go @@ -29,7 +29,6 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/p2p/nat" "github.com/ethereum/go-ethereum/p2p/netutil" "github.com/ethereum/go-ethereum/rlp" ) @@ -134,7 +133,7 @@ type timeoutEvent struct { node *Node } -func newNetwork(conn transport, ourPubkey ecdsa.PublicKey, natm nat.Interface, dbPath string, netrestrict *netutil.Netlist) (*Network, error) { +func newNetwork(conn transport, ourPubkey ecdsa.PublicKey, dbPath string, netrestrict *netutil.Netlist) (*Network, error) { ourID := PubkeyID(&ourPubkey) var db *nodeDB diff --git a/p2p/discv5/net_test.go b/p2p/discv5/net_test.go index bd234f5ba6..369282ca9c 100644 --- a/p2p/discv5/net_test.go +++ b/p2p/discv5/net_test.go @@ -28,7 +28,7 @@ import ( func TestNetwork_Lookup(t *testing.T) { key, _ := crypto.GenerateKey() - network, err := newNetwork(lookupTestnet, key.PublicKey, nil, "", nil) + network, err := newNetwork(lookupTestnet, key.PublicKey, "", nil) if err != nil { t.Fatal(err) } diff --git a/p2p/discv5/sim_test.go b/p2p/discv5/sim_test.go index bf57872e2d..543faecd48 100644 --- a/p2p/discv5/sim_test.go +++ b/p2p/discv5/sim_test.go @@ -282,7 +282,7 @@ func (s *simulation) launchNode(log bool) *Network { addr := &net.UDPAddr{IP: ip, Port: 30303} transport := &simTransport{joinTime: time.Now(), sender: id, senderAddr: addr, sim: s, priv: key} - net, err := newNetwork(transport, key.PublicKey, nil, "", nil) + net, err := newNetwork(transport, key.PublicKey, "", nil) if err != nil { panic("cannot launch new node: " + err.Error()) } diff --git a/p2p/discv5/ticket.go b/p2p/discv5/ticket.go index b45ec4d2be..023c5000d2 100644 --- a/p2p/discv5/ticket.go +++ b/p2p/discv5/ticket.go @@ -642,7 +642,7 @@ func (s *ticketStore) gotTopicNodes(from *Node, hash common.Hash, nodes []rpcNod if ip.IsUnspecified() || ip.IsLoopback() { ip = from.IP } - n := NewNode(node.ID, ip, node.UDP-1, node.TCP-1) // subtract one from port while discv5 is running in test mode on UDPport+1 + n := NewNode(node.ID, ip, node.UDP, node.TCP) select { case chn <- n: default: diff --git a/p2p/discv5/udp.go b/p2p/discv5/udp.go index 26087cd8e5..e921520768 100644 --- a/p2p/discv5/udp.go +++ b/p2p/discv5/udp.go @@ -37,7 +37,7 @@ const Version = 4 // Errors var ( errPacketTooSmall = errors.New("too small") - errBadHash = errors.New("bad hash") + errBadPrefix = errors.New("bad prefix") errExpired = errors.New("expired") errUnsolicitedReply = errors.New("unsolicited reply") errUnknownNode = errors.New("unknown node") @@ -145,10 +145,11 @@ type ( } ) -const ( - macSize = 256 / 8 - sigSize = 520 / 8 - headSize = macSize + sigSize // space of packet frame data +var ( + versionPrefix = []byte("temporary discovery v5") + versionPrefixSize = len(versionPrefix) + sigSize = 520 / 8 + headSize = versionPrefixSize + sigSize // space of packet frame data ) // Neighbors replies are sent across multiple packets to @@ -237,12 +238,12 @@ type udp struct { } // ListenUDP returns a new table that listens for UDP packets on laddr. -func ListenUDP(priv *ecdsa.PrivateKey, laddr string, natm nat.Interface, nodeDBPath string, netrestrict *netutil.Netlist) (*Network, error) { - transport, err := listenUDP(priv, laddr) +func ListenUDP(priv *ecdsa.PrivateKey, conn conn, realaddr *net.UDPAddr, nodeDBPath string, netrestrict *netutil.Netlist) (*Network, error) { + transport, err := listenUDP(priv, conn, realaddr) if err != nil { return nil, err } - net, err := newNetwork(transport, priv.PublicKey, natm, nodeDBPath, netrestrict) + net, err := newNetwork(transport, priv.PublicKey, nodeDBPath, netrestrict) if err != nil { return nil, err } @@ -251,16 +252,8 @@ func ListenUDP(priv *ecdsa.PrivateKey, laddr string, natm nat.Interface, nodeDBP return net, nil } -func listenUDP(priv *ecdsa.PrivateKey, laddr string) (*udp, error) { - addr, err := net.ResolveUDPAddr("udp", laddr) - if err != nil { - return nil, err - } - conn, err := net.ListenUDP("udp", addr) - if err != nil { - return nil, err - } - return &udp{conn: conn, priv: priv, ourEndpoint: makeEndpoint(addr, uint16(addr.Port))}, nil +func listenUDP(priv *ecdsa.PrivateKey, conn conn, realaddr *net.UDPAddr) (*udp, error) { + return &udp{conn: conn, priv: priv, ourEndpoint: makeEndpoint(realaddr, uint16(realaddr.Port))}, nil } func (t *udp) localAddr() *net.UDPAddr { @@ -372,11 +365,9 @@ func encodePacket(priv *ecdsa.PrivateKey, ptype byte, req interface{}) (p, hash log.Error(fmt.Sprint("could not sign packet:", err)) return nil, nil, err } - copy(packet[macSize:], sig) - // add the hash to the front. Note: this doesn't protect the - // packet in any way. - hash = crypto.Keccak256(packet[macSize:]) - copy(packet, hash) + copy(packet, versionPrefix) + copy(packet[versionPrefixSize:], sig) + hash = crypto.Keccak256(packet[versionPrefixSize:]) return packet, hash, nil } @@ -420,17 +411,16 @@ func decodePacket(buffer []byte, pkt *ingressPacket) error { } buf := make([]byte, len(buffer)) copy(buf, buffer) - hash, sig, sigdata := buf[:macSize], buf[macSize:headSize], buf[headSize:] - shouldhash := crypto.Keccak256(buf[macSize:]) - if !bytes.Equal(hash, shouldhash) { - return errBadHash + prefix, sig, sigdata := buf[:versionPrefixSize], buf[versionPrefixSize:headSize], buf[headSize:] + if !bytes.Equal(prefix, versionPrefix) { + return errBadPrefix } fromID, err := recoverNodeID(crypto.Keccak256(buf[headSize:]), sig) if err != nil { return err } pkt.rawData = buf - pkt.hash = hash + pkt.hash = crypto.Keccak256(buf[versionPrefixSize:]) pkt.remoteID = fromID switch pkt.ev = nodeEvent(sigdata[0]); pkt.ev { case pingPacket: diff --git a/p2p/server.go b/p2p/server.go index 922df55ba5..2cff94ea5b 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -78,9 +78,6 @@ type Config struct { // protocol should be started or not. DiscoveryV5 bool `toml:",omitempty"` - // Listener address for the V5 discovery protocol UDP traffic. - DiscoveryV5Addr string `toml:",omitempty"` - // Name sets the node name of this server. // Use common.MakeName to create a name that follows existing conventions. Name string `toml:"-"` @@ -354,6 +351,32 @@ func (srv *Server) Stop() { srv.loopWG.Wait() } +// sharedUDPConn implements a shared connection. Write sends messages to the underlying connection while read returns +// messages that were found unprocessable and sent to the unhandled channel by the primary listener. +type sharedUDPConn struct { + *net.UDPConn + unhandled chan discover.ReadPacket +} + +// ReadFromUDP implements discv5.conn +func (s *sharedUDPConn) ReadFromUDP(b []byte) (n int, addr *net.UDPAddr, err error) { + packet, ok := <-s.unhandled + if !ok { + return 0, nil, fmt.Errorf("Connection was closed") + } + l := len(packet.Data) + if l > len(b) { + l = len(b) + } + copy(b[:l], packet.Data[:l]) + return l, packet.Addr, nil +} + +// Close implements discv5.conn +func (s *sharedUDPConn) Close() error { + return nil +} + // Start starts running the server. // Servers can not be re-used after stopping. func (srv *Server) Start() (err error) { @@ -388,9 +411,43 @@ func (srv *Server) Start() (err error) { srv.peerOp = make(chan peerOpFunc) srv.peerOpDone = make(chan struct{}) + var ( + conn *net.UDPConn + sconn *sharedUDPConn + realaddr *net.UDPAddr + unhandled chan discover.ReadPacket + ) + + if !srv.NoDiscovery || srv.DiscoveryV5 { + addr, err := net.ResolveUDPAddr("udp", srv.ListenAddr) + if err != nil { + return err + } + conn, err = net.ListenUDP("udp", addr) + if err != nil { + return err + } + + realaddr = conn.LocalAddr().(*net.UDPAddr) + if srv.NAT != nil { + if !realaddr.IP.IsLoopback() { + go nat.Map(srv.NAT, srv.quit, "udp", realaddr.Port, realaddr.Port, "ethereum discovery") + } + // TODO: react to external IP changes over time. + if ext, err := srv.NAT.ExternalIP(); err == nil { + realaddr = &net.UDPAddr{IP: ext, Port: realaddr.Port} + } + } + } + + if !srv.NoDiscovery && srv.DiscoveryV5 { + unhandled = make(chan discover.ReadPacket, 100) + sconn = &sharedUDPConn{conn, unhandled} + } + // node table if !srv.NoDiscovery { - ntab, err := discover.ListenUDP(srv.PrivateKey, srv.ListenAddr, srv.NAT, srv.NodeDatabase, srv.NetRestrict) + ntab, err := discover.ListenUDP(srv.PrivateKey, conn, realaddr, unhandled, srv.NodeDatabase, srv.NetRestrict) if err != nil { return err } @@ -401,7 +458,15 @@ func (srv *Server) Start() (err error) { } if srv.DiscoveryV5 { - ntab, err := discv5.ListenUDP(srv.PrivateKey, srv.DiscoveryV5Addr, srv.NAT, "", srv.NetRestrict) //srv.NodeDatabase) + var ( + ntab *discv5.Network + err error + ) + if sconn != nil { + ntab, err = discv5.ListenUDP(srv.PrivateKey, sconn, realaddr, "", srv.NetRestrict) //srv.NodeDatabase) + } else { + ntab, err = discv5.ListenUDP(srv.PrivateKey, conn, realaddr, "", srv.NetRestrict) //srv.NodeDatabase) + } if err != nil { return err } diff --git a/params/bootnodes.go b/params/bootnodes.go index ecb1acd4f1..849b569200 100644 --- a/params/bootnodes.go +++ b/params/bootnodes.go @@ -56,7 +56,7 @@ var RinkebyV5Bootnodes = []string{ // DiscoveryV5Bootnodes are the enode URLs of the P2P bootstrap nodes for the // experimental RLPx v5 topic-discovery network. var DiscoveryV5Bootnodes = []string{ - "enode://0cc5f5ffb5d9098c8b8c62325f3797f56509bff942704687b6530992ac706e2cb946b90a34f1f19548cd3c7baccbcaea354531e5983c7d1bc0dee16ce4b6440b@40.118.3.223:30305", - "enode://1c7a64d76c0334b0418c004af2f67c50e36a3be60b5e4790bdac0439d21603469a85fad36f2473c9a80eb043ae60936df905fa28f1ff614c3e5dc34f15dcd2dc@40.118.3.223:30308", - "enode://85c85d7143ae8bb96924f2b54f1b3e70d8c4d367af305325d30a61385a432f247d2c75c45c6b4a60335060d072d7f5b35dd1d4c45f76941f62a4f83b6e75daaf@40.118.3.223:30309", + "enode://0cc5f5ffb5d9098c8b8c62325f3797f56509bff942704687b6530992ac706e2cb946b90a34f1f19548cd3c7baccbcaea354531e5983c7d1bc0dee16ce4b6440b@40.118.3.223:30304", + "enode://1c7a64d76c0334b0418c004af2f67c50e36a3be60b5e4790bdac0439d21603469a85fad36f2473c9a80eb043ae60936df905fa28f1ff614c3e5dc34f15dcd2dc@40.118.3.223:30306", + "enode://85c85d7143ae8bb96924f2b54f1b3e70d8c4d367af305325d30a61385a432f247d2c75c45c6b4a60335060d072d7f5b35dd1d4c45f76941f62a4f83b6e75daaf@40.118.3.223:30307", } From 295512f5a83923a5e3d316cda898e158e1385a44 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Mon, 22 Jan 2018 15:23:41 +0100 Subject: [PATCH 073/210] swarm/storage, swarm/network: Fix race condition There was a race condition in writing/reading chunk.SData between delivery.processReceivedChunks and dpa.retrieveWorker when the chunk was still fetching --- swarm/network/stream/delivery.go | 8 ++++---- swarm/storage/common_test.go | 12 ++++++++++++ swarm/storage/dbstore.go | 6 +++--- swarm/storage/dbstore_test.go | 4 ++-- swarm/storage/dpa.go | 4 ++-- swarm/storage/localstore.go | 19 +++++++++++++------ swarm/storage/memstore.go | 4 ++-- swarm/storage/memstore_test.go | 4 ++-- swarm/storage/netstore.go | 6 ++---- 9 files changed, 42 insertions(+), 25 deletions(-) diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go index 67181c3f22..4355997fd0 100644 --- a/swarm/network/stream/delivery.go +++ b/swarm/network/stream/delivery.go @@ -18,6 +18,7 @@ package stream import ( "errors" + "fmt" "time" "github.com/ethereum/go-ethereum/log" @@ -167,12 +168,11 @@ R: for req := range d.receiveC { // this should be has locally chunk, err := d.db.Get(req.Key) - if err != nil { - log.Error("not in db? ", "key", req.Key, "chunk", chunk) + if err == nil { continue R } - if chunk.ReqC == nil { - continue R + if err != storage.ErrFetching { + panic(fmt.Sprintf("not in db? key %v chunk %v", req.Key, chunk)) } select { case <-chunk.ReqC: diff --git a/swarm/storage/common_test.go b/swarm/storage/common_test.go index 700ba8dac0..21f8e5bbc1 100644 --- a/swarm/storage/common_test.go +++ b/swarm/storage/common_test.go @@ -20,16 +20,28 @@ import ( "bytes" "crypto/rand" "encoding/binary" + "flag" "fmt" "hash" "io" + "os" "sync" "testing" "time" "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/log" ) +var ( + loglevel = flag.Int("loglevel", 2, "verbosity of logs") +) + +func init() { + flag.Parse() + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(false)))) +} + type brokenLimitedReader struct { lr io.Reader errAt int diff --git a/swarm/storage/dbstore.go b/swarm/storage/dbstore.go index 887c2e08de..0facd67c04 100644 --- a/swarm/storage/dbstore.go +++ b/swarm/storage/dbstore.go @@ -699,7 +699,7 @@ func (s *DbStore) get(key Key) (chunk *Chunk, err error) { decodeData(data, chunk) } else { - err = notFound + err = ErrNotFound } return @@ -712,8 +712,8 @@ func newMockGetDataFunc(mockStore *mock.NodeStore) func(key Key) (data []byte, e return func(key Key) (data []byte, err error) { data, err = mockStore.Get(key) if err == mock.ErrNotFound { - // preserve notFound error - err = notFound + // preserve ErrNotFound error + err = ErrNotFound } return data, err } diff --git a/swarm/storage/dbstore_test.go b/swarm/storage/dbstore_test.go index 7f751594e5..6b86ed518e 100644 --- a/swarm/storage/dbstore_test.go +++ b/swarm/storage/dbstore_test.go @@ -142,8 +142,8 @@ func testDbStoreNotFound(t *testing.T, mock bool) { defer db.close() _, err = db.Get(ZeroKey) - if err != notFound { - t.Errorf("Expected notFound, got %v", err) + if err != ErrNotFound { + t.Errorf("Expected ErrNotFound, got %v", err) } } diff --git a/swarm/storage/dpa.go b/swarm/storage/dpa.go index b0aafe0343..7633b61f0b 100644 --- a/swarm/storage/dpa.go +++ b/swarm/storage/dpa.go @@ -48,8 +48,8 @@ const ( ) var ( - notFound = errors.New("not found") - + ErrNotFound = errors.New("not found") + ErrFetching = errors.New("chunk still fetching") // timeout interval before retrieval is timed out searchTimeout = 3 * time.Second ) diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index 893670b232..ac6d642950 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -106,7 +106,15 @@ func (self *LocalStore) Put(chunk *Chunk) { // ChunkStores are remote and can have long latency func (self *LocalStore) Get(key Key) (chunk *Chunk, err error) { chunk, err = self.memStore.Get(key) + if err == nil { + if chunk.ReqC != nil { + select { + case <-chunk.ReqC: + default: + return chunk, ErrFetching + } + } return } chunk, err = self.DbStore.Get(key) @@ -123,12 +131,11 @@ func (self *LocalStore) GetOrCreateRequest(key Key) (chunk *Chunk, created bool) var err error chunk, err = self.Get(key) if err == nil { - if chunk.ReqC == nil { - log.Trace(fmt.Sprintf("LocalStore.GetOrRetrieve: %v found locally", key)) - } else { - log.Trace(fmt.Sprintf("LocalStore.GetOrRetrieve: %v hit on an existing request", key)) - // no need to launch again - } + log.Trace(fmt.Sprintf("LocalStore.GetOrRetrieve: %v found locally", key)) + return chunk, false + } + if err == ErrFetching { + log.Trace(fmt.Sprintf("LocalStore.GetOrRetrieve: %v hit on an existing request %v", key, chunk.ReqC)) return chunk, false } // no data and no request status diff --git a/swarm/storage/memstore.go b/swarm/storage/memstore.go index 7eb0aa06c2..65affc3ffe 100644 --- a/swarm/storage/memstore.go +++ b/swarm/storage/memstore.go @@ -214,7 +214,7 @@ func (s *MemStore) Get(hash Key) (chunk *Chunk, err error) { l := hash.bits(bitpos, node.bits) st := node.subtree[l] if st == nil { - return nil, notFound + return nil, ErrNotFound } bitpos += node.bits node = st @@ -232,7 +232,7 @@ func (s *MemStore) Get(hash Key) (chunk *Chunk, err error) { } } } else { - err = notFound + err = ErrNotFound } return diff --git a/swarm/storage/memstore_test.go b/swarm/storage/memstore_test.go index 6b4bc0da56..edf87917e8 100644 --- a/swarm/storage/memstore_test.go +++ b/swarm/storage/memstore_test.go @@ -63,8 +63,8 @@ func TestMemStoreNotFound(t *testing.T) { defer m.Close() _, err := m.Get(ZeroKey) - if err != notFound { - t.Errorf("Expected notFound, got %v", err) + if err != ErrNotFound { + t.Errorf("Expected ErrNotFound, got %v", err) } } diff --git a/swarm/storage/netstore.go b/swarm/storage/netstore.go index 334cf3635a..265baa5337 100644 --- a/swarm/storage/netstore.go +++ b/swarm/storage/netstore.go @@ -17,7 +17,6 @@ package storage import ( - "encoding/binary" "time" ) @@ -40,7 +39,7 @@ func (self *NetStore) Get(key Key) (chunk *Chunk, err error) { var created bool chunk, created = self.localStore.GetOrCreateRequest(key) if chunk.ReqC == nil { - return + return chunk, nil } if created { @@ -53,10 +52,9 @@ func (self *NetStore) Get(key Key) (chunk *Chunk, err error) { select { case <-t.C: - return nil, notFound + return nil, ErrNotFound case <-chunk.ReqC: } - chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) return chunk, nil } From 48641d7308daa2956197a77208692f0c20cad7c9 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 23 Jan 2018 08:50:11 +0100 Subject: [PATCH 074/210] p2p/discv5: logs info about discv5 node info at bind time --- p2p/discv5/udp.go | 1 + 1 file changed, 1 insertion(+) diff --git a/p2p/discv5/udp.go b/p2p/discv5/udp.go index e921520768..5437718173 100644 --- a/p2p/discv5/udp.go +++ b/p2p/discv5/udp.go @@ -247,6 +247,7 @@ func ListenUDP(priv *ecdsa.PrivateKey, conn conn, realaddr *net.UDPAddr, nodeDBP if err != nil { return nil, err } + log.Info("UDP listener up", "net", net.tab.self) transport.net = net go transport.readLoop() return net, nil From 924065e19d08cc7e6af0b3a5b5b1ef3785b79bd4 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 23 Jan 2018 11:05:30 +0100 Subject: [PATCH 075/210] consensus/ethash: improve cache/dataset handling (#15864) * consensus/ethash: add maxEpoch constant * consensus/ethash: improve cache/dataset handling There are two fixes in this commit: Unmap the memory through a finalizer like the libethash wrapper did. The release logic was incorrect and freed the memory while it was being used, leading to crashes like in #14495 or #14943. Track caches and datasets using simplelru instead of reinventing LRU logic. This should make it easier to see whether it's correct. * consensus/ethash: restore 'future item' logic in lru * consensus/ethash: use mmap even in test mode This makes it possible to shorten the time taken for TestCacheFileEvict. * consensus/ethash: shuffle func calc*Size comments around * consensus/ethash: ensure future cache/dataset is in the lru cache * consensus/ethash: add issue link to the new test * consensus/ethash: fix vet * consensus/ethash: fix test * consensus: tiny issue + nitpick fixes --- consensus/ethash/algorithm.go | 6 +- consensus/ethash/algorithm_go1.7.go | 4 +- consensus/ethash/algorithm_go1.8.go | 34 +-- consensus/ethash/algorithm_go1.8_test.go | 23 +- consensus/ethash/consensus.go | 10 +- consensus/ethash/ethash.go | 288 ++++++++++------------- consensus/ethash/ethash_test.go | 39 +++ consensus/ethash/sealer.go | 17 +- 8 files changed, 208 insertions(+), 213 deletions(-) diff --git a/consensus/ethash/algorithm.go b/consensus/ethash/algorithm.go index 76f19252fa..10767bb312 100644 --- a/consensus/ethash/algorithm.go +++ b/consensus/ethash/algorithm.go @@ -355,9 +355,11 @@ func hashimotoFull(dataset []uint32, hash []byte, nonce uint64) ([]byte, []byte) return hashimoto(hash, nonce, uint64(len(dataset))*4, lookup) } +const maxEpoch = 2048 + // datasetSizes is a lookup table for the ethash dataset size for the first 2048 // epochs (i.e. 61440000 blocks). -var datasetSizes = []uint64{ +var datasetSizes = [maxEpoch]uint64{ 1073739904, 1082130304, 1090514816, 1098906752, 1107293056, 1115684224, 1124070016, 1132461952, 1140849536, 1149232768, 1157627776, 1166013824, 1174404736, 1182786944, 1191180416, @@ -771,7 +773,7 @@ var datasetSizes = []uint64{ // cacheSizes is a lookup table for the ethash verification cache size for the // first 2048 epochs (i.e. 61440000 blocks). -var cacheSizes = []uint64{ +var cacheSizes = [maxEpoch]uint64{ 16776896, 16907456, 17039296, 17170112, 17301056, 17432512, 17563072, 17693888, 17824192, 17955904, 18087488, 18218176, 18349504, 18481088, 18611392, 18742336, 18874304, 19004224, 19135936, 19267264, 19398208, diff --git a/consensus/ethash/algorithm_go1.7.go b/consensus/ethash/algorithm_go1.7.go index c34d041c32..c7f7f48e41 100644 --- a/consensus/ethash/algorithm_go1.7.go +++ b/consensus/ethash/algorithm_go1.7.go @@ -25,7 +25,7 @@ package ethash func cacheSize(block uint64) uint64 { // If we have a pre-generated value, use that epoch := int(block / epochLength) - if epoch < len(cacheSizes) { + if epoch < maxEpoch { return cacheSizes[epoch] } // We don't have a way to verify primes fast before Go 1.8 @@ -39,7 +39,7 @@ func cacheSize(block uint64) uint64 { func datasetSize(block uint64) uint64 { // If we have a pre-generated value, use that epoch := int(block / epochLength) - if epoch < len(datasetSizes) { + if epoch < maxEpoch { return datasetSizes[epoch] } // We don't have a way to verify primes fast before Go 1.8 diff --git a/consensus/ethash/algorithm_go1.8.go b/consensus/ethash/algorithm_go1.8.go index d691b758f0..975fdffe51 100644 --- a/consensus/ethash/algorithm_go1.8.go +++ b/consensus/ethash/algorithm_go1.8.go @@ -20,17 +20,20 @@ package ethash import "math/big" -// cacheSize calculates and returns the size of the ethash verification cache that -// belongs to a certain block number. The cache size grows linearly, however, we -// always take the highest prime below the linearly growing threshold in order to -// reduce the risk of accidental regularities leading to cyclic behavior. +// cacheSize returns the size of the ethash verification cache that belongs to a certain +// block number. func cacheSize(block uint64) uint64 { - // If we have a pre-generated value, use that epoch := int(block / epochLength) - if epoch < len(cacheSizes) { + if epoch < maxEpoch { return cacheSizes[epoch] } - // No known cache size, calculate manually (sanity branch only) + return calcCacheSize(epoch) +} + +// calcCacheSize calculates the cache size for epoch. The cache size grows linearly, +// however, we always take the highest prime below the linearly growing threshold in order +// to reduce the risk of accidental regularities leading to cyclic behavior. +func calcCacheSize(epoch int) uint64 { size := cacheInitBytes + cacheGrowthBytes*uint64(epoch) - hashBytes for !new(big.Int).SetUint64(size / hashBytes).ProbablyPrime(1) { // Always accurate for n < 2^64 size -= 2 * hashBytes @@ -38,17 +41,20 @@ func cacheSize(block uint64) uint64 { return size } -// datasetSize calculates and returns the size of the ethash mining dataset that -// belongs to a certain block number. The dataset size grows linearly, however, we -// always take the highest prime below the linearly growing threshold in order to -// reduce the risk of accidental regularities leading to cyclic behavior. +// datasetSize returns the size of the ethash mining dataset that belongs to a certain +// block number. func datasetSize(block uint64) uint64 { - // If we have a pre-generated value, use that epoch := int(block / epochLength) - if epoch < len(datasetSizes) { + if epoch < maxEpoch { return datasetSizes[epoch] } - // No known dataset size, calculate manually (sanity branch only) + return calcDatasetSize(epoch) +} + +// calcDatasetSize calculates the dataset size for epoch. The dataset size grows linearly, +// however, we always take the highest prime below the linearly growing threshold in order +// to reduce the risk of accidental regularities leading to cyclic behavior. +func calcDatasetSize(epoch int) uint64 { size := datasetInitBytes + datasetGrowthBytes*uint64(epoch) - mixBytes for !new(big.Int).SetUint64(size / mixBytes).ProbablyPrime(1) { // Always accurate for n < 2^64 size -= 2 * mixBytes diff --git a/consensus/ethash/algorithm_go1.8_test.go b/consensus/ethash/algorithm_go1.8_test.go index a822944a60..6648bd6a97 100644 --- a/consensus/ethash/algorithm_go1.8_test.go +++ b/consensus/ethash/algorithm_go1.8_test.go @@ -23,24 +23,15 @@ import "testing" // Tests whether the dataset size calculator works correctly by cross checking the // hard coded lookup table with the value generated by it. func TestSizeCalculations(t *testing.T) { - var tests []uint64 - - // Verify all the cache sizes from the lookup table - defer func(sizes []uint64) { cacheSizes = sizes }(cacheSizes) - tests, cacheSizes = cacheSizes, []uint64{} - - for i, test := range tests { - if size := cacheSize(uint64(i*epochLength) + 1); size != test { - t.Errorf("cache %d: cache size mismatch: have %d, want %d", i, size, test) + // Verify all the cache and dataset sizes from the lookup table. + for epoch, want := range cacheSizes { + if size := calcCacheSize(epoch); size != want { + t.Errorf("cache %d: cache size mismatch: have %d, want %d", epoch, size, want) } } - // Verify all the dataset sizes from the lookup table - defer func(sizes []uint64) { datasetSizes = sizes }(datasetSizes) - tests, datasetSizes = datasetSizes, []uint64{} - - for i, test := range tests { - if size := datasetSize(uint64(i*epochLength) + 1); size != test { - t.Errorf("dataset %d: dataset size mismatch: have %d, want %d", i, size, test) + for epoch, want := range datasetSizes { + if size := calcDatasetSize(epoch); size != want { + t.Errorf("dataset %d: dataset size mismatch: have %d, want %d", epoch, size, want) } } } diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 82d23c92b6..92a23d4a4d 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -476,7 +476,7 @@ func (ethash *Ethash) VerifySeal(chain consensus.ChainReader, header *types.Head } // Sanity check that the block number is below the lookup table size (60M blocks) number := header.Number.Uint64() - if number/epochLength >= uint64(len(cacheSizes)) { + if number/epochLength >= maxEpoch { // Go < 1.7 cannot calculate new cache/dataset sizes (no fast prime check) return errNonceOutOfRange } @@ -484,14 +484,18 @@ func (ethash *Ethash) VerifySeal(chain consensus.ChainReader, header *types.Head if header.Difficulty.Sign() <= 0 { return errInvalidDifficulty } + // Recompute the digest and PoW value and verify against the header cache := ethash.cache(number) - size := datasetSize(number) if ethash.config.PowMode == ModeTest { size = 32 * 1024 } - digest, result := hashimotoLight(size, cache, header.HashNoNonce().Bytes(), header.Nonce.Uint64()) + digest, result := hashimotoLight(size, cache.cache, header.HashNoNonce().Bytes(), header.Nonce.Uint64()) + // Caches are unmapped in a finalizer. Ensure that the cache stays live + // until after the call to hashimotoLight so it's not unmapped while being used. + runtime.KeepAlive(cache) + if !bytes.Equal(header.MixDigest[:], digest) { return errInvalidMixDigest } diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index a78b3a895d..91e20112ae 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -26,6 +26,7 @@ import ( "os" "path/filepath" "reflect" + "runtime" "strconv" "sync" "time" @@ -35,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" + "github.com/hashicorp/golang-lru/simplelru" metrics "github.com/rcrowley/go-metrics" ) @@ -142,32 +144,82 @@ func memoryMapAndGenerate(path string, size uint64, generator func(buffer []uint return memoryMap(path) } +// lru tracks caches or datasets by their last use time, keeping at most N of them. +type lru struct { + what string + new func(epoch uint64) interface{} + mu sync.Mutex + // Items are kept in a LRU cache, but there is a special case: + // We always keep an item for (highest seen epoch) + 1 as the 'future item'. + cache *simplelru.LRU + future uint64 + futureItem interface{} +} + +// newlru create a new least-recently-used cache for ither the verification caches +// or the mining datasets. +func newlru(what string, maxItems int, new func(epoch uint64) interface{}) *lru { + if maxItems <= 0 { + maxItems = 1 + } + cache, _ := simplelru.NewLRU(maxItems, func(key, value interface{}) { + log.Trace("Evicted ethash "+what, "epoch", key) + }) + return &lru{what: what, new: new, cache: cache} +} + +// get retrieves or creates an item for the given epoch. The first return value is always +// non-nil. The second return value is non-nil if lru thinks that an item will be useful in +// the near future. +func (lru *lru) get(epoch uint64) (item, future interface{}) { + lru.mu.Lock() + defer lru.mu.Unlock() + + // Get or create the item for the requested epoch. + item, ok := lru.cache.Get(epoch) + if !ok { + if lru.future > 0 && lru.future == epoch { + item = lru.futureItem + } else { + log.Trace("Requiring new ethash "+lru.what, "epoch", epoch) + item = lru.new(epoch) + } + lru.cache.Add(epoch, item) + } + // Update the 'future item' if epoch is larger than previously seen. + if epoch < maxEpoch-1 && lru.future < epoch+1 { + log.Trace("Requiring new future ethash "+lru.what, "epoch", epoch+1) + future = lru.new(epoch + 1) + lru.future = epoch + 1 + lru.futureItem = future + } + return item, future +} + // cache wraps an ethash cache with some metadata to allow easier concurrent use. type cache struct { - epoch uint64 // Epoch for which this cache is relevant + epoch uint64 // Epoch for which this cache is relevant + dump *os.File // File descriptor of the memory mapped cache + mmap mmap.MMap // Memory map itself to unmap before releasing + cache []uint32 // The actual cache data content (may be memory mapped) + once sync.Once // Ensures the cache is generated only once +} - dump *os.File // File descriptor of the memory mapped cache - mmap mmap.MMap // Memory map itself to unmap before releasing - - cache []uint32 // The actual cache data content (may be memory mapped) - used time.Time // Timestamp of the last use for smarter eviction - once sync.Once // Ensures the cache is generated only once - lock sync.Mutex // Ensures thread safety for updating the usage time +// newCache creates a new ethash verification cache and returns it as a plain Go +// interface to be usable in an LRU cache. +func newCache(epoch uint64) interface{} { + return &cache{epoch: epoch} } // generate ensures that the cache content is generated before use. func (c *cache) generate(dir string, limit int, test bool) { c.once.Do(func() { - // If we have a testing cache, generate and return - if test { - c.cache = make([]uint32, 1024/4) - generateCache(c.cache, c.epoch, seedHash(c.epoch*epochLength+1)) - return - } - // If we don't store anything on disk, generate and return size := cacheSize(c.epoch*epochLength + 1) seed := seedHash(c.epoch*epochLength + 1) - + if test { + size = 1024 + } + // If we don't store anything on disk, generate and return. if dir == "" { c.cache = make([]uint32, size/4) generateCache(c.cache, c.epoch, seed) @@ -181,6 +233,10 @@ func (c *cache) generate(dir string, limit int, test bool) { path := filepath.Join(dir, fmt.Sprintf("cache-R%d-%x%s", algorithmRevision, seed[:8], endian)) logger := log.New("epoch", c.epoch) + // We're about to mmap the file, ensure that the mapping is cleaned up when the + // cache becomes unused. + runtime.SetFinalizer(c, (*cache).finalizer) + // Try to load the file from disk and memory map it var err error c.dump, c.mmap, c.cache, err = memoryMap(path) @@ -207,49 +263,41 @@ func (c *cache) generate(dir string, limit int, test bool) { }) } -// release closes any file handlers and memory maps open. -func (c *cache) release() { +// finalizer unmaps the memory and closes the file. +func (c *cache) finalizer() { if c.mmap != nil { c.mmap.Unmap() - c.mmap = nil - } - if c.dump != nil { c.dump.Close() - c.dump = nil + c.mmap, c.dump = nil, nil } } // dataset wraps an ethash dataset with some metadata to allow easier concurrent use. type dataset struct { - epoch uint64 // Epoch for which this cache is relevant + epoch uint64 // Epoch for which this cache is relevant + dump *os.File // File descriptor of the memory mapped cache + mmap mmap.MMap // Memory map itself to unmap before releasing + dataset []uint32 // The actual cache data content + once sync.Once // Ensures the cache is generated only once +} - dump *os.File // File descriptor of the memory mapped cache - mmap mmap.MMap // Memory map itself to unmap before releasing - - dataset []uint32 // The actual cache data content - used time.Time // Timestamp of the last use for smarter eviction - once sync.Once // Ensures the cache is generated only once - lock sync.Mutex // Ensures thread safety for updating the usage time +// newDataset creates a new ethash mining dataset and returns it as a plain Go +// interface to be usable in an LRU cache. +func newDataset(epoch uint64) interface{} { + return &dataset{epoch: epoch} } // generate ensures that the dataset content is generated before use. func (d *dataset) generate(dir string, limit int, test bool) { d.once.Do(func() { - // If we have a testing dataset, generate and return - if test { - cache := make([]uint32, 1024/4) - generateCache(cache, d.epoch, seedHash(d.epoch*epochLength+1)) - - d.dataset = make([]uint32, 32*1024/4) - generateDataset(d.dataset, d.epoch, cache) - - return - } - // If we don't store anything on disk, generate and return csize := cacheSize(d.epoch*epochLength + 1) dsize := datasetSize(d.epoch*epochLength + 1) seed := seedHash(d.epoch*epochLength + 1) - + if test { + csize = 1024 + dsize = 32 * 1024 + } + // If we don't store anything on disk, generate and return if dir == "" { cache := make([]uint32, csize/4) generateCache(cache, d.epoch, seed) @@ -265,6 +313,10 @@ func (d *dataset) generate(dir string, limit int, test bool) { path := filepath.Join(dir, fmt.Sprintf("full-R%d-%x%s", algorithmRevision, seed[:8], endian)) logger := log.New("epoch", d.epoch) + // We're about to mmap the file, ensure that the mapping is cleaned up when the + // cache becomes unused. + runtime.SetFinalizer(d, (*dataset).finalizer) + // Try to load the file from disk and memory map it var err error d.dump, d.mmap, d.dataset, err = memoryMap(path) @@ -294,15 +346,12 @@ func (d *dataset) generate(dir string, limit int, test bool) { }) } -// release closes any file handlers and memory maps open. -func (d *dataset) release() { +// finalizer closes any file handlers and memory maps open. +func (d *dataset) finalizer() { if d.mmap != nil { d.mmap.Unmap() - d.mmap = nil - } - if d.dump != nil { d.dump.Close() - d.dump = nil + d.mmap, d.dump = nil, nil } } @@ -310,14 +359,12 @@ func (d *dataset) release() { func MakeCache(block uint64, dir string) { c := cache{epoch: block / epochLength} c.generate(dir, math.MaxInt32, false) - c.release() } // MakeDataset generates a new ethash dataset and optionally stores it to disk. func MakeDataset(block uint64, dir string) { d := dataset{epoch: block / epochLength} d.generate(dir, math.MaxInt32, false) - d.release() } // Mode defines the type and amount of PoW verification an ethash engine makes. @@ -347,10 +394,8 @@ type Config struct { type Ethash struct { config Config - caches map[uint64]*cache // In memory caches to avoid regenerating too often - fcache *cache // Pre-generated cache for the estimated future epoch - datasets map[uint64]*dataset // In memory datasets to avoid regenerating too often - fdataset *dataset // Pre-generated dataset for the estimated future epoch + caches *lru // In memory caches to avoid regenerating too often + datasets *lru // In memory datasets to avoid regenerating too often // Mining related fields rand *rand.Rand // Properly seeded random source for nonces @@ -380,8 +425,8 @@ func New(config Config) *Ethash { } return &Ethash{ config: config, - caches: make(map[uint64]*cache), - datasets: make(map[uint64]*dataset), + caches: newlru("cache", config.CachesInMem, newCache), + datasets: newlru("dataset", config.DatasetsInMem, newDataset), update: make(chan struct{}), hashrate: metrics.NewMeter(), } @@ -390,16 +435,7 @@ func New(config Config) *Ethash { // NewTester creates a small sized ethash PoW scheme useful only for testing // purposes. func NewTester() *Ethash { - return &Ethash{ - config: Config{ - CachesInMem: 1, - PowMode: ModeTest, - }, - caches: make(map[uint64]*cache), - datasets: make(map[uint64]*dataset), - update: make(chan struct{}), - hashrate: metrics.NewMeter(), - } + return New(Config{CachesInMem: 1, PowMode: ModeTest}) } // NewFaker creates a ethash consensus engine with a fake PoW scheme that accepts @@ -456,126 +492,40 @@ func NewShared() *Ethash { // cache tries to retrieve a verification cache for the specified block number // by first checking against a list of in-memory caches, then against caches // stored on disk, and finally generating one if none can be found. -func (ethash *Ethash) cache(block uint64) []uint32 { +func (ethash *Ethash) cache(block uint64) *cache { epoch := block / epochLength + currentI, futureI := ethash.caches.get(epoch) + current := currentI.(*cache) - // If we have a PoW for that epoch, use that - ethash.lock.Lock() - - current, future := ethash.caches[epoch], (*cache)(nil) - if current == nil { - // No in-memory cache, evict the oldest if the cache limit was reached - for len(ethash.caches) > 0 && len(ethash.caches) >= ethash.config.CachesInMem { - var evict *cache - for _, cache := range ethash.caches { - if evict == nil || evict.used.After(cache.used) { - evict = cache - } - } - delete(ethash.caches, evict.epoch) - evict.release() - - log.Trace("Evicted ethash cache", "epoch", evict.epoch, "used", evict.used) - } - // If we have the new cache pre-generated, use that, otherwise create a new one - if ethash.fcache != nil && ethash.fcache.epoch == epoch { - log.Trace("Using pre-generated cache", "epoch", epoch) - current, ethash.fcache = ethash.fcache, nil - } else { - log.Trace("Requiring new ethash cache", "epoch", epoch) - current = &cache{epoch: epoch} - } - ethash.caches[epoch] = current - - // If we just used up the future cache, or need a refresh, regenerate - if ethash.fcache == nil || ethash.fcache.epoch <= epoch { - if ethash.fcache != nil { - ethash.fcache.release() - } - log.Trace("Requiring new future ethash cache", "epoch", epoch+1) - future = &cache{epoch: epoch + 1} - ethash.fcache = future - } - // New current cache, set its initial timestamp - current.used = time.Now() - } - ethash.lock.Unlock() - - // Wait for generation finish, bump the timestamp and finalize the cache + // Wait for generation finish. current.generate(ethash.config.CacheDir, ethash.config.CachesOnDisk, ethash.config.PowMode == ModeTest) - current.lock.Lock() - current.used = time.Now() - current.lock.Unlock() - - // If we exhausted the future cache, now's a good time to regenerate it - if future != nil { + // If we need a new future cache, now's a good time to regenerate it. + if futureI != nil { + future := futureI.(*cache) go future.generate(ethash.config.CacheDir, ethash.config.CachesOnDisk, ethash.config.PowMode == ModeTest) } - return current.cache + return current } // dataset tries to retrieve a mining dataset for the specified block number // by first checking against a list of in-memory datasets, then against DAGs // stored on disk, and finally generating one if none can be found. -func (ethash *Ethash) dataset(block uint64) []uint32 { +func (ethash *Ethash) dataset(block uint64) *dataset { epoch := block / epochLength + currentI, futureI := ethash.datasets.get(epoch) + current := currentI.(*dataset) - // If we have a PoW for that epoch, use that - ethash.lock.Lock() - - current, future := ethash.datasets[epoch], (*dataset)(nil) - if current == nil { - // No in-memory dataset, evict the oldest if the dataset limit was reached - for len(ethash.datasets) > 0 && len(ethash.datasets) >= ethash.config.DatasetsInMem { - var evict *dataset - for _, dataset := range ethash.datasets { - if evict == nil || evict.used.After(dataset.used) { - evict = dataset - } - } - delete(ethash.datasets, evict.epoch) - evict.release() - - log.Trace("Evicted ethash dataset", "epoch", evict.epoch, "used", evict.used) - } - // If we have the new cache pre-generated, use that, otherwise create a new one - if ethash.fdataset != nil && ethash.fdataset.epoch == epoch { - log.Trace("Using pre-generated dataset", "epoch", epoch) - current = &dataset{epoch: ethash.fdataset.epoch} // Reload from disk - ethash.fdataset = nil - } else { - log.Trace("Requiring new ethash dataset", "epoch", epoch) - current = &dataset{epoch: epoch} - } - ethash.datasets[epoch] = current - - // If we just used up the future dataset, or need a refresh, regenerate - if ethash.fdataset == nil || ethash.fdataset.epoch <= epoch { - if ethash.fdataset != nil { - ethash.fdataset.release() - } - log.Trace("Requiring new future ethash dataset", "epoch", epoch+1) - future = &dataset{epoch: epoch + 1} - ethash.fdataset = future - } - // New current dataset, set its initial timestamp - current.used = time.Now() - } - ethash.lock.Unlock() - - // Wait for generation finish, bump the timestamp and finalize the cache + // Wait for generation finish. current.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest) - current.lock.Lock() - current.used = time.Now() - current.lock.Unlock() - - // If we exhausted the future dataset, now's a good time to regenerate it - if future != nil { + // If we need a new future dataset, now's a good time to regenerate it. + if futureI != nil { + future := futureI.(*dataset) go future.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest) } - return current.dataset + + return current } // Threads returns the number of mining threads currently enabled. This doesn't diff --git a/consensus/ethash/ethash_test.go b/consensus/ethash/ethash_test.go index b3a2f32f70..31116da437 100644 --- a/consensus/ethash/ethash_test.go +++ b/consensus/ethash/ethash_test.go @@ -17,7 +17,11 @@ package ethash import ( + "io/ioutil" "math/big" + "math/rand" + "os" + "sync" "testing" "github.com/ethereum/go-ethereum/core/types" @@ -38,3 +42,38 @@ func TestTestMode(t *testing.T) { t.Fatalf("unexpected verification error: %v", err) } } + +// This test checks that cache lru logic doesn't crash under load. +// It reproduces https://github.com/ethereum/go-ethereum/issues/14943 +func TestCacheFileEvict(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "ethash-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + e := New(Config{CachesInMem: 3, CachesOnDisk: 10, CacheDir: tmpdir, PowMode: ModeTest}) + + workers := 8 + epochs := 100 + var wg sync.WaitGroup + wg.Add(workers) + for i := 0; i < workers; i++ { + go verifyTest(&wg, e, i, epochs) + } + wg.Wait() +} + +func verifyTest(wg *sync.WaitGroup, e *Ethash, workerIndex, epochs int) { + defer wg.Done() + + const wiggle = 4 * epochLength + r := rand.New(rand.NewSource(int64(workerIndex))) + for epoch := 0; epoch < epochs; epoch++ { + block := int64(epoch)*epochLength - wiggle/2 + r.Int63n(wiggle) + if block < 0 { + block = 0 + } + head := &types.Header{Number: big.NewInt(block), Difficulty: big.NewInt(100)} + e.VerifySeal(nil, head) + } +} diff --git a/consensus/ethash/sealer.go b/consensus/ethash/sealer.go index c2447e4730..b5e742d8bb 100644 --- a/consensus/ethash/sealer.go +++ b/consensus/ethash/sealer.go @@ -97,10 +97,9 @@ func (ethash *Ethash) Seal(chain consensus.ChainReader, block *types.Block, stop func (ethash *Ethash) mine(block *types.Block, id int, seed uint64, abort chan struct{}, found chan *types.Block) { // Extract some data from the header var ( - header = block.Header() - hash = header.HashNoNonce().Bytes() - target = new(big.Int).Div(maxUint256, header.Difficulty) - + header = block.Header() + hash = header.HashNoNonce().Bytes() + target = new(big.Int).Div(maxUint256, header.Difficulty) number = header.Number.Uint64() dataset = ethash.dataset(number) ) @@ -111,13 +110,14 @@ func (ethash *Ethash) mine(block *types.Block, id int, seed uint64, abort chan s ) logger := log.New("miner", id) logger.Trace("Started ethash search for new nonces", "seed", seed) +search: for { select { case <-abort: // Mining terminated, update stats and abort logger.Trace("Ethash nonce search aborted", "attempts", nonce-seed) ethash.hashrate.Mark(attempts) - return + break search default: // We don't have to update hash rate on every nonce, so update after after 2^X nonces @@ -127,7 +127,7 @@ func (ethash *Ethash) mine(block *types.Block, id int, seed uint64, abort chan s attempts = 0 } // Compute the PoW value of this nonce - digest, result := hashimotoFull(dataset, hash, nonce) + digest, result := hashimotoFull(dataset.dataset, hash, nonce) if new(big.Int).SetBytes(result).Cmp(target) <= 0 { // Correct nonce found, create a new header with it header = types.CopyHeader(header) @@ -141,9 +141,12 @@ func (ethash *Ethash) mine(block *types.Block, id int, seed uint64, abort chan s case <-abort: logger.Trace("Ethash nonce found but discarded", "attempts", nonce-seed, "nonce", nonce) } - return + break search } nonce++ } } + // Datasets are unmapped in a finalizer. Ensure that the dataset stays live + // during sealing so it's not unmapped while being read. + runtime.KeepAlive(dataset) } From 2aa9791fc77d3d9f28c9b54e636ce25f2cc8ae99 Mon Sep 17 00:00:00 2001 From: zelig Date: Tue, 23 Jan 2018 11:40:40 +0100 Subject: [PATCH 076/210] temporary debug --- swarm/network/stream/messages.go | 24 +- swarm/network/stream/syncer.go | 3 +- swarm/network/stream/syncer_test.go | 30 +- swarm/storage/dbstore.go | 4 +- swarm/storage/memstore.go | 568 +++++++++++++++------------- 5 files changed, 347 insertions(+), 282 deletions(-) diff --git a/swarm/network/stream/messages.go b/swarm/network/stream/messages.go index a575b915a6..1cf9aeab43 100644 --- a/swarm/network/stream/messages.go +++ b/swarm/network/stream/messages.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "sync" + "time" "github.com/ethereum/go-ethereum/log" bv "github.com/ethereum/go-ethereum/swarm/network/bitvector" @@ -130,6 +131,19 @@ func (p *Peer) handleOfferedHashesMsg(req *OfferedHashesMsg) error { }(wait) } } + // done := make(chan bool) + // go func() { + // wg.Wait() + // close(done) + // }() + // go func() { + // select { + // case <-done: + // s.next <- s.batchDone(p, req, hashes) + // case <-time.After(1 * time.Second): + // p.Drop(errors.New("timeout waiting for batch to be delivered")) + // } + // }() go func() { wg.Wait() s.next <- s.batchDone(p, req, hashes) @@ -154,6 +168,9 @@ func (p *Peer) handleOfferedHashesMsg(req *OfferedHashesMsg) error { } go func() { select { + case <-time.After(1 * time.Second): + p.Drop(errors.New("timeout waiting for batch to be delivered")) + return case err := <-s.next: if err != nil { p.Drop(err) @@ -196,7 +213,12 @@ func (p *Peer) handleWantedHashesMsg(req *WantedHashesMsg) error { } hashes := s.currentBatch // launch in go routine since GetBatch blocks until new hashes arrive - go p.SendOfferedHashes(s, req.From, req.To) + go func() { + if err := p.SendOfferedHashes(s, req.From, req.To); err != nil { + p.Drop(err) + } + }() + // go p.SendOfferedHashes(s, req.From, req.To) l := len(hashes) / HashSize want, err := bv.NewFromBytes(req.Want, l) if err != nil { diff --git a/swarm/network/stream/syncer.go b/swarm/network/stream/syncer.go index 99436e7ecf..eff42b4cd0 100644 --- a/swarm/network/stream/syncer.go +++ b/swarm/network/stream/syncer.go @@ -182,15 +182,14 @@ func RegisterSwarmSyncerClient(streamer *Registry, db *storage.DBAPI) { // NeedData func (s *SwarmSyncerClient) NeedData(key []byte) (wait func()) { chunk, _ := s.db.GetOrCreateRequest(key) - log.Warn("created request", "key", chunk.Key) // TODO: we may want to request from this peer anyway even if the request exists if chunk.ReqC == nil { + log.Error("oops this is found") return nil } // create request and wait until the chunk data arrives and is stored return func() { chunk.WaitToStore() - log.Warn("stored", "key", chunk.Key) } } diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index 11de0bf0c7..d57be133cb 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -37,12 +37,12 @@ import ( const dataChunkCount = 500 func TestSyncerSimulation(t *testing.T) { - testSyncBetweenNodes(t, 2, 1, dataChunkCount, true, 1) + // testSyncBetweenNodes(t, 2, 1, dataChunkCount, true, 1) // testSyncBetweenNodes(t, 2, 1, dataChunkCount, false, 1) - testSyncBetweenNodes(t, 4, 1, dataChunkCount, true, 1) - // testSyncBetweenNodes(t, 4, 1, dataChunkCount, false, 1) - testSyncBetweenNodes(t, 8, 1, dataChunkCount, true, 1) - // testSyncBetweenNodes(t, 8, 1, dataChunkCount, false, 1) + // testSyncBetweenNodes(t, 4, 1, dataChunkCount, true, 1) + // // testSyncBetweenNodes(t, 4, 1, dataChunkCount, false, 1) + // testSyncBetweenNodes(t, 8, 1, dataChunkCount, true, 1) + // // testSyncBetweenNodes(t, 8, 1, dataChunkCount, false, 1) testSyncBetweenNodes(t, 16, 1, dataChunkCount, true, 1) // testSyncBetweenNodes(t, 16, 1, dataChunkCount, false, 1) } @@ -103,7 +103,9 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck }) } + errc := make(chan error, 1) waitPeerErrC = make(chan error) + quitC := make(chan struct{}) action := func(ctx context.Context) error { // need to wait till an aynchronous process registers the peers in streamer.peers // that is used by Subscribe @@ -122,6 +124,10 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck // each node Subscribes to each other's swarmChunkServerStreamName j := 0 return sim.CallClient(func(client *rpc.Client) error { + err := streamTesting.WatchDisconnections(sim.IDs[j], client, errc, quitC) + if err != nil { + return err + } ctx, cancel := context.WithTimeout(ctx, 1*time.Second) defer cancel() j++ @@ -135,6 +141,8 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck defer func() { checkC <- struct{}{} }() select { + case err := <-errc: + return false, err case <-ctx.Done(): return false, ctx.Err() default: @@ -148,13 +156,19 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck found++ } } - log.Debug("sync check", "bin", po, "found", found, "total", total) - return found == total, nil + log.Error("sync check", "bin", po, "found", found, "total", total) + pass := found == total + if !pass { + return false, nil + } + close(quitC) + return true, nil + } conf.Step = &simulations.Step{ Action: action, - Trigger: streamTesting.PivotTrigger(10*time.Millisecond, checkC, sim.IDs[0]), + Trigger: streamTesting.PivotTrigger(100*time.Millisecond, checkC, sim.IDs[0]), Expect: &simulations.Expectation{ Nodes: sim.IDs[0:1], Check: check, diff --git a/swarm/storage/dbstore.go b/swarm/storage/dbstore.go index 0facd67c04..d8dace4758 100644 --- a/swarm/storage/dbstore.go +++ b/swarm/storage/dbstore.go @@ -601,7 +601,9 @@ func (s *DbStore) writeBatches() { s.lock.Unlock() err := s.writeBatch(b, e, d, a) // TODO: set this error on the batch, then tell the chunk - log.Trace(fmt.Sprintf("DbStore: spawn batch write (%d chunks): %v", b.Len(), err)) + if err != nil { + log.Error(fmt.Sprintf("DbStore: spawn batch write (%d chunks): %v", b.Len(), err)) + } close(c) if e >= s.capacity { log.Trace(fmt.Sprintf("DbStore: collecting garbage...(%d chunks)", e)) diff --git a/swarm/storage/memstore.go b/swarm/storage/memstore.go index 65affc3ffe..31e2baf454 100644 --- a/swarm/storage/memstore.go +++ b/swarm/storage/memstore.go @@ -19,10 +19,7 @@ package storage import ( - "fmt" "sync" - - "github.com/ethereum/go-ethereum/log" ) const ( @@ -32,286 +29,317 @@ const ( defaultCacheCapacity = 5000 ) +// type MemStore struct { +// memtree *memTree +// entryCnt, capacity uint // stored entries +// accessCnt uint64 // access counter; oldest is thrown away when full +// dbAccessCnt uint64 +// dbStore *DbStore +// lock sync.Mutex +// } +// +// /* +// a hash prefix subtree containing subtrees or one storage entry (but never both) +// +// - access[0] stores the smallest (oldest) access count value in this subtree +// - if it contains more subtrees and its subtree count is at least 4, access[1:2] +// stores the smallest access count in the first and second halves of subtrees +// (so that access[0] = min(access[1], access[2]) +// - likewise, if subtree count is at least 8, +// access[1] = min(access[3], access[4]) +// access[2] = min(access[5], access[6]) +// (access[] is a binary tree inside the multi-bit leveled hash tree) +// */ +// +// func NewMemStore(d *DbStore, capacity uint) (m *MemStore) { +// m = &MemStore{} +// m.memtree = newMemTree(memTreeFLW, nil, 0) +// m.dbStore = d +// m.setCapacity(capacity) +// return +// } +// +// type memTree struct { +// subtree []*memTree +// parent *memTree +// parentIdx uint +// +// bits uint // log2(subtree count) +// width uint // subtree count +// +// entry *Chunk // if subtrees are present, entry should be nil +// lastDBaccess uint64 +// access []uint64 +// } +// +// func newMemTree(b uint, parent *memTree, pidx uint) (node *memTree) { +// node = new(memTree) +// node.bits = b +// node.width = 1 << b +// node.subtree = make([]*memTree, node.width) +// node.access = make([]uint64, node.width-1) +// node.parent = parent +// node.parentIdx = pidx +// if parent != nil { +// parent.subtree[pidx] = node +// } +// +// return node +// } +// +// func (node *memTree) updateAccess(a uint64) { +// aidx := uint(0) +// var aa uint64 +// oa := node.access[0] +// for node.access[aidx] == oa { +// node.access[aidx] = a +// if aidx > 0 { +// aa = node.access[((aidx-1)^1)+1] +// aidx = (aidx - 1) >> 1 +// } else { +// pidx := node.parentIdx +// node = node.parent +// if node == nil { +// return +// } +// nn := node.subtree[pidx^1] +// if nn != nil { +// aa = nn.access[0] +// } else { +// aa = 0 +// } +// aidx = (node.width + pidx - 2) >> 1 +// } +// +// if (aa != 0) && (aa < a) { +// a = aa +// } +// } +// } +// +// func (s *MemStore) setCapacity(c uint) { +// s.lock.Lock() +// defer s.lock.Unlock() +// +// for c < s.entryCnt { +// s.removeOldest() +// } +// s.capacity = c +// } +// +// // entry (not its copy) is going to be in MemStore +// func (s *MemStore) Put(entry *Chunk) { +// if s.capacity == 0 { +// return +// } +// +// s.lock.Lock() +// defer s.lock.Unlock() +// +// if s.entryCnt >= s.capacity { +// s.removeOldest() +// } +// +// s.accessCnt++ +// +// node := s.memtree +// bitpos := uint(0) +// for node.entry == nil { +// l := entry.Key.bits(bitpos, node.bits) +// st := node.subtree[l] +// if st == nil { +// st = newMemTree(memTreeLW, node, l) +// bitpos += node.bits +// node = st +// break +// } +// bitpos += node.bits +// node = st +// } +// +// if node.entry != nil { +// +// if node.entry.Key.isEqual(entry.Key) { +// node.updateAccess(s.accessCnt) +// if entry.SData == nil { +// entry.Size = node.entry.Size +// entry.SData = node.entry.SData +// } +// if entry.ReqC == nil { +// entry.ReqC = node.entry.ReqC +// } +// entry.C = node.entry.C +// node.entry = entry +// return +// } +// +// for node.entry != nil { +// +// l := node.entry.Key.bits(bitpos, node.bits) +// st := node.subtree[l] +// if st == nil { +// st = newMemTree(memTreeLW, node, l) +// } +// st.entry = node.entry +// node.entry = nil +// st.updateAccess(node.access[0]) +// +// l = entry.Key.bits(bitpos, node.bits) +// st = node.subtree[l] +// if st == nil { +// st = newMemTree(memTreeLW, node, l) +// } +// bitpos += node.bits +// node = st +// +// } +// } +// +// node.entry = entry +// node.lastDBaccess = s.dbAccessCnt +// node.updateAccess(s.accessCnt) +// s.entryCnt++ +// } +// +// func (s *MemStore) Get(hash Key) (chunk *Chunk, err error) { +// s.lock.Lock() +// defer s.lock.Unlock() +// +// node := s.memtree +// bitpos := uint(0) +// for node.entry == nil { +// l := hash.bits(bitpos, node.bits) +// st := node.subtree[l] +// if st == nil { +// return nil, ErrNotFound +// } +// bitpos += node.bits +// node = st +// } +// +// if node.entry.Key.isEqual(hash) { +// s.accessCnt++ +// node.updateAccess(s.accessCnt) +// chunk = node.entry +// if s.dbAccessCnt-node.lastDBaccess > dbForceUpdateAccessCnt { +// s.dbAccessCnt++ +// node.lastDBaccess = s.dbAccessCnt +// if s.dbStore != nil { +// s.dbStore.updateAccessCnt(hash) +// } +// } +// } else { +// err = ErrNotFound +// } +// +// return +// } +// +// func (s *MemStore) removeOldest() { +// node := s.memtree +// log.Warn("purge memstore") +// for node.entry == nil { +// +// aidx := uint(0) +// av := node.access[aidx] +// +// for aidx < node.width/2-1 { +// if av == node.access[aidx*2+1] { +// node.access[aidx] = node.access[aidx*2+2] +// aidx = aidx*2 + 1 +// } else if av == node.access[aidx*2+2] { +// node.access[aidx] = node.access[aidx*2+1] +// aidx = aidx*2 + 2 +// } else { +// panic(nil) +// } +// } +// pidx := aidx*2 + 2 - node.width +// if (node.subtree[pidx] != nil) && (av == node.subtree[pidx].access[0]) { +// if node.subtree[pidx+1] != nil { +// node.access[aidx] = node.subtree[pidx+1].access[0] +// } else { +// node.access[aidx] = 0 +// } +// } else if (node.subtree[pidx+1] != nil) && (av == node.subtree[pidx+1].access[0]) { +// if node.subtree[pidx] != nil { +// node.access[aidx] = node.subtree[pidx].access[0] +// } else { +// node.access[aidx] = 0 +// } +// pidx++ +// } else { +// panic(nil) +// } +// +// //fmt.Println(pidx) +// node = node.subtree[pidx] +// +// } +// +// log.Trace(fmt.Sprintf("Memstore Clean: Waiting for chunk %v to be saved", node.entry.Key.Log())) +// <-node.entry.dbStored +// log.Trace(fmt.Sprintf("Memstore Clean: Chunk %v saved to DBStore. Ready to clear from mem.", node.entry.Key.Log())) +// +// if node.entry.ReqC == nil { +// node.entry = nil +// s.entryCnt-- +// } else { +// return +// } +// +// node.access[0] = 0 +// +// //--- +// +// aidx := uint(0) +// for { +// aa := node.access[aidx] +// if aidx > 0 { +// aidx = (aidx - 1) >> 1 +// } else { +// pidx := node.parentIdx +// node = node.parent +// if node == nil { +// return +// } +// aidx = (node.width + pidx - 2) >> 1 +// } +// if (aa != 0) && ((aa < node.access[aidx]) || (node.access[aidx] == 0)) { +// node.access[aidx] = aa +// } +// } +// } + type MemStore struct { - memtree *memTree - entryCnt, capacity uint // stored entries - accessCnt uint64 // access counter; oldest is thrown away when full - dbAccessCnt uint64 - dbStore *DbStore - lock sync.Mutex + m map[string]*Chunk + mu sync.RWMutex } -/* -a hash prefix subtree containing subtrees or one storage entry (but never both) - -- access[0] stores the smallest (oldest) access count value in this subtree -- if it contains more subtrees and its subtree count is at least 4, access[1:2] - stores the smallest access count in the first and second halves of subtrees - (so that access[0] = min(access[1], access[2]) -- likewise, if subtree count is at least 8, - access[1] = min(access[3], access[4]) - access[2] = min(access[5], access[6]) - (access[] is a binary tree inside the multi-bit leveled hash tree) -*/ - func NewMemStore(d *DbStore, capacity uint) (m *MemStore) { - m = &MemStore{} - m.memtree = newMemTree(memTreeFLW, nil, 0) - m.dbStore = d - m.setCapacity(capacity) - return -} - -type memTree struct { - subtree []*memTree - parent *memTree - parentIdx uint - - bits uint // log2(subtree count) - width uint // subtree count - - entry *Chunk // if subtrees are present, entry should be nil - lastDBaccess uint64 - access []uint64 -} - -func newMemTree(b uint, parent *memTree, pidx uint) (node *memTree) { - node = new(memTree) - node.bits = b - node.width = 1 << b - node.subtree = make([]*memTree, node.width) - node.access = make([]uint64, node.width-1) - node.parent = parent - node.parentIdx = pidx - if parent != nil { - parent.subtree[pidx] = node - } - - return node -} - -func (node *memTree) updateAccess(a uint64) { - aidx := uint(0) - var aa uint64 - oa := node.access[0] - for node.access[aidx] == oa { - node.access[aidx] = a - if aidx > 0 { - aa = node.access[((aidx-1)^1)+1] - aidx = (aidx - 1) >> 1 - } else { - pidx := node.parentIdx - node = node.parent - if node == nil { - return - } - nn := node.subtree[pidx^1] - if nn != nil { - aa = nn.access[0] - } else { - aa = 0 - } - aidx = (node.width + pidx - 2) >> 1 - } - - if (aa != 0) && (aa < a) { - a = aa - } + return &MemStore{ + m: make(map[string]*Chunk), } } -func (s *MemStore) setCapacity(c uint) { - s.lock.Lock() - defer s.lock.Unlock() - - for c < s.entryCnt { - s.removeOldest() +func (m *MemStore) Get(key Key) (*Chunk, error) { + m.mu.RLock() + defer m.mu.RUnlock() + c, ok := m.m[string(key[:])] + if !ok { + return nil, ErrNotFound } - s.capacity = c + return c, nil } -// entry (not its copy) is going to be in MemStore -func (s *MemStore) Put(entry *Chunk) { - if s.capacity == 0 { - return - } - - s.lock.Lock() - defer s.lock.Unlock() - - if s.entryCnt >= s.capacity { - s.removeOldest() - } - - s.accessCnt++ - - node := s.memtree - bitpos := uint(0) - for node.entry == nil { - l := entry.Key.bits(bitpos, node.bits) - st := node.subtree[l] - if st == nil { - st = newMemTree(memTreeLW, node, l) - bitpos += node.bits - node = st - break - } - bitpos += node.bits - node = st - } - - if node.entry != nil { - - if node.entry.Key.isEqual(entry.Key) { - node.updateAccess(s.accessCnt) - if entry.SData == nil { - entry.Size = node.entry.Size - entry.SData = node.entry.SData - } - if entry.ReqC == nil { - entry.ReqC = node.entry.ReqC - } - entry.C = node.entry.C - node.entry = entry - return - } - - for node.entry != nil { - - l := node.entry.Key.bits(bitpos, node.bits) - st := node.subtree[l] - if st == nil { - st = newMemTree(memTreeLW, node, l) - } - st.entry = node.entry - node.entry = nil - st.updateAccess(node.access[0]) - - l = entry.Key.bits(bitpos, node.bits) - st = node.subtree[l] - if st == nil { - st = newMemTree(memTreeLW, node, l) - } - bitpos += node.bits - node = st - - } - } - - node.entry = entry - node.lastDBaccess = s.dbAccessCnt - node.updateAccess(s.accessCnt) - s.entryCnt++ +func (m *MemStore) Put(c *Chunk) { + m.mu.Lock() + defer m.mu.Unlock() + m.m[string(c.Key[:])] = c } -func (s *MemStore) Get(hash Key) (chunk *Chunk, err error) { - s.lock.Lock() - defer s.lock.Unlock() +func (m *MemStore) setCapacity(n int) { - node := s.memtree - bitpos := uint(0) - for node.entry == nil { - l := hash.bits(bitpos, node.bits) - st := node.subtree[l] - if st == nil { - return nil, ErrNotFound - } - bitpos += node.bits - node = st - } - - if node.entry.Key.isEqual(hash) { - s.accessCnt++ - node.updateAccess(s.accessCnt) - chunk = node.entry - if s.dbAccessCnt-node.lastDBaccess > dbForceUpdateAccessCnt { - s.dbAccessCnt++ - node.lastDBaccess = s.dbAccessCnt - if s.dbStore != nil { - s.dbStore.updateAccessCnt(hash) - } - } - } else { - err = ErrNotFound - } - - return -} - -func (s *MemStore) removeOldest() { - node := s.memtree - log.Warn("purge memstore") - for node.entry == nil { - - aidx := uint(0) - av := node.access[aidx] - - for aidx < node.width/2-1 { - if av == node.access[aidx*2+1] { - node.access[aidx] = node.access[aidx*2+2] - aidx = aidx*2 + 1 - } else if av == node.access[aidx*2+2] { - node.access[aidx] = node.access[aidx*2+1] - aidx = aidx*2 + 2 - } else { - panic(nil) - } - } - pidx := aidx*2 + 2 - node.width - if (node.subtree[pidx] != nil) && (av == node.subtree[pidx].access[0]) { - if node.subtree[pidx+1] != nil { - node.access[aidx] = node.subtree[pidx+1].access[0] - } else { - node.access[aidx] = 0 - } - } else if (node.subtree[pidx+1] != nil) && (av == node.subtree[pidx+1].access[0]) { - if node.subtree[pidx] != nil { - node.access[aidx] = node.subtree[pidx].access[0] - } else { - node.access[aidx] = 0 - } - pidx++ - } else { - panic(nil) - } - - //fmt.Println(pidx) - node = node.subtree[pidx] - - } - - log.Trace(fmt.Sprintf("Memstore Clean: Waiting for chunk %v to be saved", node.entry.Key.Log())) - <-node.entry.dbStored - log.Trace(fmt.Sprintf("Memstore Clean: Chunk %v saved to DBStore. Ready to clear from mem.", node.entry.Key.Log())) - - if node.entry.ReqC == nil { - node.entry = nil - s.entryCnt-- - } else { - return - } - - node.access[0] = 0 - - //--- - - aidx := uint(0) - for { - aa := node.access[aidx] - if aidx > 0 { - aidx = (aidx - 1) >> 1 - } else { - pidx := node.parentIdx - node = node.parent - if node == nil { - return - } - aidx = (node.width + pidx - 2) >> 1 - } - if (aa != 0) && ((aa < node.access[aidx]) || (node.access[aidx] == 0)) { - node.access[aidx] = aa - } - } } // Close memstore From 397c6cde1e2fd3636024cad5d23d5e06796772dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Tue, 23 Jan 2018 11:53:09 +0100 Subject: [PATCH 077/210] p2p/discv5: fix topic register panic at shutdown (#15946) --- p2p/discv5/ticket.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/discv5/ticket.go b/p2p/discv5/ticket.go index 023c5000d2..1ecef37e40 100644 --- a/p2p/discv5/ticket.go +++ b/p2p/discv5/ticket.go @@ -350,7 +350,7 @@ func (s *ticketStore) nextFilteredTicket() (*ticketRef, time.Duration) { regTime := now + mclock.AbsTime(wait) topic := ticket.t.topics[ticket.idx] - if regTime >= s.tickets[topic].nextReg { + if s.tickets[topic] != nil && regTime >= s.tickets[topic].nextReg { return ticket, wait } s.removeTicketRef(*ticket) From ec96216d1696bca2671bb7d043ba6af02c20738d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Tue, 23 Jan 2018 12:10:49 +0100 Subject: [PATCH 078/210] Chain indexer fix + new CHT (#15934) * core, light: fix chain indexer bug * light: add new CHT --- light/lightchain.go | 2 +- light/postprocess.go | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/light/lightchain.go b/light/lightchain.go index 03c7c1f0d0..0d97ce1a23 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -393,7 +393,7 @@ func (self *LightChain) InsertHeaderChain(chain []*types.Header, checkFreq int) return err } i, err := self.hc.InsertHeaderChain(chain, whFunc, start) - go self.postChainEvents(events) + self.postChainEvents(events) return i, err } diff --git a/light/postprocess.go b/light/postprocess.go index e7e513880f..32dbc102be 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -53,18 +53,18 @@ type trustedCheckpoint struct { var ( mainnetCheckpoint = trustedCheckpoint{ name: "ETH mainnet", - sectionIdx: 129, - sectionHead: common.HexToHash("64100587c8ec9a76870056d07cb0f58622552d16de6253a59cac4b580c899501"), - chtRoot: common.HexToHash("bb4fb4076cbe6923c8a8ce8f158452bbe19564959313466989fda095a60884ca"), - bloomTrieRoot: common.HexToHash("0db524b2c4a2a9520a42fd842b02d2e8fb58ff37c75cf57bd0eb82daeace6716"), + sectionIdx: 150, + sectionHead: common.HexToHash("1e2e67f289565cbe7bd4367f7960dbd73a3f7c53439e1047cd7ba331c8109e39"), + chtRoot: common.HexToHash("f2a6c9ca143d647b44523cc249f1072c8912358ab873a77a5fdc792b8df99e80"), + bloomTrieRoot: common.HexToHash("c018952fa1513c97857e79fbb9a37acaf8432d5b85e52a78eca7dff5fd5900ee"), } ropstenCheckpoint = trustedCheckpoint{ name: "Ropsten testnet", - sectionIdx: 50, - sectionHead: common.HexToHash("00bd65923a1aa67f85e6b4ae67835784dd54be165c37f056691723c55bf016bd"), - chtRoot: common.HexToHash("6f56dc61936752cc1f8c84b4addabdbe6a1c19693de3f21cb818362df2117f03"), - bloomTrieRoot: common.HexToHash("aca7d7c504d22737242effc3fdc604a762a0af9ced898036b5986c3a15220208"), + sectionIdx: 75, + sectionHead: common.HexToHash("12e68324f4578ea3e8e7fb3968167686729396c9279287fa1f1a8b51bb2d05b4"), + chtRoot: common.HexToHash("3e51dc095c69fa654a4cac766e0afff7357515b4b3c3a379c675f810363e54be"), + bloomTrieRoot: common.HexToHash("33e3a70b33c1d73aa698d496a80615e98ed31fa8f56969876180553b32333339"), } ) From ebec92e928d40d6baccfdab81212dfd59ffe58bc Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Tue, 23 Jan 2018 14:10:34 +0100 Subject: [PATCH 079/210] swarm: an attempt to debug syncer tests --- swarm/network/stream/delivery.go | 28 ++++++++++---- swarm/network/stream/delivery_test.go | 11 +++++- swarm/network/stream/messages.go | 3 ++ swarm/network/stream/syncer.go | 4 +- swarm/network/stream/syncer_test.go | 50 +++++++++++++++++++------ swarm/network/stream/testing/testing.go | 5 +-- swarm/storage/localstore.go | 1 - 7 files changed, 76 insertions(+), 26 deletions(-) diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go index 4355997fd0..606be1703a 100644 --- a/swarm/network/stream/delivery.go +++ b/swarm/network/stream/delivery.go @@ -33,11 +33,14 @@ const ( ) type Delivery struct { - db *storage.DBAPI - overlay network.Overlay - receiveC chan *ChunkDeliveryMsg - getPeer func(discover.NodeID) *Peer - quit chan struct{} + db *storage.DBAPI + overlay network.Overlay + receiveC chan *ChunkDeliveryMsg + getPeer func(discover.NodeID) *Peer + quit chan struct{} + counterIn int + counterDone int + counterHash int } func NewDelivery(overlay network.Overlay, db *storage.DBAPI) *Delivery { @@ -159,6 +162,7 @@ type ChunkDeliveryMsg struct { } func (d *Delivery) handleChunkDeliveryMsg(req *ChunkDeliveryMsg) error { + d.counterIn++ d.receiveC <- req return nil } @@ -182,10 +186,11 @@ R: chunk.SData = req.SData d.db.Put(chunk) log.Warn("reecived delivery", "hash", chunk.Key) - chunk.WaitToStore() - log.Warn("received delivery stored", "hash", chunk.Key) close(chunk.ReqC) + chunk.WaitToStore() + //log.Warn("received delivery stored", "hash", chunk.Key) log.Warn("received delivery requesters notified", "hash", chunk.Key) + d.counterDone++ } } @@ -220,3 +225,12 @@ func (d *Delivery) RequestFromPeers(hash []byte, skipCheck bool, peersToSkip ... } return errors.New("no peer found") } + +func (d *Delivery) PrintCounters(id discover.NodeID) { + if d.counterHash != d.counterDone { + log.Error(fmt.Sprintf("delivery %s: HASH and DONE not the same", id)) + } + log.Error(fmt.Sprintf("delivery %s chunks hash: %d", id, d.counterHash)) + log.Error(fmt.Sprintf("delivery %s chunks in: %d", id, d.counterIn)) + log.Error(fmt.Sprintf("delivery %s chunks done: %d", id, d.counterDone)) +} diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index 1ca89ea5e6..904830535d 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -449,7 +449,10 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck }, } startedAt := time.Now() - result, err := sim.Run(conf) + timeout := 300 * time.Second + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + result, err := sim.Run(ctx, conf) finishedAt := time.Now() if err != nil { t.Fatalf("Setting up simulation failed: %v", err) @@ -586,7 +589,11 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skip // run the simulation in the background errc := make(chan error) go func() { - _, err := sim.Run(conf) + timeout := 300 * time.Second + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + _, err := sim.Run(ctx, conf) errc <- err }() diff --git a/swarm/network/stream/messages.go b/swarm/network/stream/messages.go index 1cf9aeab43..24a63391d8 100644 --- a/swarm/network/stream/messages.go +++ b/swarm/network/stream/messages.go @@ -121,6 +121,9 @@ func (p *Peer) handleOfferedHashesMsg(req *OfferedHashesMsg) error { wg := sync.WaitGroup{} for i := 0; i < len(hashes); i += HashSize { hash := hashes[i : i+HashSize] + + p.streamer.delivery.counterHash++ + if wait := s.NeedData(hash); wait != nil { want.Set(i/HashSize, true) wg.Add(1) diff --git a/swarm/network/stream/syncer.go b/swarm/network/stream/syncer.go index eff42b4cd0..9df6ea78d5 100644 --- a/swarm/network/stream/syncer.go +++ b/swarm/network/stream/syncer.go @@ -75,7 +75,9 @@ func RegisterSwarmSyncerServer(streamer *Registry, db *storage.DBAPI) { // GetSection retrieves the actual chunk from localstore func (s *SwarmSyncerServer) GetData(key []byte) []byte { chunk, err := s.db.Get(storage.Key(key)) - if err != nil { + if err == storage.ErrFetching { + <-chunk.ReqC + } else if err != nil { return nil } return chunk.SData diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index d57be133cb..57ec1dd59b 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -34,7 +34,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/storage" ) -const dataChunkCount = 500 +const dataChunkCount = 1000 func TestSyncerSimulation(t *testing.T) { // testSyncBetweenNodes(t, 2, 1, dataChunkCount, true, 1) @@ -67,6 +67,16 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck if err != nil { t.Fatal(err.Error()) } + + defer func() { + for _, id := range sim.IDs { + deliveries[id].PrintCounters(id) + } + // for id, delivery := range deliveries { + // delivery.PrintCounters(id) + // } + }() + stores = make(map[discover.NodeID]storage.ChunkStore) deliveries = make(map[discover.NodeID]*Delivery) for i, id := range sim.IDs { @@ -91,14 +101,16 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck } // collect hashes in po 1 from all nodes - var hashes []storage.Key + hashes := make([][]storage.Key, nodes) dbs := make([]*storage.DBAPI, nodes) for i := 0; i < nodes; i++ { dbs[i] = storage.NewDBAPI(sim.Stores[i].(*storage.LocalStore)) } + totalHashes := 0 for i := 1; i < nodes; i++ { dbs[i].Iterator(0, math.MaxUint64, po, func(key storage.Key, index uint64) bool { - hashes = append(hashes, key) + hashes[i] = append(hashes[i], key) + totalHashes++ return true }) } @@ -149,15 +161,28 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck } var found int - total := len(hashes) - for _, key := range hashes { - _, err := dbs[0].Get(key) - if err == nil { - found++ + for i, n := range hashes { + for _, key := range n { + chunk, err := dbs[0].Get(key) + if err == storage.ErrFetching { + <-chunk.ReqC + found++ + } else if err == nil { + found++ + } + + log.Error("staring dbs check", "key", key) + for j := i; j > 0; j-- { + _, err := dbs[j].Get(key) + if err != nil { + log.Error("get from node", "node", sim.IDs[j], "nodeID", j, "key", key.Hex(), "err", err) + break + } + } } } - log.Error("sync check", "bin", po, "found", found, "total", total) - pass := found == total + log.Error("sync check", "bin", po, "found", found, "total", totalHashes) + pass := found == totalHashes if !pass { return false, nil } @@ -175,7 +200,10 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck }, } startedAt := time.Now() - result, err := sim.Run(conf) + timeout := 4 * time.Second + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + result, err := sim.Run(ctx, conf) finishedAt := time.Now() if err != nil { t.Fatalf("Setting up simulation failed: %v", err) diff --git a/swarm/network/stream/testing/testing.go b/swarm/network/stream/testing/testing.go index e92b009f86..e1c50919e7 100644 --- a/swarm/network/stream/testing/testing.go +++ b/swarm/network/stream/testing/testing.go @@ -170,7 +170,7 @@ func NewSimulation(conf *RunConfig) (*Simulation, func(), error) { return s, teardown, nil } -func (s *Simulation) Run(conf *RunConfig) (*simulations.StepResult, error) { +func (s *Simulation) Run(ctx context.Context, conf *RunConfig) (*simulations.StepResult, error) { // bring up nodes, launch the servive nodes := conf.NodeCount conns := conf.ConnLevel @@ -204,9 +204,6 @@ func (s *Simulation) Run(conf *RunConfig) (*simulations.StepResult, error) { // create an only locally retrieving dpa for the pivot node to test // if retriee requests have arrived - timeout := 300 * time.Second - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() result := simulations.NewSimulation(s.Net).Run(ctx, conf.Step) return result, nil } diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index ac6d642950..066a27028a 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -106,7 +106,6 @@ func (self *LocalStore) Put(chunk *Chunk) { // ChunkStores are remote and can have long latency func (self *LocalStore) Get(key Key) (chunk *Chunk, err error) { chunk, err = self.memStore.Get(key) - if err == nil { if chunk.ReqC != nil { select { From 97e4497c49bb0c40ae9949bc1d3decffc8115f75 Mon Sep 17 00:00:00 2001 From: zelig Date: Tue, 23 Jan 2018 15:06:37 +0100 Subject: [PATCH 080/210] debug --- swarm/network/stream/syncer_test.go | 50 +++++++++++++++++------------ 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index 57ec1dd59b..72f9b60568 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -107,10 +107,15 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck dbs[i] = storage.NewDBAPI(sim.Stores[i].(*storage.LocalStore)) } totalHashes := 0 - for i := 1; i < nodes; i++ { + hashCounts := make([]int, nodes) + for i := nodes - 1; i >= 0; i-- { + if i < nodes-1 { + hashCounts[i] = hashCounts[i+1] + } dbs[i].Iterator(0, math.MaxUint64, po, func(key storage.Key, index uint64) bool { hashes[i] = append(hashes[i], key) totalHashes++ + hashCounts[i]++ return true }) } @@ -160,29 +165,34 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck default: } - var found int - for i, n := range hashes { - for _, key := range n { - chunk, err := dbs[0].Get(key) - if err == storage.ErrFetching { - <-chunk.ReqC - found++ - } else if err == nil { - found++ - } - - log.Error("staring dbs check", "key", key) - for j := i; j > 0; j-- { - _, err := dbs[j].Get(key) - if err != nil { - log.Error("get from node", "node", sim.IDs[j], "nodeID", j, "key", key.Hex(), "err", err) - break + var pass bool + var i int + log.Error("staring dbs check") + for i = nodes - 1; i >= 0; i-- { + nodeHashCount := hashCounts[i] + nodeHashFound := 0 + for j := i; j < nodes; j++ { + nodeHashes := hashes[j] + for _, key := range nodeHashes { + chunk, err := dbs[i].Get(key) + if err == storage.ErrFetching { + <-chunk.ReqC + nodeHashFound++ + } else if err == nil { + nodeHashFound++ + } else { + log.Error("not found", "index", i, "origin", j, "key", key.Hex(), "err", err) } } } + log.Error("sync check", "node", sim.IDs[i], "index", i, "bin", po, "found", nodeHashFound, "total", nodeHashCount) + pass = nodeHashFound == nodeHashCount + if !pass { + break + } } - log.Error("sync check", "bin", po, "found", found, "total", totalHashes) - pass := found == totalHashes + // log.Error("sync check", "bin", po, "found", found, "total", totalHashes) + // pass := found == totalHashes if !pass { return false, nil } From 172eb9e14d16669d71386307d246cf8d6bfcd680 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Tue, 23 Jan 2018 17:50:32 +0100 Subject: [PATCH 081/210] swarm/storage, swarm/network: Fix delivery tests --- swarm/network/stream/common_test.go | 7 ++++--- swarm/storage/localstore.go | 1 + swarm/storage/netstore.go | 27 +++++++++++++++++++-------- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go index 7997e977e9..1deb6ffba9 100644 --- a/swarm/network/stream/common_test.go +++ b/swarm/network/stream/common_test.go @@ -63,11 +63,12 @@ func NewStreamerService(ctx *adapters.ServiceContext) (node.Service, error) { id := ctx.Config.ID addr := toAddr(id) kad := network.NewKademlia(addr.Over(), network.NewKadParams()) - store := stores[id] - db := storage.NewDBAPI(store.(*storage.LocalStore)) + store := stores[id].(*storage.LocalStore) + db := storage.NewDBAPI(store) delivery := NewDelivery(kad, db) deliveries[id] = delivery - r := NewRegistry(addr, delivery, store, defaultSkipCheck) + netStore := storage.NewNetStore(store, nil) + r := NewRegistry(addr, delivery, netStore, defaultSkipCheck) RegisterSwarmSyncerServer(r, db) RegisterSwarmSyncerClient(r, db) go func() { diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index 066a27028a..898e7f18ea 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -94,6 +94,7 @@ func NewTestLocalStoreForAddr(path string, basekey []byte) (*LocalStore, error) // LocalStore is itself a chunk store // unsafe, in that the data is not integrity checked func (self *LocalStore) Put(chunk *Chunk) { + chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) self.memStore.Put(chunk) go func() { self.DbStore.Put(chunk) diff --git a/swarm/storage/netstore.go b/swarm/storage/netstore.go index 265baa5337..bbf702fdc8 100644 --- a/swarm/storage/netstore.go +++ b/swarm/storage/netstore.go @@ -36,17 +36,28 @@ func NewNetStore(localStore *LocalStore, retrieve func(chunk *Chunk) error) *Net // Get is the entrypoint for local retrieve requests // waits for response or times out func (self *NetStore) Get(key Key) (chunk *Chunk, err error) { - var created bool - chunk, created = self.localStore.GetOrCreateRequest(key) - if chunk.ReqC == nil { - return chunk, nil - } - - if created { - if err := self.retrieve(chunk); err != nil { + if self.retrieve == nil { + chunk, err = self.localStore.Get(key) + if err == nil { + return chunk, nil + } + if err != ErrFetching { return nil, err } + } else { + var created bool + chunk, created = self.localStore.GetOrCreateRequest(key) + if chunk.ReqC == nil { + return chunk, nil + } + + if created { + if err := self.retrieve(chunk); err != nil { + return nil, err + } + } } + t := time.NewTicker(searchTimeout) defer t.Stop() From 05ade19302357eba6a24348f31df140ce0eca326 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kurk=C3=B3=20Mih=C3=A1ly?= Date: Tue, 23 Jan 2018 22:51:04 +0200 Subject: [PATCH 082/210] dashboard: CPU, memory, diskIO and traffic on the footer (#15950) * dashboard: footer, deep state update * dashboard: resolve asset path * dashboard: prevent state update on every reconnection * dashboard: fix linter issue * dashboard, cmd: minor UI fix, include commit hash * dashboard: gitCommit renamed to commit * dashboard: move the geth version to the right, make commit optional * dashboard: memory, traffic and CPU on footer * dashboard: fix merge * dashboard: CPU, diskIO on footer * dashboard: rename variables, use group declaration * dashboard: docs --- dashboard/assets.go | 5392 +++++++++-------- dashboard/assets/.eslintrc | 5 +- .../{components/Common.jsx => common.jsx} | 6 + dashboard/assets/components/Body.jsx | 19 +- .../{ChartGrid.jsx => ChartRow.jsx} | 46 +- dashboard/assets/components/CustomTooltip.jsx | 95 + dashboard/assets/components/Dashboard.jsx | 109 +- dashboard/assets/components/Footer.jsx | 165 +- dashboard/assets/components/Header.jsx | 62 +- dashboard/assets/components/Home.jsx | 77 - dashboard/assets/components/Main.jsx | 39 +- dashboard/assets/components/SideBar.jsx | 84 +- dashboard/assets/index.jsx | 22 +- dashboard/assets/package-lock.json | 566 +- dashboard/assets/package.json | 11 +- dashboard/assets/types/content.jsx | 10 +- dashboard/config.go | 2 +- dashboard/cpu.go | 35 + dashboard/cpu_windows.go | 23 + dashboard/dashboard.go | 146 +- dashboard/message.go | 10 +- vendor/github.com/StackExchange/wmi/LICENSE | 20 + vendor/github.com/StackExchange/wmi/README.md | 6 + .../StackExchange/wmi/swbemservices.go | 260 + vendor/github.com/StackExchange/wmi/wmi.go | 486 ++ .../github.com/elastic/gosigar/CHANGELOG.md | 102 + vendor/github.com/elastic/gosigar/LICENSE | 201 + vendor/github.com/elastic/gosigar/NOTICE | 9 + vendor/github.com/elastic/gosigar/README.md | 57 + vendor/github.com/elastic/gosigar/Vagrantfile | 25 + vendor/github.com/elastic/gosigar/codecov.yml | 21 + .../elastic/gosigar/concrete_sigar.go | 83 + .../elastic/gosigar/sigar_darwin.go | 494 ++ .../elastic/gosigar/sigar_format.go | 126 + .../elastic/gosigar/sigar_freebsd.go | 108 + .../elastic/gosigar/sigar_interface.go | 197 + .../github.com/elastic/gosigar/sigar_linux.go | 84 + .../elastic/gosigar/sigar_linux_common.go | 468 ++ .../elastic/gosigar/sigar_openbsd.go | 418 ++ .../github.com/elastic/gosigar/sigar_stub.go | 71 + .../github.com/elastic/gosigar/sigar_unix.go | 69 + .../github.com/elastic/gosigar/sigar_util.go | 22 + .../elastic/gosigar/sigar_windows.go | 437 ++ .../elastic/gosigar/sys/windows/doc.go | 2 + .../elastic/gosigar/sys/windows/ntquery.go | 132 + .../elastic/gosigar/sys/windows/privileges.go | 272 + .../gosigar/sys/windows/syscall_windows.go | 385 ++ .../elastic/gosigar/sys/windows/version.go | 43 + .../gosigar/sys/windows/zsyscall_windows.go | 260 + vendor/github.com/go-ole/go-ole/ChangeLog.md | 49 + vendor/github.com/go-ole/go-ole/LICENSE | 21 + vendor/github.com/go-ole/go-ole/README.md | 46 + vendor/github.com/go-ole/go-ole/appveyor.yml | 54 + vendor/github.com/go-ole/go-ole/com.go | 329 + vendor/github.com/go-ole/go-ole/com_func.go | 174 + vendor/github.com/go-ole/go-ole/connect.go | 192 + vendor/github.com/go-ole/go-ole/constants.go | 153 + vendor/github.com/go-ole/go-ole/error.go | 51 + vendor/github.com/go-ole/go-ole/error_func.go | 8 + .../github.com/go-ole/go-ole/error_windows.go | 24 + vendor/github.com/go-ole/go-ole/guid.go | 284 + .../go-ole/go-ole/iconnectionpoint.go | 20 + .../go-ole/go-ole/iconnectionpoint_func.go | 21 + .../go-ole/go-ole/iconnectionpoint_windows.go | 43 + .../go-ole/iconnectionpointcontainer.go | 17 + .../go-ole/iconnectionpointcontainer_func.go | 11 + .../iconnectionpointcontainer_windows.go | 25 + vendor/github.com/go-ole/go-ole/idispatch.go | 94 + .../go-ole/go-ole/idispatch_func.go | 19 + .../go-ole/go-ole/idispatch_windows.go | 197 + .../github.com/go-ole/go-ole/ienumvariant.go | 19 + .../go-ole/go-ole/ienumvariant_func.go | 19 + .../go-ole/go-ole/ienumvariant_windows.go | 63 + .../github.com/go-ole/go-ole/iinspectable.go | 18 + .../go-ole/go-ole/iinspectable_func.go | 15 + .../go-ole/go-ole/iinspectable_windows.go | 72 + .../go-ole/go-ole/iprovideclassinfo.go | 21 + .../go-ole/go-ole/iprovideclassinfo_func.go | 7 + .../go-ole/iprovideclassinfo_windows.go | 21 + vendor/github.com/go-ole/go-ole/itypeinfo.go | 34 + .../go-ole/go-ole/itypeinfo_func.go | 7 + .../go-ole/go-ole/itypeinfo_windows.go | 21 + vendor/github.com/go-ole/go-ole/iunknown.go | 57 + .../github.com/go-ole/go-ole/iunknown_func.go | 19 + .../go-ole/go-ole/iunknown_windows.go | 58 + vendor/github.com/go-ole/go-ole/ole.go | 157 + .../go-ole/go-ole/oleutil/connection.go | 100 + .../go-ole/go-ole/oleutil/connection_func.go | 10 + .../go-ole/oleutil/connection_windows.go | 58 + .../go-ole/go-ole/oleutil/go-get.go | 6 + .../go-ole/go-ole/oleutil/oleutil.go | 127 + vendor/github.com/go-ole/go-ole/safearray.go | 27 + .../go-ole/go-ole/safearray_func.go | 211 + .../go-ole/go-ole/safearray_windows.go | 337 ++ .../go-ole/go-ole/safearrayconversion.go | 140 + .../go-ole/go-ole/safearrayslices.go | 33 + vendor/github.com/go-ole/go-ole/utility.go | 101 + vendor/github.com/go-ole/go-ole/variables.go | 16 + vendor/github.com/go-ole/go-ole/variant.go | 105 + .../github.com/go-ole/go-ole/variant_386.go | 11 + .../github.com/go-ole/go-ole/variant_amd64.go | 12 + .../github.com/go-ole/go-ole/variant_s390x.go | 12 + vendor/github.com/go-ole/go-ole/vt_string.go | 58 + vendor/github.com/go-ole/go-ole/winrt.go | 99 + vendor/github.com/go-ole/go-ole/winrt_doc.go | 36 + vendor/github.com/pkg/errors/LICENSE | 23 + vendor/github.com/pkg/errors/README.md | 52 + vendor/github.com/pkg/errors/appveyor.yml | 32 + vendor/github.com/pkg/errors/errors.go | 269 + vendor/github.com/pkg/errors/stack.go | 187 + vendor/vendor.json | 36 + 111 files changed, 13162 insertions(+), 3158 deletions(-) rename dashboard/assets/{components/Common.jsx => common.jsx} (95%) rename dashboard/assets/components/{ChartGrid.jsx => ChartRow.jsx} (55%) create mode 100644 dashboard/assets/components/CustomTooltip.jsx delete mode 100644 dashboard/assets/components/Home.jsx create mode 100644 dashboard/cpu.go create mode 100644 dashboard/cpu_windows.go create mode 100644 vendor/github.com/StackExchange/wmi/LICENSE create mode 100644 vendor/github.com/StackExchange/wmi/README.md create mode 100644 vendor/github.com/StackExchange/wmi/swbemservices.go create mode 100644 vendor/github.com/StackExchange/wmi/wmi.go create mode 100644 vendor/github.com/elastic/gosigar/CHANGELOG.md create mode 100644 vendor/github.com/elastic/gosigar/LICENSE create mode 100644 vendor/github.com/elastic/gosigar/NOTICE create mode 100644 vendor/github.com/elastic/gosigar/README.md create mode 100644 vendor/github.com/elastic/gosigar/Vagrantfile create mode 100644 vendor/github.com/elastic/gosigar/codecov.yml create mode 100644 vendor/github.com/elastic/gosigar/concrete_sigar.go create mode 100644 vendor/github.com/elastic/gosigar/sigar_darwin.go create mode 100644 vendor/github.com/elastic/gosigar/sigar_format.go create mode 100644 vendor/github.com/elastic/gosigar/sigar_freebsd.go create mode 100644 vendor/github.com/elastic/gosigar/sigar_interface.go create mode 100644 vendor/github.com/elastic/gosigar/sigar_linux.go create mode 100644 vendor/github.com/elastic/gosigar/sigar_linux_common.go create mode 100644 vendor/github.com/elastic/gosigar/sigar_openbsd.go create mode 100644 vendor/github.com/elastic/gosigar/sigar_stub.go create mode 100644 vendor/github.com/elastic/gosigar/sigar_unix.go create mode 100644 vendor/github.com/elastic/gosigar/sigar_util.go create mode 100644 vendor/github.com/elastic/gosigar/sigar_windows.go create mode 100644 vendor/github.com/elastic/gosigar/sys/windows/doc.go create mode 100644 vendor/github.com/elastic/gosigar/sys/windows/ntquery.go create mode 100644 vendor/github.com/elastic/gosigar/sys/windows/privileges.go create mode 100644 vendor/github.com/elastic/gosigar/sys/windows/syscall_windows.go create mode 100644 vendor/github.com/elastic/gosigar/sys/windows/version.go create mode 100644 vendor/github.com/elastic/gosigar/sys/windows/zsyscall_windows.go create mode 100644 vendor/github.com/go-ole/go-ole/ChangeLog.md create mode 100644 vendor/github.com/go-ole/go-ole/LICENSE create mode 100644 vendor/github.com/go-ole/go-ole/README.md create mode 100644 vendor/github.com/go-ole/go-ole/appveyor.yml create mode 100644 vendor/github.com/go-ole/go-ole/com.go create mode 100644 vendor/github.com/go-ole/go-ole/com_func.go create mode 100644 vendor/github.com/go-ole/go-ole/connect.go create mode 100644 vendor/github.com/go-ole/go-ole/constants.go create mode 100644 vendor/github.com/go-ole/go-ole/error.go create mode 100644 vendor/github.com/go-ole/go-ole/error_func.go create mode 100644 vendor/github.com/go-ole/go-ole/error_windows.go create mode 100644 vendor/github.com/go-ole/go-ole/guid.go create mode 100644 vendor/github.com/go-ole/go-ole/iconnectionpoint.go create mode 100644 vendor/github.com/go-ole/go-ole/iconnectionpoint_func.go create mode 100644 vendor/github.com/go-ole/go-ole/iconnectionpoint_windows.go create mode 100644 vendor/github.com/go-ole/go-ole/iconnectionpointcontainer.go create mode 100644 vendor/github.com/go-ole/go-ole/iconnectionpointcontainer_func.go create mode 100644 vendor/github.com/go-ole/go-ole/iconnectionpointcontainer_windows.go create mode 100644 vendor/github.com/go-ole/go-ole/idispatch.go create mode 100644 vendor/github.com/go-ole/go-ole/idispatch_func.go create mode 100644 vendor/github.com/go-ole/go-ole/idispatch_windows.go create mode 100644 vendor/github.com/go-ole/go-ole/ienumvariant.go create mode 100644 vendor/github.com/go-ole/go-ole/ienumvariant_func.go create mode 100644 vendor/github.com/go-ole/go-ole/ienumvariant_windows.go create mode 100644 vendor/github.com/go-ole/go-ole/iinspectable.go create mode 100644 vendor/github.com/go-ole/go-ole/iinspectable_func.go create mode 100644 vendor/github.com/go-ole/go-ole/iinspectable_windows.go create mode 100644 vendor/github.com/go-ole/go-ole/iprovideclassinfo.go create mode 100644 vendor/github.com/go-ole/go-ole/iprovideclassinfo_func.go create mode 100644 vendor/github.com/go-ole/go-ole/iprovideclassinfo_windows.go create mode 100644 vendor/github.com/go-ole/go-ole/itypeinfo.go create mode 100644 vendor/github.com/go-ole/go-ole/itypeinfo_func.go create mode 100644 vendor/github.com/go-ole/go-ole/itypeinfo_windows.go create mode 100644 vendor/github.com/go-ole/go-ole/iunknown.go create mode 100644 vendor/github.com/go-ole/go-ole/iunknown_func.go create mode 100644 vendor/github.com/go-ole/go-ole/iunknown_windows.go create mode 100644 vendor/github.com/go-ole/go-ole/ole.go create mode 100644 vendor/github.com/go-ole/go-ole/oleutil/connection.go create mode 100644 vendor/github.com/go-ole/go-ole/oleutil/connection_func.go create mode 100644 vendor/github.com/go-ole/go-ole/oleutil/connection_windows.go create mode 100644 vendor/github.com/go-ole/go-ole/oleutil/go-get.go create mode 100644 vendor/github.com/go-ole/go-ole/oleutil/oleutil.go create mode 100644 vendor/github.com/go-ole/go-ole/safearray.go create mode 100644 vendor/github.com/go-ole/go-ole/safearray_func.go create mode 100644 vendor/github.com/go-ole/go-ole/safearray_windows.go create mode 100644 vendor/github.com/go-ole/go-ole/safearrayconversion.go create mode 100644 vendor/github.com/go-ole/go-ole/safearrayslices.go create mode 100644 vendor/github.com/go-ole/go-ole/utility.go create mode 100644 vendor/github.com/go-ole/go-ole/variables.go create mode 100644 vendor/github.com/go-ole/go-ole/variant.go create mode 100644 vendor/github.com/go-ole/go-ole/variant_386.go create mode 100644 vendor/github.com/go-ole/go-ole/variant_amd64.go create mode 100644 vendor/github.com/go-ole/go-ole/variant_s390x.go create mode 100644 vendor/github.com/go-ole/go-ole/vt_string.go create mode 100644 vendor/github.com/go-ole/go-ole/winrt.go create mode 100644 vendor/github.com/go-ole/go-ole/winrt_doc.go create mode 100644 vendor/github.com/pkg/errors/LICENSE create mode 100644 vendor/github.com/pkg/errors/README.md create mode 100644 vendor/github.com/pkg/errors/appveyor.yml create mode 100644 vendor/github.com/pkg/errors/errors.go create mode 100644 vendor/github.com/pkg/errors/stack.go diff --git a/dashboard/assets.go b/dashboard/assets.go index b2c1203234..8337cf080d 100644 --- a/dashboard/assets.go +++ b/dashboard/assets.go @@ -6,6 +6,7 @@ package dashboard import ( + "crypto/sha256" "fmt" "io/ioutil" "os" @@ -15,8 +16,9 @@ import ( ) type asset struct { - bytes []byte - info os.FileInfo + bytes []byte + info os.FileInfo + digest [sha256.Size]byte } type bindataFileInfo struct { @@ -82,7 +84,7 @@ func dashboardHtml() (*asset, error) { } info := bindataFileInfo{name: "dashboard.html", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x6b, 0xd9, 0xa6, 0xeb, 0x32, 0x49, 0x9b, 0xe5, 0x3a, 0xcb, 0x99, 0xd3, 0xb6, 0x69, 0x7f, 0xde, 0x35, 0x9d, 0x5, 0x96, 0x84, 0xc0, 0x14, 0xef, 0xbe, 0x58, 0x10, 0x5e, 0x40, 0xf2, 0x12, 0x97}} return a, nil } @@ -114,11 +116,11 @@ var _bundleJs = []byte((((((((((`!function(modules) { return __webpack_require__.d(getter, "a", getter), getter; }, __webpack_require__.o = function(object, property) { return Object.prototype.hasOwnProperty.call(object, property); - }, __webpack_require__.p = "", __webpack_require__(__webpack_require__.s = 336); + }, __webpack_require__.p = "", __webpack_require__(__webpack_require__.s = 331); }([ function(module, exports, __webpack_require__) { "use strict"; (function(process) { - "production" === process.env.NODE_ENV ? module.exports = __webpack_require__(337) : module.exports = __webpack_require__(338); + "production" === process.env.NODE_ENV ? module.exports = __webpack_require__(332) : module.exports = __webpack_require__(333); }).call(exports, __webpack_require__(2)); }, function(module, exports, __webpack_require__) { (function(process) { @@ -126,8 +128,8 @@ var _bundleJs = []byte((((((((((`!function(modules) { var REACT_ELEMENT_TYPE = "function" == typeof Symbol && Symbol.for && Symbol.for("react.element") || 60103, isValidElement = function(object) { return "object" == typeof object && null !== object && object.$$typeof === REACT_ELEMENT_TYPE; }; - module.exports = __webpack_require__(379)(isValidElement, !0); - } else module.exports = __webpack_require__(380)(); + module.exports = __webpack_require__(374)(isValidElement, !0); + } else module.exports = __webpack_require__(375)(); }).call(exports, __webpack_require__(2)); }, function(module, exports) { function defaultSetTimout() { @@ -247,6 +249,11 @@ var _bundleJs = []byte((((((((((`!function(modules) { } return Array.from(arr); } + function _objectWithoutProperties(obj, keys) { + var target = {}; + for (var i in obj) keys.indexOf(i) >= 0 || Object.prototype.hasOwnProperty.call(obj, i) && (target[i] = obj[i]); + return target; + } __webpack_require__.d(__webpack_exports__, "c", function() { return PRESENTATION_ATTRIBUTES; }), __webpack_require__.d(__webpack_exports__, "a", function() { @@ -282,7 +289,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }), __webpack_require__.d(__webpack_exports__, "o", function() { return parseChildIndex; }); - var __WEBPACK_IMPORTED_MODULE_0_lodash_isNil__ = __webpack_require__(20), __WEBPACK_IMPORTED_MODULE_0_lodash_isNil___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_isNil__), __WEBPACK_IMPORTED_MODULE_1_lodash_isString__ = __webpack_require__(165), __WEBPACK_IMPORTED_MODULE_1_lodash_isString___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_lodash_isString__), __WEBPACK_IMPORTED_MODULE_2_lodash_isObject__ = __webpack_require__(32), __WEBPACK_IMPORTED_MODULE_2_lodash_isObject___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_lodash_isObject__), __WEBPACK_IMPORTED_MODULE_3_lodash_isFunction__ = __webpack_require__(8), __WEBPACK_IMPORTED_MODULE_3_lodash_isFunction___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_lodash_isFunction__), __WEBPACK_IMPORTED_MODULE_4_lodash_isArray__ = __webpack_require__(12), __WEBPACK_IMPORTED_MODULE_4_lodash_isArray___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_lodash_isArray__), __WEBPACK_IMPORTED_MODULE_5_react__ = __webpack_require__(0), __WEBPACK_IMPORTED_MODULE_5_react___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_5_react__), __WEBPACK_IMPORTED_MODULE_6_prop_types__ = __webpack_require__(1), __WEBPACK_IMPORTED_MODULE_6_prop_types___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_6_prop_types__), __WEBPACK_IMPORTED_MODULE_7__DataUtils__ = __webpack_require__(9), __WEBPACK_IMPORTED_MODULE_8__PureRender__ = __webpack_require__(5), PRESENTATION_ATTRIBUTES = { + var __WEBPACK_IMPORTED_MODULE_0_lodash_isNil__ = __webpack_require__(20), __WEBPACK_IMPORTED_MODULE_0_lodash_isNil___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_isNil__), __WEBPACK_IMPORTED_MODULE_1_lodash_isString__ = __webpack_require__(164), __WEBPACK_IMPORTED_MODULE_1_lodash_isString___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_lodash_isString__), __WEBPACK_IMPORTED_MODULE_2_lodash_isObject__ = __webpack_require__(31), __WEBPACK_IMPORTED_MODULE_2_lodash_isObject___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_lodash_isObject__), __WEBPACK_IMPORTED_MODULE_3_lodash_isFunction__ = __webpack_require__(8), __WEBPACK_IMPORTED_MODULE_3_lodash_isFunction___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_lodash_isFunction__), __WEBPACK_IMPORTED_MODULE_4_lodash_isArray__ = __webpack_require__(12), __WEBPACK_IMPORTED_MODULE_4_lodash_isArray___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_lodash_isArray__), __WEBPACK_IMPORTED_MODULE_5_react__ = __webpack_require__(0), __WEBPACK_IMPORTED_MODULE_5_react___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_5_react__), __WEBPACK_IMPORTED_MODULE_6_prop_types__ = __webpack_require__(1), __WEBPACK_IMPORTED_MODULE_6_prop_types___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_6_prop_types__), __WEBPACK_IMPORTED_MODULE_7__DataUtils__ = __webpack_require__(9), __WEBPACK_IMPORTED_MODULE_8__PureRender__ = __webpack_require__(5), PRESENTATION_ATTRIBUTES = { alignmentBaseline: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.string, angle: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.number, baselineShift: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.string, @@ -438,7 +445,12 @@ var _bundleJs = []byte((((((((((`!function(modules) { entry && entry.type && __WEBPACK_IMPORTED_MODULE_1_lodash_isString___default()(entry.type) && SVG_TAGS.indexOf(entry.type) >= 0 && svgElements.push(entry); }), svgElements; }, isSingleChildEqual = function(nextChild, prevChild) { - return !(!__WEBPACK_IMPORTED_MODULE_0_lodash_isNil___default()(nextChild) || !__WEBPACK_IMPORTED_MODULE_0_lodash_isNil___default()(prevChild)) || !__WEBPACK_IMPORTED_MODULE_0_lodash_isNil___default()(nextChild) && !__WEBPACK_IMPORTED_MODULE_0_lodash_isNil___default()(prevChild) && Object(__WEBPACK_IMPORTED_MODULE_8__PureRender__.b)(nextChild.props, prevChild.props); + if (__WEBPACK_IMPORTED_MODULE_0_lodash_isNil___default()(nextChild) && __WEBPACK_IMPORTED_MODULE_0_lodash_isNil___default()(prevChild)) return !0; + if (!__WEBPACK_IMPORTED_MODULE_0_lodash_isNil___default()(nextChild) && !__WEBPACK_IMPORTED_MODULE_0_lodash_isNil___default()(prevChild)) { + var _ref = nextChild.props || {}, nextChildren = _ref.children, nextProps = _objectWithoutProperties(_ref, [ "children" ]), _ref2 = prevChild.props || {}, prevChildren = _ref2.children, prevProps = _objectWithoutProperties(_ref2, [ "children" ]); + return nextChildren && prevChildren ? Object(__WEBPACK_IMPORTED_MODULE_8__PureRender__.b)(nextProps, prevProps) && isChildrenEqual(nextChildren, prevChildren) : !nextChildren && !prevChildren && Object(__WEBPACK_IMPORTED_MODULE_8__PureRender__.b)(nextProps, prevProps); + } + return !1; }, isChildrenEqual = function isChildrenEqual(nextChildren, prevChildren) { if (nextChildren === prevChildren) return !0; if (__WEBPACK_IMPORTED_MODULE_5_react__.Children.count(nextChildren) !== __WEBPACK_IMPORTED_MODULE_5_react__.Children.count(prevChildren)) return !1; @@ -497,7 +509,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, function(module, exports, __webpack_require__) { "use strict"; exports.__esModule = !0; - var _assign = __webpack_require__(206), _assign2 = function(obj) { + var _assign = __webpack_require__(205), _assign2 = function(obj) { return obj && obj.__esModule ? obj : { default: obj }; @@ -515,7 +527,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { var tag = baseGetTag(value); return tag == funcTag || tag == genTag || tag == asyncTag || tag == proxyTag; } - var baseGetTag = __webpack_require__(42), isObject = __webpack_require__(32), asyncTag = "[object AsyncFunction]", funcTag = "[object Function]", genTag = "[object GeneratorFunction]", proxyTag = "[object Proxy]"; + var baseGetTag = __webpack_require__(42), isObject = __webpack_require__(31), asyncTag = "[object AsyncFunction]", funcTag = "[object Function]", genTag = "[object GeneratorFunction]", proxyTag = "[object Proxy]"; module.exports = isFunction; }, function(module, __webpack_exports__, __webpack_require__) { "use strict"; @@ -540,7 +552,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }), __webpack_require__.d(__webpack_exports__, "a", function() { return findEntryInArray; }); - var __WEBPACK_IMPORTED_MODULE_0_lodash_get__ = __webpack_require__(109), __WEBPACK_IMPORTED_MODULE_0_lodash_get___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_get__), __WEBPACK_IMPORTED_MODULE_1_lodash_isArray__ = __webpack_require__(12), __WEBPACK_IMPORTED_MODULE_1_lodash_isArray___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_lodash_isArray__), __WEBPACK_IMPORTED_MODULE_2_lodash_isNaN__ = __webpack_require__(116), __WEBPACK_IMPORTED_MODULE_2_lodash_isNaN___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_lodash_isNaN__), __WEBPACK_IMPORTED_MODULE_3_lodash_isNumber__ = __webpack_require__(170), __WEBPACK_IMPORTED_MODULE_3_lodash_isNumber___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_lodash_isNumber__), __WEBPACK_IMPORTED_MODULE_4_lodash_isString__ = __webpack_require__(165), __WEBPACK_IMPORTED_MODULE_4_lodash_isString___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_lodash_isString__), mathSign = function(value) { + var __WEBPACK_IMPORTED_MODULE_0_lodash_get__ = __webpack_require__(165), __WEBPACK_IMPORTED_MODULE_0_lodash_get___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_get__), __WEBPACK_IMPORTED_MODULE_1_lodash_isArray__ = __webpack_require__(12), __WEBPACK_IMPORTED_MODULE_1_lodash_isArray___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_lodash_isArray__), __WEBPACK_IMPORTED_MODULE_2_lodash_isNaN__ = __webpack_require__(117), __WEBPACK_IMPORTED_MODULE_2_lodash_isNaN___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_lodash_isNaN__), __WEBPACK_IMPORTED_MODULE_3_lodash_isNumber__ = __webpack_require__(170), __WEBPACK_IMPORTED_MODULE_3_lodash_isNumber___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_lodash_isNumber__), __WEBPACK_IMPORTED_MODULE_4_lodash_isString__ = __webpack_require__(164), __WEBPACK_IMPORTED_MODULE_4_lodash_isString___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_lodash_isString__), mathSign = function(value) { return 0 === value ? 0 : value > 0 ? 1 : -1; }, isPercent = function(value) { return __WEBPACK_IMPORTED_MODULE_4_lodash_isString___default()(value) && value.indexOf("%") === value.length - 1; @@ -596,42 +608,38 @@ var _bundleJs = []byte((((((((((`!function(modules) { } Object.defineProperty(exports, "__esModule", { value: !0 - }), exports.sheetsManager = exports.preset = void 0; - var _keys = __webpack_require__(36), _keys2 = _interopRequireDefault(_keys), _extends2 = __webpack_require__(7), _extends3 = _interopRequireDefault(_extends2), _getPrototypeOf = __webpack_require__(26), _getPrototypeOf2 = _interopRequireDefault(_getPrototypeOf), _classCallCheck2 = __webpack_require__(27), _classCallCheck3 = _interopRequireDefault(_classCallCheck2), _createClass2 = __webpack_require__(28), _createClass3 = _interopRequireDefault(_createClass2), _possibleConstructorReturn2 = __webpack_require__(29), _possibleConstructorReturn3 = _interopRequireDefault(_possibleConstructorReturn2), _inherits2 = __webpack_require__(30), _inherits3 = _interopRequireDefault(_inherits2), _objectWithoutProperties2 = __webpack_require__(6), _objectWithoutProperties3 = _interopRequireDefault(_objectWithoutProperties2), _map = __webpack_require__(402), _map2 = _interopRequireDefault(_map), _minSafeInteger = __webpack_require__(418), _minSafeInteger2 = _interopRequireDefault(_minSafeInteger), _react = __webpack_require__(0), _react2 = _interopRequireDefault(_react), _propTypes = __webpack_require__(1), _propTypes2 = _interopRequireDefault(_propTypes), _warning = __webpack_require__(11), _warning2 = _interopRequireDefault(_warning), _hoistNonReactStatics = __webpack_require__(151), _hoistNonReactStatics2 = _interopRequireDefault(_hoistNonReactStatics), _wrapDisplayName = __webpack_require__(74), _wrapDisplayName2 = _interopRequireDefault(_wrapDisplayName), _getDisplayName = __webpack_require__(227), _getDisplayName2 = _interopRequireDefault(_getDisplayName), _contextTypes = __webpack_require__(421), _contextTypes2 = _interopRequireDefault(_contextTypes), _jss = __webpack_require__(229), _jssGlobal = __webpack_require__(444), _jssGlobal2 = _interopRequireDefault(_jssGlobal), _jssNested = __webpack_require__(445), _jssNested2 = _interopRequireDefault(_jssNested), _jssCamelCase = __webpack_require__(446), _jssCamelCase2 = _interopRequireDefault(_jssCamelCase), _jssDefaultUnit = __webpack_require__(447), _jssDefaultUnit2 = _interopRequireDefault(_jssDefaultUnit), _jssVendorPrefixer = __webpack_require__(449), _jssVendorPrefixer2 = _interopRequireDefault(_jssVendorPrefixer), _jssPropsSort = __webpack_require__(454), _jssPropsSort2 = _interopRequireDefault(_jssPropsSort), _ns = __webpack_require__(228), ns = function(obj) { + }), exports.sheetsManager = void 0; + var _keys = __webpack_require__(41), _keys2 = _interopRequireDefault(_keys), _extends2 = __webpack_require__(7), _extends3 = _interopRequireDefault(_extends2), _getPrototypeOf = __webpack_require__(26), _getPrototypeOf2 = _interopRequireDefault(_getPrototypeOf), _classCallCheck2 = __webpack_require__(27), _classCallCheck3 = _interopRequireDefault(_classCallCheck2), _createClass2 = __webpack_require__(28), _createClass3 = _interopRequireDefault(_createClass2), _possibleConstructorReturn2 = __webpack_require__(29), _possibleConstructorReturn3 = _interopRequireDefault(_possibleConstructorReturn2), _inherits2 = __webpack_require__(30), _inherits3 = _interopRequireDefault(_inherits2), _objectWithoutProperties2 = __webpack_require__(6), _objectWithoutProperties3 = _interopRequireDefault(_objectWithoutProperties2), _map = __webpack_require__(397), _map2 = _interopRequireDefault(_map), _minSafeInteger = __webpack_require__(413), _minSafeInteger2 = _interopRequireDefault(_minSafeInteger), _react = __webpack_require__(0), _react2 = _interopRequireDefault(_react), _propTypes = __webpack_require__(1), _propTypes2 = _interopRequireDefault(_propTypes), _warning = __webpack_require__(11), _warning2 = _interopRequireDefault(_warning), _hoistNonReactStatics = __webpack_require__(152), _hoistNonReactStatics2 = _interopRequireDefault(_hoistNonReactStatics), _getDisplayName = __webpack_require__(226), _getDisplayName2 = _interopRequireDefault(_getDisplayName), _wrapDisplayName = __webpack_require__(75), _wrapDisplayName2 = _interopRequireDefault(_wrapDisplayName), _contextTypes = __webpack_require__(416), _contextTypes2 = _interopRequireDefault(_contextTypes), _jss = __webpack_require__(228), _ns = __webpack_require__(227), ns = function(obj) { if (obj && obj.__esModule) return obj; var newObj = {}; if (null != obj) for (var key in obj) Object.prototype.hasOwnProperty.call(obj, key) && (newObj[key] = obj[key]); return newObj.default = obj, newObj; - }(_ns), _createMuiTheme = __webpack_require__(150), _createMuiTheme2 = _interopRequireDefault(_createMuiTheme), _themeListener = __webpack_require__(149), _themeListener2 = _interopRequireDefault(_themeListener), _createGenerateClassName = __webpack_require__(455), _createGenerateClassName2 = _interopRequireDefault(_createGenerateClassName), _getStylesCreator = __webpack_require__(456), _getStylesCreator2 = _interopRequireDefault(_getStylesCreator), preset = exports.preset = function() { - return { - plugins: [ (0, _jssGlobal2.default)(), (0, _jssNested2.default)(), (0, _jssCamelCase2.default)(), (0, - _jssDefaultUnit2.default)(), (0, _jssVendorPrefixer2.default)(), (0, _jssPropsSort2.default)() ] - }; - }, jss = (0, _jss.create)(preset()), generateClassName = (0, _createGenerateClassName2.default)(), indexCounter = _minSafeInteger2.default, sheetsManager = exports.sheetsManager = new _map2.default(), noopTheme = {}, defaultTheme = void 0, withStyles = function(stylesOrCreator) { + }(_ns), _jssPreset = __webpack_require__(439), _jssPreset2 = _interopRequireDefault(_jssPreset), _createMuiTheme = __webpack_require__(151), _createMuiTheme2 = _interopRequireDefault(_createMuiTheme), _themeListener = __webpack_require__(150), _themeListener2 = _interopRequireDefault(_themeListener), _createGenerateClassName = __webpack_require__(451), _createGenerateClassName2 = _interopRequireDefault(_createGenerateClassName), _getStylesCreator = __webpack_require__(452), _getStylesCreator2 = _interopRequireDefault(_getStylesCreator), jss = (0, + _jss.create)((0, _jssPreset2.default)()), generateClassName = (0, _createGenerateClassName2.default)(), indexCounter = _minSafeInteger2.default, sheetsManager = exports.sheetsManager = new _map2.default(), noopTheme = {}, defaultTheme = void 0, withStyles = function(stylesOrCreator) { var options = arguments.length > 1 && void 0 !== arguments[1] ? arguments[1] : {}; return function(Component) { - var _options$withTheme = options.withTheme, withTheme = void 0 !== _options$withTheme && _options$withTheme, flip = options.flip, name = options.name, styleSheetOptions = (0, + var _options$withTheme = options.withTheme, withTheme = void 0 !== _options$withTheme && _options$withTheme, _options$flip = options.flip, flip = void 0 === _options$flip ? null : _options$flip, name = options.name, styleSheetOptions = (0, _objectWithoutProperties3.default)(options, [ "withTheme", "flip", "name" ]), stylesCreator = (0, _getStylesCreator2.default)(stylesOrCreator), listenToTheme = stylesCreator.themingEnabled || withTheme || "string" == typeof name; - void 0 === stylesCreator.options.index && (indexCounter += 1, stylesCreator.options.index = indexCounter), - "production" !== process.env.NODE_ENV && (0, _warning2.default)(indexCounter < 0, [ "Material-UI: you might have a memory leak.", "The indexCounter is not supposed to grow that much." ].join(" ")); - var Style = function(_React$Component) { - function Style(props, context) { - (0, _classCallCheck3.default)(this, Style); - var _this = (0, _possibleConstructorReturn3.default)(this, (Style.__proto__ || (0, - _getPrototypeOf2.default)(Style)).call(this, props, context)); - _this.state = {}, _this.unsubscribeId = null, _this.jss = null, _this.sheetsManager = sheetsManager, - _this.disableStylesGeneration = !1, _this.stylesCreatorSaved = null, _this.theme = null, - _this.sheetOptions = null, _this.theme = null; + indexCounter += 1, stylesCreator.options.index = indexCounter, "production" !== process.env.NODE_ENV && (0, + _warning2.default)(indexCounter < 0, [ "Material-UI: you might have a memory leak.", "The indexCounter is not supposed to grow that much." ].join(" ")); + var WithStyles = function(_React$Component) { + function WithStyles(props, context) { + (0, _classCallCheck3.default)(this, WithStyles); + var _this = (0, _possibleConstructorReturn3.default)(this, (WithStyles.__proto__ || (0, + _getPrototypeOf2.default)(WithStyles)).call(this, props, context)); + _this.state = {}, _this.disableStylesGeneration = !1, _this.jss = null, _this.sheetOptions = null, + _this.sheetsManager = sheetsManager, _this.stylesCreatorSaved = null, _this.theme = null, + _this.unsubscribeId = null, _this.jss = _this.context[ns.jss] || jss; var muiThemeProviderOptions = _this.context.muiThemeProviderOptions; - return _this.jss = _this.context[ns.jss] || jss, muiThemeProviderOptions && (muiThemeProviderOptions.sheetsManager && (_this.sheetsManager = muiThemeProviderOptions.sheetsManager), + return muiThemeProviderOptions && (muiThemeProviderOptions.sheetsManager && (_this.sheetsManager = muiThemeProviderOptions.sheetsManager), _this.disableStylesGeneration = muiThemeProviderOptions.disableStylesGeneration), _this.stylesCreatorSaved = stylesCreator, _this.sheetOptions = (0, _extends3.default)({ generateClassName: generateClassName }, _this.context[ns.sheetOptions]), _this.theme = listenToTheme ? _themeListener2.default.initial(context) || getDefaultTheme() : noopTheme, _this; } - return (0, _inherits3.default)(Style, _React$Component), (0, _createClass3.default)(Style, [ { + return (0, _inherits3.default)(WithStyles, _React$Component), (0, _createClass3.default)(WithStyles, [ { key: "componentWillMount", value: function() { this.attach(this.theme); @@ -669,10 +677,11 @@ var _bundleJs = []byte((((((((((`!function(modules) { refs: 0, sheet: null }, sheetManager.set(theme, sheetManagerTheme)), 0 === sheetManagerTheme.refs) { - var styles = stylesCreatorSaved.create(theme, name), meta = void 0; - "production" !== process.env.NODE_ENV && (meta = name || (0, _getDisplayName2.default)(Component)); + var styles = stylesCreatorSaved.create(theme, name), meta = name; + "production" === process.env.NODE_ENV || meta || (meta = (0, _getDisplayName2.default)(Component)); var sheet = this.jss.createStyleSheet(styles, (0, _extends3.default)({ meta: meta, + classNamePrefix: meta, flip: "boolean" == typeof flip ? flip : "rtl" === theme.direction, link: !1 }, this.sheetOptions, stylesCreatorSaved.options, { @@ -723,17 +732,17 @@ var _bundleJs = []byte((((((((((`!function(modules) { ref: innerRef })); } - } ]), Style; + } ]), WithStyles; }(_react2.default.Component); - return Style.propTypes = "production" !== process.env.NODE_ENV ? { + return WithStyles.propTypes = "production" !== process.env.NODE_ENV ? { classes: _propTypes2.default.object, innerRef: _propTypes2.default.func - } : {}, Style.contextTypes = (0, _extends3.default)({ + } : {}, WithStyles.contextTypes = (0, _extends3.default)({ muiThemeProviderOptions: _propTypes2.default.object }, _contextTypes2.default, listenToTheme ? _themeListener2.default.contextTypes : {}), - "production" !== process.env.NODE_ENV && (Style.displayName = (0, _wrapDisplayName2.default)(Component, "withStyles")), - (0, _hoistNonReactStatics2.default)(Style, Component), "production" !== process.env.NODE_ENV && (Style.Naked = Component, - Style.options = options), Style; + "production" !== process.env.NODE_ENV && (WithStyles.displayName = (0, _wrapDisplayName2.default)(Component, "WithStyles")), + (0, _hoistNonReactStatics2.default)(WithStyles, Component), "production" !== process.env.NODE_ENV && (WithStyles.Naked = Component, + WithStyles.options = options), WithStyles; }; }; exports.default = withStyles; @@ -765,7 +774,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, function(module, exports, __webpack_require__) { "use strict"; exports.__esModule = !0; - var _defineProperty = __webpack_require__(142), _defineProperty2 = function(obj) { + var _defineProperty = __webpack_require__(143), _defineProperty2 = function(obj) { return obj && obj.__esModule ? obj : { default: obj }; @@ -803,7 +812,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }; Layer.propTypes = propTypes, __webpack_exports__.a = Layer; }, function(module, exports, __webpack_require__) { - var global = __webpack_require__(159), core = __webpack_require__(160), hide = __webpack_require__(246), redefine = __webpack_require__(521), ctx = __webpack_require__(524), $export = function(type, name, source) { + var global = __webpack_require__(158), core = __webpack_require__(159), hide = __webpack_require__(244), redefine = __webpack_require__(534), ctx = __webpack_require__(537), $export = function(type, name, source) { var key, own, out, exp, IS_FORCED = type & $export.F, IS_GLOBAL = type & $export.G, IS_STATIC = type & $export.S, IS_PROTO = type & $export.P, IS_BIND = type & $export.B, target = IS_GLOBAL ? global : IS_STATIC ? global[name] || (global[name] = {}) : (global[name] || {}).prototype, exports = IS_GLOBAL ? core : core[name] || (core[name] = {}), expProto = exports.prototype || (exports.prototype = {}); IS_GLOBAL && (source = name); for (key in source) own = !IS_FORCED && target && void 0 !== target[key], out = (own ? target : source)[key], @@ -891,8 +900,8 @@ var _bundleJs = []byte((((((((((`!function(modules) { }), __webpack_require__.d(__webpack_exports__, "y", function() { return parseDomainOfCategoryAxis; }); - var __WEBPACK_IMPORTED_MODULE_0_lodash_isEqual__ = __webpack_require__(34), __WEBPACK_IMPORTED_MODULE_0_lodash_isEqual___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_isEqual__), __WEBPACK_IMPORTED_MODULE_1_lodash_sortBy__ = __webpack_require__(285), __WEBPACK_IMPORTED_MODULE_1_lodash_sortBy___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_lodash_sortBy__), __WEBPACK_IMPORTED_MODULE_2_lodash_isNaN__ = __webpack_require__(116), __WEBPACK_IMPORTED_MODULE_2_lodash_isNaN___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_lodash_isNaN__), __WEBPACK_IMPORTED_MODULE_3_lodash_isString__ = __webpack_require__(165), __WEBPACK_IMPORTED_MODULE_3_lodash_isString___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_lodash_isString__), __WEBPACK_IMPORTED_MODULE_4_lodash_max__ = __webpack_require__(692), __WEBPACK_IMPORTED_MODULE_4_lodash_max___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_lodash_max__), __WEBPACK_IMPORTED_MODULE_5_lodash_min__ = __webpack_require__(288), __WEBPACK_IMPORTED_MODULE_5_lodash_min___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_5_lodash_min__), __WEBPACK_IMPORTED_MODULE_6_lodash_isArray__ = __webpack_require__(12), __WEBPACK_IMPORTED_MODULE_6_lodash_isArray___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_6_lodash_isArray__), __WEBPACK_IMPORTED_MODULE_7_lodash_isFunction__ = __webpack_require__(8), __WEBPACK_IMPORTED_MODULE_7_lodash_isFunction___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_7_lodash_isFunction__), __WEBPACK_IMPORTED_MODULE_8_lodash_get__ = __webpack_require__(109), __WEBPACK_IMPORTED_MODULE_8_lodash_get___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_8_lodash_get__), __WEBPACK_IMPORTED_MODULE_9_lodash_isNil__ = __webpack_require__(20), __WEBPACK_IMPORTED_MODULE_9_lodash_isNil___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_9_lodash_isNil__), __WEBPACK_IMPORTED_MODULE_10_recharts_scale__ = __webpack_require__(693), __WEBPACK_IMPORTED_MODULE_11_d3_scale__ = (__webpack_require__.n(__WEBPACK_IMPORTED_MODULE_10_recharts_scale__), - __webpack_require__(291)), __WEBPACK_IMPORTED_MODULE_12_d3_shape__ = __webpack_require__(173), __WEBPACK_IMPORTED_MODULE_13__DataUtils__ = __webpack_require__(9), __WEBPACK_IMPORTED_MODULE_14__cartesian_ReferenceDot__ = __webpack_require__(324), __WEBPACK_IMPORTED_MODULE_15__cartesian_ReferenceLine__ = __webpack_require__(325), __WEBPACK_IMPORTED_MODULE_16__cartesian_ReferenceArea__ = __webpack_require__(326), __WEBPACK_IMPORTED_MODULE_17__cartesian_ErrorBar__ = __webpack_require__(90), __WEBPACK_IMPORTED_MODULE_18__component_Legend__ = __webpack_require__(171), __WEBPACK_IMPORTED_MODULE_19__ReactUtils__ = __webpack_require__(4), _extends = Object.assign || function(target) { + var __WEBPACK_IMPORTED_MODULE_0_lodash_isEqual__ = __webpack_require__(34), __WEBPACK_IMPORTED_MODULE_0_lodash_isEqual___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_isEqual__), __WEBPACK_IMPORTED_MODULE_1_lodash_sortBy__ = __webpack_require__(281), __WEBPACK_IMPORTED_MODULE_1_lodash_sortBy___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_lodash_sortBy__), __WEBPACK_IMPORTED_MODULE_2_lodash_isNaN__ = __webpack_require__(117), __WEBPACK_IMPORTED_MODULE_2_lodash_isNaN___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_lodash_isNaN__), __WEBPACK_IMPORTED_MODULE_3_lodash_isString__ = __webpack_require__(164), __WEBPACK_IMPORTED_MODULE_3_lodash_isString___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_lodash_isString__), __WEBPACK_IMPORTED_MODULE_4_lodash_max__ = __webpack_require__(702), __WEBPACK_IMPORTED_MODULE_4_lodash_max___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_lodash_max__), __WEBPACK_IMPORTED_MODULE_5_lodash_min__ = __webpack_require__(284), __WEBPACK_IMPORTED_MODULE_5_lodash_min___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_5_lodash_min__), __WEBPACK_IMPORTED_MODULE_6_lodash_isArray__ = __webpack_require__(12), __WEBPACK_IMPORTED_MODULE_6_lodash_isArray___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_6_lodash_isArray__), __WEBPACK_IMPORTED_MODULE_7_lodash_isFunction__ = __webpack_require__(8), __WEBPACK_IMPORTED_MODULE_7_lodash_isFunction___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_7_lodash_isFunction__), __WEBPACK_IMPORTED_MODULE_8_lodash_get__ = __webpack_require__(165), __WEBPACK_IMPORTED_MODULE_8_lodash_get___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_8_lodash_get__), __WEBPACK_IMPORTED_MODULE_9_lodash_isNil__ = __webpack_require__(20), __WEBPACK_IMPORTED_MODULE_9_lodash_isNil___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_9_lodash_isNil__), __WEBPACK_IMPORTED_MODULE_10_recharts_scale__ = __webpack_require__(703), __WEBPACK_IMPORTED_MODULE_11_d3_scale__ = (__webpack_require__.n(__WEBPACK_IMPORTED_MODULE_10_recharts_scale__), + __webpack_require__(287)), __WEBPACK_IMPORTED_MODULE_12_d3_shape__ = __webpack_require__(173), __WEBPACK_IMPORTED_MODULE_13__DataUtils__ = __webpack_require__(9), __WEBPACK_IMPORTED_MODULE_14__cartesian_ReferenceDot__ = __webpack_require__(320), __WEBPACK_IMPORTED_MODULE_15__cartesian_ReferenceLine__ = __webpack_require__(321), __WEBPACK_IMPORTED_MODULE_16__cartesian_ReferenceArea__ = __webpack_require__(322), __WEBPACK_IMPORTED_MODULE_17__cartesian_ErrorBar__ = __webpack_require__(91), __WEBPACK_IMPORTED_MODULE_18__component_Legend__ = __webpack_require__(171), __WEBPACK_IMPORTED_MODULE_19__ReactUtils__ = __webpack_require__(4), _extends = Object.assign || function(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) Object.prototype.hasOwnProperty.call(source, key) && (target[key] = source[key]); @@ -1427,7 +1436,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { __webpack_exports__.a = newInterval; var t0 = new Date(), t1 = new Date(); }, function(module, exports, __webpack_require__) { - var global = __webpack_require__(24), core = __webpack_require__(17), ctx = __webpack_require__(47), hide = __webpack_require__(41), $export = function(type, name, source) { + var global = __webpack_require__(24), core = __webpack_require__(17), ctx = __webpack_require__(47), hide = __webpack_require__(40), $export = function(type, name, source) { var key, own, out, IS_FORCED = type & $export.F, IS_GLOBAL = type & $export.G, IS_STATIC = type & $export.S, IS_PROTO = type & $export.P, IS_BIND = type & $export.B, IS_WRAP = type & $export.W, exports = IS_GLOBAL ? core : core[name] || (core[name] = {}), expProto = exports.prototype, target = IS_GLOBAL ? global : IS_STATIC ? global[name] : (global[name] || {}).prototype; IS_GLOBAL && (source = name); for (key in source) (own = !IS_FORCED && target && void 0 !== target[key]) && key in exports || (out = own ? target[key] : source[key], @@ -1460,12 +1469,12 @@ var _bundleJs = []byte((((((((((`!function(modules) { } module.exports = isNil; }, function(module, exports, __webpack_require__) { - var store = __webpack_require__(139)("wks"), uid = __webpack_require__(97), Symbol = __webpack_require__(24).Symbol, USE_SYMBOL = "function" == typeof Symbol; + var store = __webpack_require__(140)("wks"), uid = __webpack_require__(98), Symbol = __webpack_require__(24).Symbol, USE_SYMBOL = "function" == typeof Symbol; (module.exports = function(name) { return store[name] || (store[name] = USE_SYMBOL && Symbol[name] || (USE_SYMBOL ? Symbol : uid)("Symbol." + name)); }).store = store; }, function(module, exports, __webpack_require__) { - var anObject = __webpack_require__(48), IE8_DOM_DEFINE = __webpack_require__(208), toPrimitive = __webpack_require__(133), dP = Object.defineProperty; + var anObject = __webpack_require__(48), IE8_DOM_DEFINE = __webpack_require__(207), toPrimitive = __webpack_require__(134), dP = Object.defineProperty; exports.f = __webpack_require__(25) ? Object.defineProperty : function(O, P, Attributes) { if (anObject(O), P = toPrimitive(P, !0), anObject(Attributes), IE8_DOM_DEFINE) try { return dP(O, P, Attributes); @@ -1604,7 +1613,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }); }, function(module, exports, __webpack_require__) { module.exports = { - default: __webpack_require__(355), + default: __webpack_require__(350), __esModule: !0 }; }, function(module, exports, __webpack_require__) { @@ -1615,7 +1624,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, function(module, exports, __webpack_require__) { "use strict"; exports.__esModule = !0; - var _defineProperty = __webpack_require__(142), _defineProperty2 = function(obj) { + var _defineProperty = __webpack_require__(143), _defineProperty2 = function(obj) { return obj && obj.__esModule ? obj : { default: obj }; @@ -1636,7 +1645,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, function(module, exports, __webpack_require__) { "use strict"; exports.__esModule = !0; - var _typeof2 = __webpack_require__(99), _typeof3 = function(obj) { + var _typeof2 = __webpack_require__(100), _typeof3 = function(obj) { return obj && obj.__esModule ? obj : { default: obj }; @@ -1653,7 +1662,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }; } exports.__esModule = !0; - var _setPrototypeOf = __webpack_require__(372), _setPrototypeOf2 = _interopRequireDefault(_setPrototypeOf), _create = __webpack_require__(376), _create2 = _interopRequireDefault(_create), _typeof2 = __webpack_require__(99), _typeof3 = _interopRequireDefault(_typeof2); + var _setPrototypeOf = __webpack_require__(367), _setPrototypeOf2 = _interopRequireDefault(_setPrototypeOf), _create = __webpack_require__(371), _create2 = _interopRequireDefault(_create), _typeof2 = __webpack_require__(100), _typeof3 = _interopRequireDefault(_typeof2); exports.default = function(subClass, superClass) { if ("function" != typeof superClass && null !== superClass) throw new TypeError("Super expression must either be null or a function, not " + (void 0 === superClass ? "undefined" : (0, _typeof3.default)(superClass))); @@ -1666,15 +1675,15 @@ var _bundleJs = []byte((((((((((`!function(modules) { } }), superClass && (_setPrototypeOf2.default ? (0, _setPrototypeOf2.default)(subClass, superClass) : subClass.__proto__ = superClass); }; -}, function(module, exports, __webpack_require__) { - var freeGlobal = __webpack_require__(248), freeSelf = "object" == typeof self && self && self.Object === Object && self, root = freeGlobal || freeSelf || Function("return this")(); - module.exports = root; }, function(module, exports) { function isObject(value) { var type = typeof value; return null != value && ("object" == type || "function" == type); } module.exports = isObject; +}, function(module, exports, __webpack_require__) { + var freeGlobal = __webpack_require__(242), freeSelf = "object" == typeof self && self && self.Object === Object && self, root = freeGlobal || freeSelf || Function("return this")(); + module.exports = root; }, function(module, exports, __webpack_require__) { "use strict"; function _interopRequireDefault(obj) { @@ -1685,7 +1694,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { Object.defineProperty(exports, "__esModule", { value: !0 }), exports.translateStyle = exports.AnimateGroup = exports.configBezier = exports.configSpring = void 0; - var _Animate = __webpack_require__(266), _Animate2 = _interopRequireDefault(_Animate), _easing = __webpack_require__(278), _util = __webpack_require__(122), _AnimateGroup = __webpack_require__(668), _AnimateGroup2 = _interopRequireDefault(_AnimateGroup); + var _Animate = __webpack_require__(263), _Animate2 = _interopRequireDefault(_Animate), _easing = __webpack_require__(275), _util = __webpack_require__(123), _AnimateGroup = __webpack_require__(679), _AnimateGroup2 = _interopRequireDefault(_AnimateGroup); exports.configSpring = _easing.configSpring, exports.configBezier = _easing.configBezier, exports.AnimateGroup = _AnimateGroup2.default, exports.translateStyle = _util.translateStyle, exports.default = _Animate2.default; @@ -1699,11 +1708,6 @@ var _bundleJs = []byte((((((((((`!function(modules) { module.exports = function(it) { return "object" == typeof it ? null !== it : "function" == typeof it; }; -}, function(module, exports, __webpack_require__) { - module.exports = { - default: __webpack_require__(383), - __esModule: !0 - }; }, function(module, exports) { function isObjectLike(value) { return null != value && "object" == typeof value; @@ -1711,32 +1715,32 @@ var _bundleJs = []byte((((((((((`!function(modules) { module.exports = isObjectLike; }, function(module, __webpack_exports__, __webpack_require__) { "use strict"; - var __WEBPACK_IMPORTED_MODULE_0__src_bisect__ = __webpack_require__(292); + var __WEBPACK_IMPORTED_MODULE_0__src_bisect__ = __webpack_require__(288); __webpack_require__.d(__webpack_exports__, "b", function() { return __WEBPACK_IMPORTED_MODULE_0__src_bisect__.a; }); - var __WEBPACK_IMPORTED_MODULE_1__src_ascending__ = __webpack_require__(63); + var __WEBPACK_IMPORTED_MODULE_1__src_ascending__ = __webpack_require__(64); __webpack_require__.d(__webpack_exports__, "a", function() { return __WEBPACK_IMPORTED_MODULE_1__src_ascending__.a; }); - var __WEBPACK_IMPORTED_MODULE_2__src_bisector__ = __webpack_require__(293); + var __WEBPACK_IMPORTED_MODULE_2__src_bisector__ = __webpack_require__(289); __webpack_require__.d(__webpack_exports__, "c", function() { return __WEBPACK_IMPORTED_MODULE_2__src_bisector__.a; }); - var __WEBPACK_IMPORTED_MODULE_18__src_quantile__ = (__webpack_require__(697), __webpack_require__(698), - __webpack_require__(295), __webpack_require__(297), __webpack_require__(699), __webpack_require__(702), - __webpack_require__(703), __webpack_require__(301), __webpack_require__(704), __webpack_require__(705), - __webpack_require__(706), __webpack_require__(707), __webpack_require__(302), __webpack_require__(294), - __webpack_require__(708), __webpack_require__(186)); + var __WEBPACK_IMPORTED_MODULE_18__src_quantile__ = (__webpack_require__(707), __webpack_require__(708), + __webpack_require__(291), __webpack_require__(293), __webpack_require__(709), __webpack_require__(712), + __webpack_require__(713), __webpack_require__(297), __webpack_require__(714), __webpack_require__(715), + __webpack_require__(716), __webpack_require__(717), __webpack_require__(298), __webpack_require__(290), + __webpack_require__(718), __webpack_require__(185)); __webpack_require__.d(__webpack_exports__, "d", function() { return __WEBPACK_IMPORTED_MODULE_18__src_quantile__.a; }); - var __WEBPACK_IMPORTED_MODULE_19__src_range__ = __webpack_require__(299); + var __WEBPACK_IMPORTED_MODULE_19__src_range__ = __webpack_require__(295); __webpack_require__.d(__webpack_exports__, "e", function() { return __WEBPACK_IMPORTED_MODULE_19__src_range__.a; }); - var __WEBPACK_IMPORTED_MODULE_23__src_ticks__ = (__webpack_require__(709), __webpack_require__(710), - __webpack_require__(711), __webpack_require__(300)); + var __WEBPACK_IMPORTED_MODULE_23__src_ticks__ = (__webpack_require__(719), __webpack_require__(720), + __webpack_require__(721), __webpack_require__(296)); __webpack_require__.d(__webpack_exports__, "h", function() { return __WEBPACK_IMPORTED_MODULE_23__src_ticks__.a; }), __webpack_require__.d(__webpack_exports__, "f", function() { @@ -1744,7 +1748,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }), __webpack_require__.d(__webpack_exports__, "g", function() { return __WEBPACK_IMPORTED_MODULE_23__src_ticks__.c; }); - __webpack_require__(303), __webpack_require__(296), __webpack_require__(712); + __webpack_require__(299), __webpack_require__(292), __webpack_require__(722); }, function(module, __webpack_exports__, __webpack_require__) { "use strict"; __webpack_require__.d(__webpack_exports__, "d", function() { @@ -1775,17 +1779,22 @@ var _bundleJs = []byte((((((((((`!function(modules) { return arg; }, module.exports = emptyFunction; }, function(module, exports, __webpack_require__) { - var dP = __webpack_require__(22), createDesc = __webpack_require__(70); + var dP = __webpack_require__(22), createDesc = __webpack_require__(71); module.exports = __webpack_require__(25) ? function(object, key, value) { return dP.f(object, key, createDesc(1, value)); } : function(object, key, value) { return object[key] = value, object; }; +}, function(module, exports, __webpack_require__) { + module.exports = { + default: __webpack_require__(378), + __esModule: !0 + }; }, function(module, exports, __webpack_require__) { function baseGetTag(value) { return null == value ? void 0 === value ? undefinedTag : nullTag : symToStringTag && symToStringTag in Object(value) ? getRawTag(value) : objectToString(value); } - var Symbol = __webpack_require__(77), getRawTag = __webpack_require__(543), objectToString = __webpack_require__(544), nullTag = "[object Null]", undefinedTag = "[object Undefined]", symToStringTag = Symbol ? Symbol.toStringTag : void 0; + var Symbol = __webpack_require__(77), getRawTag = __webpack_require__(520), objectToString = __webpack_require__(521), nullTag = "[object Null]", undefinedTag = "[object Undefined]", symToStringTag = Symbol ? Symbol.toStringTag : void 0; module.exports = baseGetTag; }, function(module, __webpack_exports__, __webpack_require__) { "use strict"; @@ -1811,7 +1820,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { className: __WEBPACK_IMPORTED_MODULE_5_classnames___default()("recharts-label", className) }, attrs, positionAttrs), label); } - var __WEBPACK_IMPORTED_MODULE_0_lodash_isObject__ = __webpack_require__(32), __WEBPACK_IMPORTED_MODULE_0_lodash_isObject___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_isObject__), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction__ = __webpack_require__(8), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_lodash_isFunction__), __WEBPACK_IMPORTED_MODULE_2_lodash_isNil__ = __webpack_require__(20), __WEBPACK_IMPORTED_MODULE_2_lodash_isNil___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_lodash_isNil__), __WEBPACK_IMPORTED_MODULE_3_react__ = __webpack_require__(0), __WEBPACK_IMPORTED_MODULE_3_react___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_react__), __WEBPACK_IMPORTED_MODULE_4_prop_types__ = __webpack_require__(1), __WEBPACK_IMPORTED_MODULE_4_prop_types___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_prop_types__), __WEBPACK_IMPORTED_MODULE_5_classnames__ = __webpack_require__(3), __WEBPACK_IMPORTED_MODULE_5_classnames___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_5_classnames__), __WEBPACK_IMPORTED_MODULE_6__Text__ = __webpack_require__(55), __WEBPACK_IMPORTED_MODULE_7__util_ReactUtils__ = __webpack_require__(4), __WEBPACK_IMPORTED_MODULE_8__util_DataUtils__ = __webpack_require__(9), __WEBPACK_IMPORTED_MODULE_9__util_PolarUtils__ = __webpack_require__(23), _extends = Object.assign || function(target) { + var __WEBPACK_IMPORTED_MODULE_0_lodash_isObject__ = __webpack_require__(31), __WEBPACK_IMPORTED_MODULE_0_lodash_isObject___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_isObject__), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction__ = __webpack_require__(8), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_lodash_isFunction__), __WEBPACK_IMPORTED_MODULE_2_lodash_isNil__ = __webpack_require__(20), __WEBPACK_IMPORTED_MODULE_2_lodash_isNil___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_lodash_isNil__), __WEBPACK_IMPORTED_MODULE_3_react__ = __webpack_require__(0), __WEBPACK_IMPORTED_MODULE_3_react___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_react__), __WEBPACK_IMPORTED_MODULE_4_prop_types__ = __webpack_require__(1), __WEBPACK_IMPORTED_MODULE_4_prop_types___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_prop_types__), __WEBPACK_IMPORTED_MODULE_5_classnames__ = __webpack_require__(3), __WEBPACK_IMPORTED_MODULE_5_classnames___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_5_classnames__), __WEBPACK_IMPORTED_MODULE_6__Text__ = __webpack_require__(55), __WEBPACK_IMPORTED_MODULE_7__util_ReactUtils__ = __webpack_require__(4), __WEBPACK_IMPORTED_MODULE_8__util_DataUtils__ = __webpack_require__(9), __WEBPACK_IMPORTED_MODULE_9__util_PolarUtils__ = __webpack_require__(23), _extends = Object.assign || function(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) Object.prototype.hasOwnProperty.call(source, key) && (target[key] = source[key]); @@ -2023,7 +2032,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { __webpack_exports__.a = Label; }, function(module, __webpack_exports__, __webpack_require__) { "use strict"; - var __WEBPACK_IMPORTED_MODULE_0__src_color__ = __webpack_require__(189); + var __WEBPACK_IMPORTED_MODULE_0__src_color__ = __webpack_require__(188); __webpack_require__.d(__webpack_exports__, "a", function() { return __WEBPACK_IMPORTED_MODULE_0__src_color__.e; }), __webpack_require__.d(__webpack_exports__, "f", function() { @@ -2031,13 +2040,13 @@ var _bundleJs = []byte((((((((((`!function(modules) { }), __webpack_require__.d(__webpack_exports__, "d", function() { return __WEBPACK_IMPORTED_MODULE_0__src_color__.f; }); - var __WEBPACK_IMPORTED_MODULE_1__src_lab__ = __webpack_require__(720); + var __WEBPACK_IMPORTED_MODULE_1__src_lab__ = __webpack_require__(730); __webpack_require__.d(__webpack_exports__, "e", function() { return __WEBPACK_IMPORTED_MODULE_1__src_lab__.a; }), __webpack_require__.d(__webpack_exports__, "c", function() { return __WEBPACK_IMPORTED_MODULE_1__src_lab__.b; }); - var __WEBPACK_IMPORTED_MODULE_2__src_cubehelix__ = __webpack_require__(721); + var __WEBPACK_IMPORTED_MODULE_2__src_cubehelix__ = __webpack_require__(731); __webpack_require__.d(__webpack_exports__, "b", function() { return __WEBPACK_IMPORTED_MODULE_2__src_cubehelix__.a; }); @@ -2073,7 +2082,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { })); })) : null; } - var __WEBPACK_IMPORTED_MODULE_0_lodash_isObject__ = __webpack_require__(32), __WEBPACK_IMPORTED_MODULE_0_lodash_isObject___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_isObject__), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction__ = __webpack_require__(8), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_lodash_isFunction__), __WEBPACK_IMPORTED_MODULE_2_lodash_isNil__ = __webpack_require__(20), __WEBPACK_IMPORTED_MODULE_2_lodash_isNil___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_lodash_isNil__), __WEBPACK_IMPORTED_MODULE_3_lodash_last__ = __webpack_require__(771), __WEBPACK_IMPORTED_MODULE_3_lodash_last___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_lodash_last__), __WEBPACK_IMPORTED_MODULE_4_lodash_isArray__ = __webpack_require__(12), __WEBPACK_IMPORTED_MODULE_4_lodash_isArray___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_lodash_isArray__), __WEBPACK_IMPORTED_MODULE_5_react__ = __webpack_require__(0), __WEBPACK_IMPORTED_MODULE_5_react___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_5_react__), __WEBPACK_IMPORTED_MODULE_6_prop_types__ = __webpack_require__(1), __WEBPACK_IMPORTED_MODULE_6_prop_types___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_6_prop_types__), __WEBPACK_IMPORTED_MODULE_7__Label__ = __webpack_require__(43), __WEBPACK_IMPORTED_MODULE_8__container_Layer__ = __webpack_require__(14), __WEBPACK_IMPORTED_MODULE_9__util_ReactUtils__ = __webpack_require__(4), __WEBPACK_IMPORTED_MODULE_10__util_ChartUtils__ = __webpack_require__(16), _extends = Object.assign || function(target) { + var __WEBPACK_IMPORTED_MODULE_0_lodash_isObject__ = __webpack_require__(31), __WEBPACK_IMPORTED_MODULE_0_lodash_isObject___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_isObject__), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction__ = __webpack_require__(8), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_lodash_isFunction__), __WEBPACK_IMPORTED_MODULE_2_lodash_isNil__ = __webpack_require__(20), __WEBPACK_IMPORTED_MODULE_2_lodash_isNil___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_lodash_isNil__), __WEBPACK_IMPORTED_MODULE_3_lodash_last__ = __webpack_require__(781), __WEBPACK_IMPORTED_MODULE_3_lodash_last___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_lodash_last__), __WEBPACK_IMPORTED_MODULE_4_lodash_isArray__ = __webpack_require__(12), __WEBPACK_IMPORTED_MODULE_4_lodash_isArray___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_lodash_isArray__), __WEBPACK_IMPORTED_MODULE_5_react__ = __webpack_require__(0), __WEBPACK_IMPORTED_MODULE_5_react___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_5_react__), __WEBPACK_IMPORTED_MODULE_6_prop_types__ = __webpack_require__(1), __WEBPACK_IMPORTED_MODULE_6_prop_types___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_6_prop_types__), __WEBPACK_IMPORTED_MODULE_7__Label__ = __webpack_require__(43), __WEBPACK_IMPORTED_MODULE_8__container_Layer__ = __webpack_require__(14), __WEBPACK_IMPORTED_MODULE_9__util_ReactUtils__ = __webpack_require__(4), __WEBPACK_IMPORTED_MODULE_10__util_ChartUtils__ = __webpack_require__(16), _extends = Object.assign || function(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) Object.prototype.hasOwnProperty.call(source, key) && (target[key] = source[key]); @@ -2157,7 +2166,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { } }), superClass && (Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass); } - var __WEBPACK_IMPORTED_MODULE_0_lodash_sortBy__ = __webpack_require__(285), __WEBPACK_IMPORTED_MODULE_0_lodash_sortBy___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_sortBy__), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction__ = __webpack_require__(8), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_lodash_isFunction__), __WEBPACK_IMPORTED_MODULE_2_lodash_get__ = __webpack_require__(109), __WEBPACK_IMPORTED_MODULE_2_lodash_get___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_lodash_get__), __WEBPACK_IMPORTED_MODULE_3_lodash_range__ = __webpack_require__(333), __WEBPACK_IMPORTED_MODULE_3_lodash_range___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_lodash_range__), __WEBPACK_IMPORTED_MODULE_4_lodash_throttle__ = __webpack_require__(779), __WEBPACK_IMPORTED_MODULE_4_lodash_throttle___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_lodash_throttle__), __WEBPACK_IMPORTED_MODULE_5_lodash_isNil__ = __webpack_require__(20), __WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_5_lodash_isNil__), __WEBPACK_IMPORTED_MODULE_6_react__ = __webpack_require__(0), __WEBPACK_IMPORTED_MODULE_6_react___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_6_react__), __WEBPACK_IMPORTED_MODULE_7_prop_types__ = __webpack_require__(1), __WEBPACK_IMPORTED_MODULE_7_prop_types___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_7_prop_types__), __WEBPACK_IMPORTED_MODULE_8_classnames__ = __webpack_require__(3), __WEBPACK_IMPORTED_MODULE_8_classnames___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_8_classnames__), __WEBPACK_IMPORTED_MODULE_9__container_Surface__ = __webpack_require__(76), __WEBPACK_IMPORTED_MODULE_10__container_Layer__ = __webpack_require__(14), __WEBPACK_IMPORTED_MODULE_11__component_Tooltip__ = __webpack_require__(121), __WEBPACK_IMPORTED_MODULE_12__component_Legend__ = __webpack_require__(171), __WEBPACK_IMPORTED_MODULE_13__shape_Curve__ = __webpack_require__(65), __WEBPACK_IMPORTED_MODULE_14__shape_Cross__ = __webpack_require__(327), __WEBPACK_IMPORTED_MODULE_15__shape_Sector__ = __webpack_require__(127), __WEBPACK_IMPORTED_MODULE_16__shape_Dot__ = __webpack_require__(57), __WEBPACK_IMPORTED_MODULE_17__shape_Rectangle__ = __webpack_require__(64), __WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__ = __webpack_require__(4), __WEBPACK_IMPORTED_MODULE_19__cartesian_CartesianAxis__ = __webpack_require__(334), __WEBPACK_IMPORTED_MODULE_20__cartesian_Brush__ = __webpack_require__(332), __WEBPACK_IMPORTED_MODULE_21__util_DOMUtils__ = __webpack_require__(185), __WEBPACK_IMPORTED_MODULE_22__util_DataUtils__ = __webpack_require__(9), __WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__ = __webpack_require__(16), __WEBPACK_IMPORTED_MODULE_24__util_PolarUtils__ = __webpack_require__(23), __WEBPACK_IMPORTED_MODULE_25__util_PureRender__ = __webpack_require__(5), __WEBPACK_IMPORTED_MODULE_26__util_Events__ = __webpack_require__(780), _extends = Object.assign || function(target) { + var __WEBPACK_IMPORTED_MODULE_0_lodash_sortBy__ = __webpack_require__(281), __WEBPACK_IMPORTED_MODULE_0_lodash_sortBy___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_sortBy__), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction__ = __webpack_require__(8), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_lodash_isFunction__), __WEBPACK_IMPORTED_MODULE_2_lodash_range__ = __webpack_require__(329), __WEBPACK_IMPORTED_MODULE_2_lodash_range___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_lodash_range__), __WEBPACK_IMPORTED_MODULE_3_lodash_throttle__ = __webpack_require__(790), __WEBPACK_IMPORTED_MODULE_3_lodash_throttle___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_lodash_throttle__), __WEBPACK_IMPORTED_MODULE_4_lodash_isNil__ = __webpack_require__(20), __WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_lodash_isNil__), __WEBPACK_IMPORTED_MODULE_5_react__ = __webpack_require__(0), __WEBPACK_IMPORTED_MODULE_5_react___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_5_react__), __WEBPACK_IMPORTED_MODULE_6_prop_types__ = __webpack_require__(1), __WEBPACK_IMPORTED_MODULE_6_prop_types___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_6_prop_types__), __WEBPACK_IMPORTED_MODULE_7_classnames__ = __webpack_require__(3), __WEBPACK_IMPORTED_MODULE_7_classnames___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_7_classnames__), __WEBPACK_IMPORTED_MODULE_8__container_Surface__ = __webpack_require__(78), __WEBPACK_IMPORTED_MODULE_9__container_Layer__ = __webpack_require__(14), __WEBPACK_IMPORTED_MODULE_10__component_Tooltip__ = __webpack_require__(122), __WEBPACK_IMPORTED_MODULE_11__component_Legend__ = __webpack_require__(171), __WEBPACK_IMPORTED_MODULE_12__shape_Curve__ = __webpack_require__(66), __WEBPACK_IMPORTED_MODULE_13__shape_Cross__ = __webpack_require__(323), __WEBPACK_IMPORTED_MODULE_14__shape_Sector__ = __webpack_require__(128), __WEBPACK_IMPORTED_MODULE_15__shape_Dot__ = __webpack_require__(57), __WEBPACK_IMPORTED_MODULE_16__shape_Rectangle__ = __webpack_require__(65), __WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__ = __webpack_require__(4), __WEBPACK_IMPORTED_MODULE_18__cartesian_CartesianAxis__ = __webpack_require__(330), __WEBPACK_IMPORTED_MODULE_19__cartesian_Brush__ = __webpack_require__(328), __WEBPACK_IMPORTED_MODULE_20__util_DOMUtils__ = __webpack_require__(184), __WEBPACK_IMPORTED_MODULE_21__util_DataUtils__ = __webpack_require__(9), __WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__ = __webpack_require__(16), __WEBPACK_IMPORTED_MODULE_23__util_PolarUtils__ = __webpack_require__(23), __WEBPACK_IMPORTED_MODULE_24__util_PureRender__ = __webpack_require__(5), __WEBPACK_IMPORTED_MODULE_25__util_Events__ = __webpack_require__(791), _extends = Object.assign || function(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) Object.prototype.hasOwnProperty.call(source, key) && (target[key] = source[key]); @@ -2195,22 +2204,22 @@ var _bundleJs = []byte((((((((((`!function(modules) { props: props }, defaultState, { updateId: 0 - }))), _this.uniqueChartId = __WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(props.id) ? Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.j)("recharts") : props.id, - props.throttleDelay && (_this.triggeredAfterMouseMove = __WEBPACK_IMPORTED_MODULE_4_lodash_throttle___default()(_this.triggeredAfterMouseMove, props.throttleDelay)), + }))), _this.uniqueChartId = __WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(props.id) ? Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.j)("recharts") : props.id, + props.throttleDelay && (_this.triggeredAfterMouseMove = __WEBPACK_IMPORTED_MODULE_3_lodash_throttle___default()(_this.triggeredAfterMouseMove, props.throttleDelay)), _this; } return _inherits(CategoricalChartWrapper, _Component), _createClass(CategoricalChartWrapper, [ { key: "componentDidMount", value: function() { - __WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(this.props.syncId) || this.addListener(); + __WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(this.props.syncId) || this.addListener(); } }, { key: "componentWillReceiveProps", value: function(nextProps) { var _props = this.props, data = _props.data, children = _props.children, width = _props.width, height = _props.height, layout = _props.layout, stackOffset = _props.stackOffset, margin = _props.margin, updateId = this.state.updateId; - if (nextProps.data === data && nextProps.width === width && nextProps.height === height && nextProps.layout === layout && nextProps.stackOffset === stackOffset && Object(__WEBPACK_IMPORTED_MODULE_25__util_PureRender__.b)(nextProps.margin, margin)) { - if (!Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.m)(nextProps.children, children)) { - var hasGlobalData = !__WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(nextProps.data), newUpdateId = hasGlobalData ? updateId : updateId + 1, _state = this.state, dataStartIndex = _state.dataStartIndex, dataEndIndex = _state.dataEndIndex, _defaultState = _extends({}, this.constructor.createDefaultState(nextProps), { + if (nextProps.data === data && nextProps.width === width && nextProps.height === height && nextProps.layout === layout && nextProps.stackOffset === stackOffset && Object(__WEBPACK_IMPORTED_MODULE_24__util_PureRender__.b)(nextProps.margin, margin)) { + if (!Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.m)(nextProps.children, children)) { + var hasGlobalData = !__WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(nextProps.data), newUpdateId = hasGlobalData ? updateId : updateId + 1, _state = this.state, dataStartIndex = _state.dataStartIndex, dataEndIndex = _state.dataEndIndex, _defaultState = _extends({}, this.constructor.createDefaultState(nextProps), { dataEndIndex: dataEndIndex, dataStartIndex: dataStartIndex }); @@ -2232,19 +2241,19 @@ var _bundleJs = []byte((((((((((`!function(modules) { updateId: updateId + 1 })))); } - __WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(this.props.syncId) && !__WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(nextProps.syncId) && this.addListener(), - !__WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(this.props.syncId) && __WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(nextProps.syncId) && this.removeListener(); + __WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(this.props.syncId) && !__WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(nextProps.syncId) && this.addListener(), + !__WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(this.props.syncId) && __WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(nextProps.syncId) && this.removeListener(); } }, { key: "componentWillUnmount", value: function() { - __WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(this.props.syncId) || this.removeListener(), + __WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(this.props.syncId) || this.removeListener(), "function" == typeof this.triggeredAfterMouseMove.cancel && this.triggeredAfterMouseMove.cancel(); } }, { key: "getAxisMap", value: function(props, _ref2) { - var _ref2$axisType = _ref2.axisType, axisType = void 0 === _ref2$axisType ? "xAxis" : _ref2$axisType, AxisComp = _ref2.AxisComp, graphicalItems = _ref2.graphicalItems, stackGroups = _ref2.stackGroups, dataStartIndex = _ref2.dataStartIndex, dataEndIndex = _ref2.dataEndIndex, children = props.children, axisIdKey = axisType + "Id", axes = Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.h)(children, AxisComp), axisMap = {}; + var _ref2$axisType = _ref2.axisType, axisType = void 0 === _ref2$axisType ? "xAxis" : _ref2$axisType, AxisComp = _ref2.AxisComp, graphicalItems = _ref2.graphicalItems, stackGroups = _ref2.stackGroups, dataStartIndex = _ref2.dataStartIndex, dataEndIndex = _ref2.dataEndIndex, children = props.children, axisIdKey = axisType + "Id", axes = Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.h)(children, AxisComp), axisMap = {}; return axes && axes.length ? axisMap = this.getAxisMapByAxes(props, { axes: axes, graphicalItems: graphicalItems, @@ -2266,7 +2275,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, { key: "getAxisMapByAxes", value: function(props, _ref3) { - var _this2 = this, axes = _ref3.axes, graphicalItems = _ref3.graphicalItems, axisType = _ref3.axisType, axisIdKey = _ref3.axisIdKey, stackGroups = _ref3.stackGroups, dataStartIndex = _ref3.dataStartIndex, dataEndIndex = _ref3.dataEndIndex, layout = props.layout, children = props.children, stackOffset = props.stackOffset, isCategorial = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.x)(layout, axisType); + var _this2 = this, axes = _ref3.axes, graphicalItems = _ref3.graphicalItems, axisType = _ref3.axisType, axisIdKey = _ref3.axisIdKey, stackGroups = _ref3.stackGroups, dataStartIndex = _ref3.dataStartIndex, dataEndIndex = _ref3.dataEndIndex, layout = props.layout, children = props.children, stackOffset = props.stackOffset, isCategorial = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.x)(layout, axisType); return axes.reduce(function(result, child) { var _child$props = child.props, type = _child$props.type, dataKey = _child$props.dataKey, allowDataOverflow = _child$props.allowDataOverflow, allowDuplicatedCategory = _child$props.allowDuplicatedCategory, scale = _child$props.scale, ticks = _child$props.ticks, axisId = child.props[axisIdKey], displayedData = _this2.constructor.getDisplayedData(props, { graphicalItems: graphicalItems.filter(function(item) { @@ -2278,28 +2287,28 @@ var _bundleJs = []byte((((((((((`!function(modules) { if (!result[axisId]) { var domain = void 0, duplicateDomain = void 0, categoricalDomain = void 0; if (dataKey) { - if (domain = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.n)(displayedData, dataKey, type), + if (domain = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.n)(displayedData, dataKey, type), "category" === type && isCategorial) { - var duplicate = Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.d)(domain); - allowDuplicatedCategory && duplicate ? (duplicateDomain = domain, domain = __WEBPACK_IMPORTED_MODULE_3_lodash_range___default()(0, len)) : allowDuplicatedCategory || (domain = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.y)(child.props.domain, domain, child).reduce(function(finalDomain, entry) { + var duplicate = Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.d)(domain); + allowDuplicatedCategory && duplicate ? (duplicateDomain = domain, domain = __WEBPACK_IMPORTED_MODULE_2_lodash_range___default()(0, len)) : allowDuplicatedCategory || (domain = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.y)(child.props.domain, domain, child).reduce(function(finalDomain, entry) { return finalDomain.indexOf(entry) >= 0 ? finalDomain : [].concat(_toConsumableArray(finalDomain), [ entry ]); }, [])); } else if ("category" === type) domain = allowDuplicatedCategory ? domain.filter(function(entry) { - return "" !== entry && !__WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(entry); - }) : Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.y)(child.props.domain, domain, child).reduce(function(finalDomain, entry) { - return finalDomain.indexOf(entry) >= 0 || "" === entry || __WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(entry) ? finalDomain : [].concat(_toConsumableArray(finalDomain), [ entry ]); + return "" !== entry && !__WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(entry); + }) : Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.y)(child.props.domain, domain, child).reduce(function(finalDomain, entry) { + return finalDomain.indexOf(entry) >= 0 || "" === entry || __WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(entry) ? finalDomain : [].concat(_toConsumableArray(finalDomain), [ entry ]); }, []); else if ("number" === type) { - var errorBarsDomain = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.z)(displayedData, graphicalItems.filter(function(item) { + var errorBarsDomain = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.z)(displayedData, graphicalItems.filter(function(item) { return item.props[axisIdKey] === axisId && !item.props.hide; }), dataKey, axisType); errorBarsDomain && (domain = errorBarsDomain); } - !isCategorial || "number" !== type && "auto" === scale || (categoricalDomain = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.n)(displayedData, dataKey, "category")); - } else domain = isCategorial ? __WEBPACK_IMPORTED_MODULE_3_lodash_range___default()(0, len) : stackGroups && stackGroups[axisId] && stackGroups[axisId].hasStack && "number" === type ? "expand" === stackOffset ? [ 0, 1 ] : Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.p)(stackGroups[axisId].stackGroups, dataStartIndex, dataEndIndex) : Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.o)(displayedData, graphicalItems.filter(function(item) { + !isCategorial || "number" !== type && "auto" === scale || (categoricalDomain = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.n)(displayedData, dataKey, "category")); + } else domain = isCategorial ? __WEBPACK_IMPORTED_MODULE_2_lodash_range___default()(0, len) : stackGroups && stackGroups[axisId] && stackGroups[axisId].hasStack && "number" === type ? "expand" === stackOffset ? [ 0, 1 ] : Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.p)(stackGroups[axisId].stackGroups, dataStartIndex, dataEndIndex) : Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.o)(displayedData, graphicalItems.filter(function(item) { return item.props[axisIdKey] === axisId && !item.props.hide; }), type, !0); - return "number" === type && (domain = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.e)(children, domain, axisId, axisType, ticks), - child.props.domain && (domain = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.B)(child.props.domain, domain, allowDataOverflow))), + return "number" === type && (domain = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.e)(children, domain, axisId, axisType, ticks), + child.props.domain && (domain = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.B)(child.props.domain, domain, allowDataOverflow))), _extends({}, result, _defineProperty({}, axisId, _extends({}, child.props, { axisType: axisType, domain: domain, @@ -2320,16 +2329,16 @@ var _bundleJs = []byte((((((((((`!function(modules) { graphicalItems: graphicalItems, dataStartIndex: dataStartIndex, dataEndIndex: dataEndIndex - }), len = displayedData.length, isCategorial = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.x)(layout, axisType), index = -1; + }), len = displayedData.length, isCategorial = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.x)(layout, axisType), index = -1; return graphicalItems.reduce(function(result, child) { var axisId = child.props[axisIdKey]; if (!result[axisId]) { index++; var domain = void 0; - return isCategorial ? domain = __WEBPACK_IMPORTED_MODULE_3_lodash_range___default()(0, len) : stackGroups && stackGroups[axisId] && stackGroups[axisId].hasStack ? (domain = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.p)(stackGroups[axisId].stackGroups, dataStartIndex, dataEndIndex), - domain = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.e)(children, domain, axisId, axisType)) : (domain = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.B)(Axis.defaultProps.domain, Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.o)(displayedData, graphicalItems.filter(function(item) { + return isCategorial ? domain = __WEBPACK_IMPORTED_MODULE_2_lodash_range___default()(0, len) : stackGroups && stackGroups[axisId] && stackGroups[axisId].hasStack ? (domain = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.p)(stackGroups[axisId].stackGroups, dataStartIndex, dataEndIndex), + domain = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.e)(children, domain, axisId, axisType)) : (domain = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.B)(Axis.defaultProps.domain, Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.o)(displayedData, graphicalItems.filter(function(item) { return item.props[axisIdKey] === axisId && !item.props.hide; - }), "number"), Axis.defaultProps.allowDataOverflow), domain = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.e)(children, domain, axisId, axisType)), + }), "number"), Axis.defaultProps.allowDataOverflow), domain = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.e)(children, domain, axisId, axisType)), _extends({}, result, _defineProperty({}, axisId, _extends({ axisType: axisType }, Axis.defaultProps, { @@ -2347,9 +2356,9 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, { key: "getActiveCoordinate", value: function(tooltipTicks, activeIndex, rangeObj) { - var layout = this.props.layout, entry = __WEBPACK_IMPORTED_MODULE_2_lodash_get___default()(tooltipTicks.filter(function(tick) { + var layout = this.props.layout, entry = tooltipTicks.find(function(tick) { return tick && tick.index === activeIndex; - }), "[0]"); + }); if (entry) { if ("horizontal" === layout) return { x: entry.coordinate, @@ -2361,13 +2370,13 @@ var _bundleJs = []byte((((((((((`!function(modules) { }; if ("centric" === layout) { var _angle = entry.coordinate, _radius = rangeObj.radius; - return _extends({}, rangeObj, Object(__WEBPACK_IMPORTED_MODULE_24__util_PolarUtils__.e)(rangeObj.cx, rangeObj.cy, _radius, _angle), { + return _extends({}, rangeObj, Object(__WEBPACK_IMPORTED_MODULE_23__util_PolarUtils__.e)(rangeObj.cx, rangeObj.cy, _radius, _angle), { angle: _angle, radius: _radius }); } var radius = entry.coordinate, angle = rangeObj.angle; - return _extends({}, rangeObj, Object(__WEBPACK_IMPORTED_MODULE_24__util_PolarUtils__.e)(rangeObj.cx, rangeObj.cy, radius, angle), { + return _extends({}, rangeObj, Object(__WEBPACK_IMPORTED_MODULE_23__util_PolarUtils__.e)(rangeObj.cx, rangeObj.cy, radius, angle), { angle: angle, radius: radius }); @@ -2378,17 +2387,17 @@ var _bundleJs = []byte((((((((((`!function(modules) { key: "getMouseInfo", value: function(event) { if (!this.container) return null; - var containerOffset = Object(__WEBPACK_IMPORTED_MODULE_21__util_DOMUtils__.b)(this.container), e = Object(__WEBPACK_IMPORTED_MODULE_21__util_DOMUtils__.a)(event, containerOffset), rangeObj = this.inRange(e.chartX, e.chartY); + var containerOffset = Object(__WEBPACK_IMPORTED_MODULE_20__util_DOMUtils__.b)(this.container), e = Object(__WEBPACK_IMPORTED_MODULE_20__util_DOMUtils__.a)(event, containerOffset), rangeObj = this.inRange(e.chartX, e.chartY); if (!rangeObj) return null; var _state2 = this.state, xAxisMap = _state2.xAxisMap, yAxisMap = _state2.yAxisMap; if ("axis" !== eventType && xAxisMap && yAxisMap) { - var xScale = Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.b)(xAxisMap).scale, yScale = Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.b)(yAxisMap).scale, xValue = xScale && xScale.invert ? xScale.invert(e.chartX) : null, yValue = yScale && yScale.invert ? yScale.invert(e.chartY) : null; + var xScale = Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.b)(xAxisMap).scale, yScale = Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.b)(yAxisMap).scale, xValue = xScale && xScale.invert ? xScale.invert(e.chartX) : null, yValue = yScale && yScale.invert ? yScale.invert(e.chartY) : null; return _extends({}, e, { xValue: xValue, yValue: yValue }); } - var _state3 = this.state, ticks = _state3.orderedTooltipTicks, axis = _state3.tooltipAxis, tooltipTicks = _state3.tooltipTicks, pos = this.calculateTooltipPos(rangeObj), activeIndex = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.b)(pos, ticks, tooltipTicks, axis); + var _state3 = this.state, ticks = _state3.orderedTooltipTicks, axis = _state3.tooltipAxis, tooltipTicks = _state3.tooltipTicks, pos = this.calculateTooltipPos(rangeObj), activeIndex = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.b)(pos, ticks, tooltipTicks, axis); if (activeIndex >= 0 && tooltipTicks) { var activeLabel = tooltipTicks[activeIndex] && tooltipTicks[activeIndex].value, activePayload = this.getTooltipContent(activeIndex, activeLabel), activeCoordinate = this.getActiveCoordinate(ticks, activeIndex, rangeObj); return _extends({}, e, { @@ -2407,14 +2416,14 @@ var _bundleJs = []byte((((((((((`!function(modules) { return activeIndex < 0 || !graphicalItems || !graphicalItems.length || activeIndex >= displayedData.length ? null : graphicalItems.reduce(function(result, child) { if (child.props.hide) return result; var _child$props2 = child.props, dataKey = _child$props2.dataKey, name = _child$props2.name, unit = _child$props2.unit, formatter = _child$props2.formatter, data = _child$props2.data, payload = void 0; - return payload = tooltipAxis.dataKey && !tooltipAxis.allowDuplicatedCategory ? Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.a)(data || displayedData, tooltipAxis.dataKey, activeLabel) : displayedData[activeIndex], - payload ? [].concat(_toConsumableArray(result), [ _extends({}, Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.k)(child), { + return payload = tooltipAxis.dataKey && !tooltipAxis.allowDuplicatedCategory ? Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.a)(data || displayedData, tooltipAxis.dataKey, activeLabel) : displayedData[activeIndex], + payload ? [].concat(_toConsumableArray(result), [ _extends({}, Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.k)(child), { dataKey: dataKey, unit: unit, formatter: formatter, name: name || dataKey, - color: Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.r)(child), - value: Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.w)(payload, dataKey), + color: Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.r)(child), + value: Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.w)(payload, dataKey), payload: payload }) ]) : result; }, []); @@ -2422,7 +2431,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, { key: "getFormatItems", value: function(props, currentState) { - var _this3 = this, graphicalItems = currentState.graphicalItems, stackGroups = currentState.stackGroups, offset = currentState.offset, updateId = currentState.updateId, dataStartIndex = currentState.dataStartIndex, dataEndIndex = currentState.dataEndIndex, barSize = props.barSize, layout = props.layout, barGap = props.barGap, barCategoryGap = props.barCategoryGap, globalMaxBarSize = props.maxBarSize, _getAxisNameByLayout = this.getAxisNameByLayout(layout), numericAxisName = _getAxisNameByLayout.numericAxisName, cateAxisName = _getAxisNameByLayout.cateAxisName, hasBar = this.constructor.hasBar(graphicalItems), sizeList = hasBar && Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.i)({ + var _this3 = this, graphicalItems = currentState.graphicalItems, stackGroups = currentState.stackGroups, offset = currentState.offset, updateId = currentState.updateId, dataStartIndex = currentState.dataStartIndex, dataEndIndex = currentState.dataEndIndex, barSize = props.barSize, layout = props.layout, barGap = props.barGap, barCategoryGap = props.barCategoryGap, globalMaxBarSize = props.maxBarSize, _getAxisNameByLayout = this.getAxisNameByLayout(layout), numericAxisName = _getAxisNameByLayout.numericAxisName, cateAxisName = _getAxisNameByLayout.cateAxisName, hasBar = this.constructor.hasBar(graphicalItems), sizeList = hasBar && Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.i)({ barSize: barSize, stackGroups: stackGroups }), formatedItems = []; @@ -2433,9 +2442,9 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, item), _item$props = item.props, dataKey = _item$props.dataKey, childMaxBarSize = _item$props.maxBarSize, numericAxisId = item.props[numericAxisName + "Id"], cateAxisId = item.props[cateAxisName + "Id"], axisObj = axisComponents.reduce(function(result, entry) { var _extends4, axisMap = currentState[entry.axisType + "Map"], id = item.props[entry.axisType + "Id"], axis = axisMap && axisMap[id]; return _extends({}, result, (_extends4 = {}, _defineProperty(_extends4, entry.axisType, axis), - _defineProperty(_extends4, entry.axisType + "Ticks", Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.u)(axis)), + _defineProperty(_extends4, entry.axisType + "Ticks", Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.u)(axis)), _extends4)); - }, {}), cateAxis = axisObj[cateAxisName], cateTicks = axisObj[cateAxisName + "Ticks"], stackedData = stackGroups && stackGroups[numericAxisId] && stackGroups[numericAxisId].hasStack && Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.t)(item, stackGroups[numericAxisId].stackGroups), bandSize = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.g)(cateAxis, cateTicks), maxBarSize = __WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(childMaxBarSize) ? globalMaxBarSize : childMaxBarSize, barPosition = hasBar && Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.h)({ + }, {}), cateAxis = axisObj[cateAxisName], cateTicks = axisObj[cateAxisName + "Ticks"], stackedData = stackGroups && stackGroups[numericAxisId] && stackGroups[numericAxisId].hasStack && Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.t)(item, stackGroups[numericAxisId].stackGroups), bandSize = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.g)(cateAxis, cateTicks), maxBarSize = __WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(childMaxBarSize) ? globalMaxBarSize : childMaxBarSize, barPosition = hasBar && Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.h)({ barGap: barGap, barCategoryGap: barCategoryGap, bandSize: bandSize, @@ -2457,13 +2466,13 @@ var _bundleJs = []byte((((((((((`!function(modules) { layout: layout, dataStartIndex: dataStartIndex, dataEndIndex: dataEndIndex, - onItemMouseLeave: Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.d)(_this3.handleItemMouseLeave, null, item.props.onMouseLeave), - onItemMouseEnter: Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.d)(_this3.handleItemMouseEnter, null, item.props.onMouseEnter) + onItemMouseLeave: Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.d)(_this3.handleItemMouseLeave, null, item.props.onMouseLeave), + onItemMouseEnter: Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.d)(_this3.handleItemMouseEnter, null, item.props.onMouseEnter) })), (_extends5 = { key: item.key || "item-" + index }, _defineProperty(_extends5, numericAxisName, axisObj[numericAxisName]), _defineProperty(_extends5, cateAxisName, axisObj[cateAxisName]), _defineProperty(_extends5, "animationId", updateId), _extends5)), - childIndex: Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.o)(item, props.children), + childIndex: Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.o)(item, props.children), item: item }); } @@ -2488,9 +2497,9 @@ var _bundleJs = []byte((((((((((`!function(modules) { var layout = this.props.layout, _state6 = this.state, activeCoordinate = _state6.activeCoordinate, offset = _state6.offset, x1 = void 0, y1 = void 0, x2 = void 0, y2 = void 0; if ("horizontal" === layout) x1 = activeCoordinate.x, x2 = x1, y1 = offset.top, y2 = offset.top + offset.height; else if ("vertical" === layout) y1 = activeCoordinate.y, - y2 = y1, x1 = offset.left, x2 = offset.left + offset.width; else if (!__WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(activeCoordinate.cx) || !__WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(activeCoordinate.cy)) { + y2 = y1, x1 = offset.left, x2 = offset.left + offset.width; else if (!__WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(activeCoordinate.cx) || !__WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(activeCoordinate.cy)) { if ("centric" !== layout) { - var _cx = activeCoordinate.cx, _cy = activeCoordinate.cy, radius = activeCoordinate.radius, startAngle = activeCoordinate.startAngle, endAngle = activeCoordinate.endAngle, startPoint = Object(__WEBPACK_IMPORTED_MODULE_24__util_PolarUtils__.e)(_cx, _cy, radius, startAngle), endPoint = Object(__WEBPACK_IMPORTED_MODULE_24__util_PolarUtils__.e)(_cx, _cy, radius, endAngle); + var _cx = activeCoordinate.cx, _cy = activeCoordinate.cy, radius = activeCoordinate.radius, startAngle = activeCoordinate.startAngle, endAngle = activeCoordinate.endAngle, startPoint = Object(__WEBPACK_IMPORTED_MODULE_23__util_PolarUtils__.e)(_cx, _cy, radius, startAngle), endPoint = Object(__WEBPACK_IMPORTED_MODULE_23__util_PolarUtils__.e)(_cx, _cy, radius, endAngle); return { points: [ startPoint, endPoint ], cx: _cx, @@ -2500,7 +2509,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { endAngle: endAngle }; } - var cx = activeCoordinate.cx, cy = activeCoordinate.cy, innerRadius = activeCoordinate.innerRadius, outerRadius = activeCoordinate.outerRadius, angle = activeCoordinate.angle, innerPoint = Object(__WEBPACK_IMPORTED_MODULE_24__util_PolarUtils__.e)(cx, cy, innerRadius, angle), outerPoint = Object(__WEBPACK_IMPORTED_MODULE_24__util_PolarUtils__.e)(cx, cy, outerRadius, angle); + var cx = activeCoordinate.cx, cy = activeCoordinate.cy, innerRadius = activeCoordinate.innerRadius, outerRadius = activeCoordinate.outerRadius, angle = activeCoordinate.angle, innerPoint = Object(__WEBPACK_IMPORTED_MODULE_23__util_PolarUtils__.e)(cx, cy, innerRadius, angle), outerPoint = Object(__WEBPACK_IMPORTED_MODULE_23__util_PolarUtils__.e)(cx, cy, outerRadius, angle); x1 = innerPoint.x, y1 = innerPoint.y, x2 = outerPoint.x, y2 = outerPoint.y; } return [ { @@ -2547,8 +2556,8 @@ var _bundleJs = []byte((((((((((`!function(modules) { } var _state7 = this.state, angleAxisMap = _state7.angleAxisMap, radiusAxisMap = _state7.radiusAxisMap; if (angleAxisMap && radiusAxisMap) { - var angleAxis = Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.b)(angleAxisMap); - return Object(__WEBPACK_IMPORTED_MODULE_24__util_PolarUtils__.d)({ + var angleAxis = Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.b)(angleAxisMap); + return Object(__WEBPACK_IMPORTED_MODULE_23__util_PolarUtils__.d)({ x: x, y: y }, angleAxis); @@ -2558,22 +2567,22 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, { key: "parseEventsOfWrapper", value: function() { - var children = this.props.children, tooltipItem = Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.i)(children, __WEBPACK_IMPORTED_MODULE_11__component_Tooltip__.a), tooltipEvents = tooltipItem && "axis" === eventType ? { + var children = this.props.children, tooltipItem = Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.i)(children, __WEBPACK_IMPORTED_MODULE_10__component_Tooltip__.a), tooltipEvents = tooltipItem && "axis" === eventType ? { onMouseEnter: this.handleMouseEnter, onMouseMove: this.handleMouseMove, onMouseLeave: this.handleMouseLeave, onTouchMove: this.handleTouchMove - } : {}, outerEvents = Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.e)(this.props, this.handleOuterEvent); + } : {}, outerEvents = Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.e)(this.props, this.handleOuterEvent); return _extends({}, outerEvents, tooltipEvents); } }, { key: "updateStateOfAxisMapsOffsetAndStackGroups", value: function(_ref5) { var _this4 = this, props = _ref5.props, dataStartIndex = _ref5.dataStartIndex, dataEndIndex = _ref5.dataEndIndex, updateId = _ref5.updateId; - if (!Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.q)({ + if (!Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.q)({ props: props })) return null; - var children = props.children, layout = props.layout, stackOffset = props.stackOffset, data = props.data, reverseStackOrder = props.reverseStackOrder, _getAxisNameByLayout2 = this.getAxisNameByLayout(layout), numericAxisName = _getAxisNameByLayout2.numericAxisName, cateAxisName = _getAxisNameByLayout2.cateAxisName, graphicalItems = Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.h)(children, GraphicalChild), stackGroups = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.s)(data, graphicalItems, numericAxisName + "Id", cateAxisName + "Id", stackOffset, reverseStackOrder), axisObj = axisComponents.reduce(function(result, entry) { + var children = props.children, layout = props.layout, stackOffset = props.stackOffset, data = props.data, reverseStackOrder = props.reverseStackOrder, _getAxisNameByLayout2 = this.getAxisNameByLayout(layout), numericAxisName = _getAxisNameByLayout2.numericAxisName, cateAxisName = _getAxisNameByLayout2.cateAxisName, graphicalItems = Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.h)(children, GraphicalChild), stackGroups = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.s)(data, graphicalItems, numericAxisName + "Id", cateAxisName + "Id", stackOffset, reverseStackOrder), axisObj = axisComponents.reduce(function(result, entry) { var name = entry.axisType + "Map"; return _extends({}, result, _defineProperty({}, name, _this4.getAxisMap(props, _extends({}, entry, { graphicalItems: graphicalItems, @@ -2606,19 +2615,19 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, { key: "addListener", value: function() { - __WEBPACK_IMPORTED_MODULE_26__util_Events__.b.on(__WEBPACK_IMPORTED_MODULE_26__util_Events__.a, this.handleReceiveSyncEvent), - __WEBPACK_IMPORTED_MODULE_26__util_Events__.b.setMaxListeners && __WEBPACK_IMPORTED_MODULE_26__util_Events__.b._maxListeners && __WEBPACK_IMPORTED_MODULE_26__util_Events__.b.setMaxListeners(__WEBPACK_IMPORTED_MODULE_26__util_Events__.b._maxListeners + 1); + __WEBPACK_IMPORTED_MODULE_25__util_Events__.b.on(__WEBPACK_IMPORTED_MODULE_25__util_Events__.a, this.handleReceiveSyncEvent), + __WEBPACK_IMPORTED_MODULE_25__util_Events__.b.setMaxListeners && __WEBPACK_IMPORTED_MODULE_25__util_Events__.b._maxListeners && __WEBPACK_IMPORTED_MODULE_25__util_Events__.b.setMaxListeners(__WEBPACK_IMPORTED_MODULE_25__util_Events__.b._maxListeners + 1); } }, { key: "removeListener", value: function() { - __WEBPACK_IMPORTED_MODULE_26__util_Events__.b.removeListener(__WEBPACK_IMPORTED_MODULE_26__util_Events__.a, this.handleReceiveSyncEvent), - __WEBPACK_IMPORTED_MODULE_26__util_Events__.b.setMaxListeners && __WEBPACK_IMPORTED_MODULE_26__util_Events__.b._maxListeners && __WEBPACK_IMPORTED_MODULE_26__util_Events__.b.setMaxListeners(__WEBPACK_IMPORTED_MODULE_26__util_Events__.b._maxListeners - 1); + __WEBPACK_IMPORTED_MODULE_25__util_Events__.b.removeListener(__WEBPACK_IMPORTED_MODULE_25__util_Events__.a, this.handleReceiveSyncEvent), + __WEBPACK_IMPORTED_MODULE_25__util_Events__.b.setMaxListeners && __WEBPACK_IMPORTED_MODULE_25__util_Events__.b._maxListeners && __WEBPACK_IMPORTED_MODULE_25__util_Events__.b.setMaxListeners(__WEBPACK_IMPORTED_MODULE_25__util_Events__.b._maxListeners - 1); } }, { key: "calculateOffset", value: function(_ref6) { - var props = _ref6.props, graphicalItems = _ref6.graphicalItems, _ref6$xAxisMap = _ref6.xAxisMap, xAxisMap = void 0 === _ref6$xAxisMap ? {} : _ref6$xAxisMap, _ref6$yAxisMap = _ref6.yAxisMap, yAxisMap = void 0 === _ref6$yAxisMap ? {} : _ref6$yAxisMap, width = props.width, height = props.height, children = props.children, margin = props.margin || {}, brushItem = Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.i)(children, __WEBPACK_IMPORTED_MODULE_20__cartesian_Brush__.a), legendItem = Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.i)(children, __WEBPACK_IMPORTED_MODULE_12__component_Legend__.a), offsetH = Object.keys(yAxisMap).reduce(function(result, id) { + var props = _ref6.props, graphicalItems = _ref6.graphicalItems, _ref6$xAxisMap = _ref6.xAxisMap, xAxisMap = void 0 === _ref6$xAxisMap ? {} : _ref6$xAxisMap, _ref6$yAxisMap = _ref6.yAxisMap, yAxisMap = void 0 === _ref6$yAxisMap ? {} : _ref6$yAxisMap, width = props.width, height = props.height, children = props.children, margin = props.margin || {}, brushItem = Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.i)(children, __WEBPACK_IMPORTED_MODULE_19__cartesian_Brush__.a), legendItem = Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.i)(children, __WEBPACK_IMPORTED_MODULE_11__component_Legend__.a), offsetH = Object.keys(yAxisMap).reduce(function(result, id) { var entry = yAxisMap[id], orientation = entry.orientation; return entry.mirror || entry.hide ? result : _extends({}, result, _defineProperty({}, orientation, result[orientation] + entry.width)); }, { @@ -2631,10 +2640,10 @@ var _bundleJs = []byte((((((((((`!function(modules) { top: margin.top || 0, bottom: margin.bottom || 0 }), offset = _extends({}, offsetV, offsetH), brushBottom = offset.bottom; - if (brushItem && (offset.bottom += brushItem.props.height || __WEBPACK_IMPORTED_MODULE_20__cartesian_Brush__.a.defaultProps.height), + if (brushItem && (offset.bottom += brushItem.props.height || __WEBPACK_IMPORTED_MODULE_19__cartesian_Brush__.a.defaultProps.height), legendItem && this.legendInstance) { var legendBox = this.legendInstance.getBBox(); - offset = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.a)(offset, graphicalItems, props, legendBox); + offset = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.a)(offset, graphicalItems, props, legendBox); } return _extends({ brushBottom: brushBottom @@ -2647,14 +2656,14 @@ var _bundleJs = []byte((((((((((`!function(modules) { key: "triggerSyncEvent", value: function(data) { var syncId = this.props.syncId; - __WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(syncId) || __WEBPACK_IMPORTED_MODULE_26__util_Events__.b.emit(__WEBPACK_IMPORTED_MODULE_26__util_Events__.a, syncId, this.uniqueChartId, data); + __WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(syncId) || __WEBPACK_IMPORTED_MODULE_25__util_Events__.b.emit(__WEBPACK_IMPORTED_MODULE_25__util_Events__.a, syncId, this.uniqueChartId, data); } }, { key: "filterFormatItem", value: function(item, displayName, childIndex) { for (var formatedGraphicalItems = this.state.formatedGraphicalItems, i = 0, len = formatedGraphicalItems.length; i < len; i++) { var entry = formatedGraphicalItems[i]; - if (entry.item === item || entry.props.key === item.key || displayName === Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.j)(entry.item.type) && childIndex === entry.childIndex) return entry; + if (entry.item === item || entry.props.key === item.key || displayName === Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.j)(entry.item.type) && childIndex === entry.childIndex) return entry; } return null; } @@ -2662,7 +2671,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { key: "renderAxis", value: function(axisOptions, element, displayName, index) { var _props2 = this.props, width = _props2.width, height = _props2.height; - return __WEBPACK_IMPORTED_MODULE_6_react___default.a.createElement(__WEBPACK_IMPORTED_MODULE_19__cartesian_CartesianAxis__.a, _extends({}, axisOptions, { + return __WEBPACK_IMPORTED_MODULE_5_react___default.a.createElement(__WEBPACK_IMPORTED_MODULE_18__cartesian_CartesianAxis__.a, _extends({}, axisOptions, { className: "recharts-" + axisOptions.axisType + " " + axisOptions.axisType, key: element.key || displayName + "-" + index, viewBox: { @@ -2677,7 +2686,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, { key: "renderLegend", value: function() { - var _this5 = this, formatedGraphicalItems = this.state.formatedGraphicalItems, _props3 = this.props, children = _props3.children, width = _props3.width, height = _props3.height, margin = this.props.margin || {}, legendWidth = width - (margin.left || 0) - (margin.right || 0), legendHeight = height - (margin.top || 0) - (margin.bottom || 0), props = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.q)({ + var _this5 = this, formatedGraphicalItems = this.state.formatedGraphicalItems, _props3 = this.props, children = _props3.children, width = _props3.width, height = _props3.height, margin = this.props.margin || {}, legendWidth = width - (margin.left || 0) - (margin.right || 0), legendHeight = height - (margin.top || 0) - (margin.bottom || 0), props = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.q)({ children: children, formatedGraphicalItems: formatedGraphicalItems, legendWidth: legendWidth, @@ -2686,7 +2695,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }); if (!props) return null; var item = props.item, otherProps = _objectWithoutProperties(props, [ "item" ]); - return Object(__WEBPACK_IMPORTED_MODULE_6_react__.cloneElement)(item, _extends({}, otherProps, { + return Object(__WEBPACK_IMPORTED_MODULE_5_react__.cloneElement)(item, _extends({}, otherProps, { chartWidth: width, chartHeight: height, margin: margin, @@ -2699,10 +2708,10 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, { key: "renderTooltip", value: function() { - var children = this.props.children, tooltipItem = Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.i)(children, __WEBPACK_IMPORTED_MODULE_11__component_Tooltip__.a); + var children = this.props.children, tooltipItem = Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.i)(children, __WEBPACK_IMPORTED_MODULE_10__component_Tooltip__.a); if (!tooltipItem) return null; var _state8 = this.state, isTooltipActive = _state8.isTooltipActive, activeCoordinate = _state8.activeCoordinate, activePayload = _state8.activePayload, activeLabel = _state8.activeLabel, offset = _state8.offset; - return Object(__WEBPACK_IMPORTED_MODULE_6_react__.cloneElement)(tooltipItem, { + return Object(__WEBPACK_IMPORTED_MODULE_5_react__.cloneElement)(tooltipItem, { viewBox: _extends({}, offset, { x: offset.left, y: offset.top @@ -2717,8 +2726,8 @@ var _bundleJs = []byte((((((((((`!function(modules) { key: "renderActiveDot", value: function(option, props) { var dot = void 0; - return dot = Object(__WEBPACK_IMPORTED_MODULE_6_react__.isValidElement)(option) ? Object(__WEBPACK_IMPORTED_MODULE_6_react__.cloneElement)(option, props) : __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction___default()(option) ? option(props) : __WEBPACK_IMPORTED_MODULE_6_react___default.a.createElement(__WEBPACK_IMPORTED_MODULE_16__shape_Dot__.a, props), - __WEBPACK_IMPORTED_MODULE_6_react___default.a.createElement(__WEBPACK_IMPORTED_MODULE_10__container_Layer__.a, { + return dot = Object(__WEBPACK_IMPORTED_MODULE_5_react__.isValidElement)(option) ? Object(__WEBPACK_IMPORTED_MODULE_5_react__.cloneElement)(option, props) : __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction___default()(option) ? option(props) : __WEBPACK_IMPORTED_MODULE_5_react___default.a.createElement(__WEBPACK_IMPORTED_MODULE_15__shape_Dot__.a, props), + __WEBPACK_IMPORTED_MODULE_5_react___default.a.createElement(__WEBPACK_IMPORTED_MODULE_9__container_Layer__.a, { className: "recharts-active-dot", key: props.key }, dot); @@ -2732,13 +2741,13 @@ var _bundleJs = []byte((((((((((`!function(modules) { cx: activePoint.x, cy: activePoint.y, r: 4, - fill: Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.r)(item.item), + fill: Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.r)(item.item), strokeWidth: 2, stroke: "#fff", payload: activePoint.payload, value: activePoint.value, key: key + "-activePoint-" + childIndex - }, Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.k)(activeDot), Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.e)(activeDot)); + }, Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.k)(activeDot), Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.e)(activeDot)); return result.push(this.renderActiveDot(activeDot, dotProps, childIndex)), basePoint ? result.push(this.renderActiveDot(activeDot, _extends({}, dotProps, { cx: basePoint.x, cy: basePoint.y, @@ -2749,8 +2758,8 @@ var _bundleJs = []byte((((((((((`!function(modules) { key: "render", value: function() { var _this6 = this; - if (!Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.q)(this)) return null; - var _props4 = this.props, children = _props4.children, className = _props4.className, width = _props4.width, height = _props4.height, style = _props4.style, compact = _props4.compact, others = _objectWithoutProperties(_props4, [ "children", "className", "width", "height", "style", "compact" ]), attrs = Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.k)(others), map = { + if (!Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.q)(this)) return null; + var _props4 = this.props, children = _props4.children, className = _props4.className, width = _props4.width, height = _props4.height, style = _props4.style, compact = _props4.compact, others = _objectWithoutProperties(_props4, [ "children", "className", "width", "height", "style", "compact" ]), attrs = Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.k)(others), map = { CartesianGrid: { handler: this.renderGrid, once: !0 @@ -2810,13 +2819,13 @@ var _bundleJs = []byte((((((((((`!function(modules) { handler: this.renderPolarAxis } }; - if (compact) return __WEBPACK_IMPORTED_MODULE_6_react___default.a.createElement(__WEBPACK_IMPORTED_MODULE_9__container_Surface__.a, _extends({}, attrs, { + if (compact) return __WEBPACK_IMPORTED_MODULE_5_react___default.a.createElement(__WEBPACK_IMPORTED_MODULE_8__container_Surface__.a, _extends({}, attrs, { width: width, height: height - }), Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.p)(children, map)); + }), Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.p)(children, map)); var events = this.parseEventsOfWrapper(); - return __WEBPACK_IMPORTED_MODULE_6_react___default.a.createElement("div", _extends({ - className: __WEBPACK_IMPORTED_MODULE_8_classnames___default()("recharts-wrapper", className), + return __WEBPACK_IMPORTED_MODULE_5_react___default.a.createElement("div", _extends({ + className: __WEBPACK_IMPORTED_MODULE_7_classnames___default()("recharts-wrapper", className), style: _extends({}, style, { position: "relative", cursor: "default", @@ -2827,43 +2836,43 @@ var _bundleJs = []byte((((((((((`!function(modules) { ref: function(node) { _this6.container = node; } - }), __WEBPACK_IMPORTED_MODULE_6_react___default.a.createElement(__WEBPACK_IMPORTED_MODULE_9__container_Surface__.a, _extends({}, attrs, { + }), __WEBPACK_IMPORTED_MODULE_5_react___default.a.createElement(__WEBPACK_IMPORTED_MODULE_8__container_Surface__.a, _extends({}, attrs, { width: width, height: height - }), Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.p)(children, map)), this.renderLegend(), this.renderTooltip()); + }), Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.p)(children, map)), this.renderLegend(), this.renderTooltip()); } } ]), CategoricalChartWrapper; - }(__WEBPACK_IMPORTED_MODULE_6_react__.Component), _class.displayName = chartName, + }(__WEBPACK_IMPORTED_MODULE_5_react__.Component), _class.displayName = chartName, _class.propTypes = _extends({ - syncId: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.oneOfType([ __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.string, __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.number ]), - compact: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.bool, - width: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.number, - height: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.number, - data: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.arrayOf(__WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.object), - layout: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.oneOf([ "horizontal", "vertical" ]), - stackOffset: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.oneOf([ "sign", "expand", "none", "wiggle", "silhouette" ]), - throttleDelay: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.number, - margin: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.shape({ - top: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.number, - right: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.number, - bottom: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.number, - left: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.number + syncId: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.oneOfType([ __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.string, __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.number ]), + compact: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.bool, + width: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.number, + height: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.number, + data: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.arrayOf(__WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.object), + layout: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.oneOf([ "horizontal", "vertical" ]), + stackOffset: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.oneOf([ "sign", "expand", "none", "wiggle", "silhouette" ]), + throttleDelay: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.number, + margin: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.shape({ + top: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.number, + right: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.number, + bottom: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.number, + left: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.number }), - barCategoryGap: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.oneOfType([ __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.number, __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.string ]), - barGap: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.oneOfType([ __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.number, __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.string ]), - barSize: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.oneOfType([ __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.number, __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.string ]), - maxBarSize: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.number, - style: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.object, - className: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.string, - children: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.oneOfType([ __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.arrayOf(__WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.node), __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.node ]), - onClick: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.func, - onMouseLeave: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.func, - onMouseEnter: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.func, - onMouseMove: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.func, - onMouseDown: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.func, - onMouseUp: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.func, - reverseStackOrder: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.bool, - id: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.string + barCategoryGap: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.oneOfType([ __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.number, __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.string ]), + barGap: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.oneOfType([ __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.number, __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.string ]), + barSize: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.oneOfType([ __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.number, __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.string ]), + maxBarSize: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.number, + style: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.object, + className: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.string, + children: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.oneOfType([ __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.arrayOf(__WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.node), __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.node ]), + onClick: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.func, + onMouseLeave: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.func, + onMouseEnter: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.func, + onMouseMove: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.func, + onMouseDown: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.func, + onMouseUp: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.func, + reverseStackOrder: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.bool, + id: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.string }, propTypes), _class.defaultProps = _extends({ layout: "horizontal", stackOffset: "none", @@ -2877,7 +2886,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, reverseStackOrder: !1 }, defaultProps), _class.createDefaultState = function(props) { - var children = props.children, brushItem = Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.i)(children, __WEBPACK_IMPORTED_MODULE_20__cartesian_Brush__.a); + var children = props.children, brushItem = Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.i)(children, __WEBPACK_IMPORTED_MODULE_19__cartesian_Brush__.a); return { chartX: 0, chartY: 0, @@ -2888,7 +2897,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }; }, _class.hasBar = function(graphicalItems) { return !(!graphicalItems || !graphicalItems.length) && graphicalItems.some(function(item) { - var name = Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.j)(item && item.type); + var name = Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.j)(item && item.type); return name && name.indexOf("Bar") >= 0; }); }, _class.getDisplayedData = function(props, _ref8, item) { @@ -2899,7 +2908,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { if (itemsData && itemsData.length > 0) return itemsData; if (item && item.props && item.props.data && item.props.data.length > 0) return item.props.data; var data = props.data; - return data && data.length && Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.g)(dataStartIndex) && Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.g)(dataEndIndex) ? data.slice(dataStartIndex, dataEndIndex + 1) : []; + return data && data.length && Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.g)(dataStartIndex) && Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.g)(dataEndIndex) ? data.slice(dataStartIndex, dataEndIndex + 1) : []; }, _initialiseProps = function() { var _this7 = this; this.handleLegendBBoxUpdate = function(box) { @@ -2916,7 +2925,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { var _props5 = _this7.props, syncId = _props5.syncId, layout = _props5.layout, updateId = _this7.state.updateId; if (syncId === cId && chartId !== _this7.uniqueChartId) { var dataStartIndex = data.dataStartIndex, dataEndIndex = data.dataEndIndex; - if (__WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(data.dataStartIndex) && __WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(data.dataEndIndex)) if (__WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(data.activeTooltipIndex)) _this7.setState(data); else { + if (__WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(data.dataStartIndex) && __WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(data.dataEndIndex)) if (__WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(data.activeTooltipIndex)) _this7.setState(data); else { var chartX = data.chartX, chartY = data.chartY, activeTooltipIndex = data.activeTooltipIndex, _state10 = _this7.state, offset = _state10.offset, tooltipTicks = _state10.tooltipTicks; if (!offset) return; var viewBox = _extends({}, offset, { @@ -2945,15 +2954,17 @@ var _bundleJs = []byte((((((((((`!function(modules) { var startIndex = _ref9.startIndex, endIndex = _ref9.endIndex; if (startIndex !== _this7.state.dataStartIndex || endIndex !== _this7.state.dataEndIndex) { var updateId = _this7.state.updateId; - _this7.setState(_extends({ - dataStartIndex: startIndex, - dataEndIndex: endIndex - }, _this7.updateStateOfAxisMapsOffsetAndStackGroups({ - props: _this7.props, - dataStartIndex: startIndex, - dataEndIndex: endIndex, - updateId: updateId - }))), _this7.triggerSyncEvent({ + _this7.setState(function() { + return _extends({ + dataStartIndex: startIndex, + dataEndIndex: endIndex + }, _this7.updateStateOfAxisMapsOffsetAndStackGroups({ + props: _this7.props, + dataStartIndex: startIndex, + dataEndIndex: endIndex, + updateId: updateId + })); + }), _this7.triggerSyncEvent({ dataStartIndex: startIndex, dataEndIndex: endIndex }); @@ -2974,18 +2985,22 @@ var _bundleJs = []byte((((((((((`!function(modules) { }; _this7.setState(nextState), _this7.triggerSyncEvent(nextState), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction___default()(onMouseMove) && onMouseMove(nextState, e); }, this.handleItemMouseEnter = function(el) { - _this7.setState({ - isTooltipActive: !0, - activeItem: el, - activePayload: el.tooltipPayload, - activeCoordinate: el.tooltipPosition || { - x: el.cx, - y: el.cy - } + _this7.setState(function() { + return { + isTooltipActive: !0, + activeItem: el, + activePayload: el.tooltipPayload, + activeCoordinate: el.tooltipPosition || { + x: el.cx, + y: el.cy + } + }; }); }, this.handleItemMouseLeave = function() { - _this7.setState({ - isTooltipActive: !1 + _this7.setState(function() { + return { + isTooltipActive: !1 + }; }); }, this.handleMouseMove = function(e) { e && __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction___default()(e.persist) && e.persist(), @@ -2996,7 +3011,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }; _this7.setState(nextState), _this7.triggerSyncEvent(nextState), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction___default()(onMouseLeave) && onMouseLeave(nextState, e); }, this.handleOuterEvent = function(e) { - var eventName = Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.l)(e); + var eventName = Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.l)(e); if (eventName && __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction___default()(_this7.props[eventName])) { var mouse = _this7.getMouseInfo(e); (0, _this7.props[eventName])(mouse, e); @@ -3020,8 +3035,8 @@ var _bundleJs = []byte((((((((((`!function(modules) { null != e.changedTouches && e.changedTouches.length > 0 && _this7.handleMouseMove(e.changedTouches[0]); }, this.verticalCoordinatesGenerator = function(_ref10) { var xAxis = _ref10.xAxis, width = _ref10.width, height = _ref10.height, offset = _ref10.offset; - return Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.m)(__WEBPACK_IMPORTED_MODULE_19__cartesian_CartesianAxis__.a.getTicks(_extends({}, __WEBPACK_IMPORTED_MODULE_19__cartesian_CartesianAxis__.a.defaultProps, xAxis, { - ticks: Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.u)(xAxis, !0), + return Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.m)(__WEBPACK_IMPORTED_MODULE_18__cartesian_CartesianAxis__.a.getTicks(_extends({}, __WEBPACK_IMPORTED_MODULE_18__cartesian_CartesianAxis__.a.defaultProps, xAxis, { + ticks: Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.u)(xAxis, !0), viewBox: { x: 0, y: 0, @@ -3031,8 +3046,8 @@ var _bundleJs = []byte((((((((((`!function(modules) { })), offset.left, offset.left + offset.width); }, this.horizontalCoordinatesGenerator = function(_ref11) { var yAxis = _ref11.yAxis, width = _ref11.width, height = _ref11.height, offset = _ref11.offset; - return Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.m)(__WEBPACK_IMPORTED_MODULE_19__cartesian_CartesianAxis__.a.getTicks(_extends({}, __WEBPACK_IMPORTED_MODULE_19__cartesian_CartesianAxis__.a.defaultProps, yAxis, { - ticks: Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.u)(yAxis, !0), + return Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.m)(__WEBPACK_IMPORTED_MODULE_18__cartesian_CartesianAxis__.a.getTicks(_extends({}, __WEBPACK_IMPORTED_MODULE_18__cartesian_CartesianAxis__.a.defaultProps, yAxis, { + ticks: Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.u)(yAxis, !0), viewBox: { x: 0, y: 0, @@ -3041,23 +3056,23 @@ var _bundleJs = []byte((((((((((`!function(modules) { } })), offset.top, offset.top + offset.height); }, this.axesTicksGenerator = function(axis) { - return Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.u)(axis, !0); + return Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.u)(axis, !0); }, this.tooltipTicksGenerator = function(axisMap) { - var axis = Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.b)(axisMap), tooltipTicks = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.u)(axis, !1, !0); + var axis = Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.b)(axisMap), tooltipTicks = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.u)(axis, !1, !0); return { tooltipTicks: tooltipTicks, orderedTooltipTicks: __WEBPACK_IMPORTED_MODULE_0_lodash_sortBy___default()(tooltipTicks, function(o) { return o.coordinate; }), tooltipAxis: axis, - tooltipAxisBandSize: Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.g)(axis) + tooltipAxisBandSize: Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.g)(axis) }; }, this.renderCursor = function(element) { var _state11 = _this7.state, isTooltipActive = _state11.isTooltipActive, activeCoordinate = _state11.activeCoordinate, activePayload = _state11.activePayload, offset = _state11.offset; if (!(element && element.props.cursor && isTooltipActive && activeCoordinate)) return null; - var layout = _this7.props.layout, restProps = void 0, cursorComp = __WEBPACK_IMPORTED_MODULE_13__shape_Curve__.a; - if ("ScatterChart" === chartName) restProps = activeCoordinate, cursorComp = __WEBPACK_IMPORTED_MODULE_14__shape_Cross__.a; else if ("BarChart" === chartName) restProps = _this7.getCursorRectangle(), - cursorComp = __WEBPACK_IMPORTED_MODULE_17__shape_Rectangle__.a; else if ("radial" === layout) { + var layout = _this7.props.layout, restProps = void 0, cursorComp = __WEBPACK_IMPORTED_MODULE_12__shape_Curve__.a; + if ("ScatterChart" === chartName) restProps = activeCoordinate, cursorComp = __WEBPACK_IMPORTED_MODULE_13__shape_Cross__.a; else if ("BarChart" === chartName) restProps = _this7.getCursorRectangle(), + cursorComp = __WEBPACK_IMPORTED_MODULE_16__shape_Rectangle__.a; else if ("radial" === layout) { var _getCursorPoints = _this7.getCursorPoints(), cx = _getCursorPoints.cx, cy = _getCursorPoints.cy, radius = _getCursorPoints.radius, startAngle = _getCursorPoints.startAngle, endAngle = _getCursorPoints.endAngle; restProps = { cx: cx, @@ -3066,24 +3081,24 @@ var _bundleJs = []byte((((((((((`!function(modules) { endAngle: endAngle, innerRadius: radius, outerRadius: radius - }, cursorComp = __WEBPACK_IMPORTED_MODULE_15__shape_Sector__.a; + }, cursorComp = __WEBPACK_IMPORTED_MODULE_14__shape_Sector__.a; } else restProps = { points: _this7.getCursorPoints() - }, cursorComp = __WEBPACK_IMPORTED_MODULE_13__shape_Curve__.a; + }, cursorComp = __WEBPACK_IMPORTED_MODULE_12__shape_Curve__.a; var key = element.key || "_recharts-cursor", cursorProps = _extends({ stroke: "#ccc" - }, offset, restProps, Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.k)(element.props.cursor), { + }, offset, restProps, Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.k)(element.props.cursor), { payload: activePayload, key: key, className: "recharts-tooltip-cursor" }); - return Object(__WEBPACK_IMPORTED_MODULE_6_react__.isValidElement)(element.props.cursor) ? Object(__WEBPACK_IMPORTED_MODULE_6_react__.cloneElement)(element.props.cursor, cursorProps) : Object(__WEBPACK_IMPORTED_MODULE_6_react__.createElement)(cursorComp, cursorProps); + return Object(__WEBPACK_IMPORTED_MODULE_5_react__.isValidElement)(element.props.cursor) ? Object(__WEBPACK_IMPORTED_MODULE_5_react__.cloneElement)(element.props.cursor, cursorProps) : Object(__WEBPACK_IMPORTED_MODULE_5_react__.createElement)(cursorComp, cursorProps); }, this.renderPolarAxis = function(element, displayName, index) { var axisType = element.type.axisType, axisMap = _this7.state[axisType + "Map"], axisOption = axisMap[element.props[axisType + "Id"]]; - return Object(__WEBPACK_IMPORTED_MODULE_6_react__.cloneElement)(element, _extends({}, axisOption, { + return Object(__WEBPACK_IMPORTED_MODULE_5_react__.cloneElement)(element, _extends({}, axisOption, { className: axisType, key: element.key || displayName + "-" + index, - ticks: Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.u)(axisOption, !0) + ticks: Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.u)(axisOption, !0) })); }, this.renderXAxis = function(element, displayName, index) { var xAxisMap = _this7.state.xAxisMap, axisObj = xAxisMap[element.props.xAxisId]; @@ -3092,13 +3107,13 @@ var _bundleJs = []byte((((((((((`!function(modules) { var yAxisMap = _this7.state.yAxisMap, axisObj = yAxisMap[element.props.yAxisId]; return _this7.renderAxis(axisObj, element, displayName, index); }, this.renderGrid = function(element) { - var _state12 = _this7.state, xAxisMap = _state12.xAxisMap, yAxisMap = _state12.yAxisMap, offset = _state12.offset, _props6 = _this7.props, width = _props6.width, height = _props6.height, xAxis = Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.b)(xAxisMap), yAxis = Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.b)(yAxisMap), props = element.props || {}; - return Object(__WEBPACK_IMPORTED_MODULE_6_react__.cloneElement)(element, { + var _state12 = _this7.state, xAxisMap = _state12.xAxisMap, yAxisMap = _state12.yAxisMap, offset = _state12.offset, _props6 = _this7.props, width = _props6.width, height = _props6.height, xAxis = Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.b)(xAxisMap), yAxis = Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.b)(yAxisMap), props = element.props || {}; + return Object(__WEBPACK_IMPORTED_MODULE_5_react__.cloneElement)(element, { key: element.key || "grid", - x: Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.g)(props.x) ? props.x : offset.left, - y: Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.g)(props.y) ? props.y : offset.top, - width: Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.g)(props.width) ? props.width : offset.width, - height: Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.g)(props.height) ? props.height : offset.height, + x: Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.g)(props.x) ? props.x : offset.left, + y: Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.g)(props.y) ? props.y : offset.top, + width: Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.g)(props.width) ? props.width : offset.width, + height: Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.g)(props.height) ? props.height : offset.height, xAxis: xAxis, yAxis: yAxis, offset: offset, @@ -3108,12 +3123,12 @@ var _bundleJs = []byte((((((((((`!function(modules) { horizontalCoordinatesGenerator: _this7.horizontalCoordinatesGenerator }); }, this.renderPolarGrid = function(element) { - var _state13 = _this7.state, radiusAxisMap = _state13.radiusAxisMap, angleAxisMap = _state13.angleAxisMap, radiusAxis = Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.b)(radiusAxisMap), angleAxis = Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.b)(angleAxisMap), cx = angleAxis.cx, cy = angleAxis.cy, innerRadius = angleAxis.innerRadius, outerRadius = angleAxis.outerRadius; - return Object(__WEBPACK_IMPORTED_MODULE_6_react__.cloneElement)(element, { - polarAngles: Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.u)(angleAxis, !0).map(function(entry) { + var _state13 = _this7.state, radiusAxisMap = _state13.radiusAxisMap, angleAxisMap = _state13.angleAxisMap, radiusAxis = Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.b)(radiusAxisMap), angleAxis = Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.b)(angleAxisMap), cx = angleAxis.cx, cy = angleAxis.cy, innerRadius = angleAxis.innerRadius, outerRadius = angleAxis.outerRadius; + return Object(__WEBPACK_IMPORTED_MODULE_5_react__.cloneElement)(element, { + polarAngles: Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.u)(angleAxis, !0).map(function(entry) { return entry.coordinate; }), - polarRadius: Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.u)(radiusAxis, !0).map(function(entry) { + polarRadius: Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.u)(radiusAxis, !0).map(function(entry) { return entry.coordinate; }), cx: cx, @@ -3124,13 +3139,13 @@ var _bundleJs = []byte((((((((((`!function(modules) { }); }, this.renderBrush = function(element) { var _props7 = _this7.props, margin = _props7.margin, data = _props7.data, _state14 = _this7.state, offset = _state14.offset, dataStartIndex = _state14.dataStartIndex, dataEndIndex = _state14.dataEndIndex, updateId = _state14.updateId; - return Object(__WEBPACK_IMPORTED_MODULE_6_react__.cloneElement)(element, { + return Object(__WEBPACK_IMPORTED_MODULE_5_react__.cloneElement)(element, { key: element.key || "_recharts-brush", - onChange: Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.d)(_this7.handleBrushChange, null, element.props.onChange), + onChange: Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.d)(_this7.handleBrushChange, null, element.props.onChange), data: data, - x: Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.g)(element.props.x) ? element.props.x : offset.left, - y: Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.g)(element.props.y) ? element.props.y : offset.top + offset.height + offset.brushBottom - (margin.bottom || 0), - width: Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.g)(element.props.width) ? element.props.width : offset.width, + x: Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.g)(element.props.x) ? element.props.x : offset.left, + y: Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.g)(element.props.y) ? element.props.y : offset.top + offset.height + offset.brushBottom - (margin.bottom || 0), + width: Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.g)(element.props.width) ? element.props.width : offset.width, startIndex: dataStartIndex, endIndex: dataEndIndex, updateId: "brush-" + updateId @@ -3138,7 +3153,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, this.renderReferenceElement = function(element, displayName, index) { if (!element) return null; var _state15 = _this7.state, xAxisMap = _state15.xAxisMap, yAxisMap = _state15.yAxisMap, offset = _state15.offset, _element$props = element.props, xAxisId = _element$props.xAxisId, yAxisId = _element$props.yAxisId; - return Object(__WEBPACK_IMPORTED_MODULE_6_react__.cloneElement)(element, { + return Object(__WEBPACK_IMPORTED_MODULE_5_react__.cloneElement)(element, { key: element.key || displayName + "-" + index, xAxis: xAxisMap[xAxisId], yAxis: yAxisMap[yAxisId], @@ -3152,12 +3167,12 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, this.renderGraphicChild = function(element, displayName, index) { var item = _this7.filterFormatItem(element, displayName, index); if (!item) return null; - var graphicalItem = Object(__WEBPACK_IMPORTED_MODULE_6_react__.cloneElement)(element, item.props), _state16 = _this7.state, isTooltipActive = _state16.isTooltipActive, tooltipAxis = _state16.tooltipAxis, activeTooltipIndex = _state16.activeTooltipIndex, activeLabel = _state16.activeLabel, children = _this7.props.children, tooltipItem = Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.i)(children, __WEBPACK_IMPORTED_MODULE_11__component_Tooltip__.a), _item$props2 = item.props, points = _item$props2.points, isRange = _item$props2.isRange, baseLine = _item$props2.baseLine, _item$item$props2 = item.item.props, activeDot = _item$item$props2.activeDot; + var graphicalItem = Object(__WEBPACK_IMPORTED_MODULE_5_react__.cloneElement)(element, item.props), _state16 = _this7.state, isTooltipActive = _state16.isTooltipActive, tooltipAxis = _state16.tooltipAxis, activeTooltipIndex = _state16.activeTooltipIndex, activeLabel = _state16.activeLabel, children = _this7.props.children, tooltipItem = Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.i)(children, __WEBPACK_IMPORTED_MODULE_10__component_Tooltip__.a), _item$props2 = item.props, points = _item$props2.points, isRange = _item$props2.isRange, baseLine = _item$props2.baseLine, _item$item$props2 = item.item.props, activeDot = _item$item$props2.activeDot; if (!_item$item$props2.hide && isTooltipActive && tooltipItem && activeDot && activeTooltipIndex >= 0) { var activePoint = void 0, basePoint = void 0; - if (tooltipAxis.dataKey && !tooltipAxis.allowDuplicatedCategory ? (activePoint = Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.a)(points, "payload." + tooltipAxis.dataKey, activeLabel), - basePoint = isRange && baseLine && Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.a)(baseLine, "payload." + tooltipAxis.dataKey, activeLabel)) : (activePoint = points[activeTooltipIndex], - basePoint = isRange && baseLine && baseLine[activeTooltipIndex]), !__WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(activePoint)) return [ graphicalItem ].concat(_toConsumableArray(_this7.renderActivePoints({ + if (tooltipAxis.dataKey && !tooltipAxis.allowDuplicatedCategory ? (activePoint = Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.a)(points, "payload." + tooltipAxis.dataKey, activeLabel), + basePoint = isRange && baseLine && Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.a)(baseLine, "payload." + tooltipAxis.dataKey, activeLabel)) : (activePoint = points[activeTooltipIndex], + basePoint = isRange && baseLine && baseLine[activeTooltipIndex]), !__WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(activePoint)) return [ graphicalItem ].concat(_toConsumableArray(_this7.renderActivePoints({ item: item, activePoint: activePoint, basePoint: basePoint, @@ -3171,7 +3186,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }; __webpack_exports__.a = generateCategoricalChart; }, function(module, exports, __webpack_require__) { - var aFunction = __webpack_require__(207); + var aFunction = __webpack_require__(206); module.exports = function(fn, that, length) { if (aFunction(fn), void 0 === that) return fn; switch (length) { @@ -3268,7 +3283,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { Object.defineProperty(exports, "__esModule", { value: !0 }); - var _typeof2 = __webpack_require__(99), _typeof3 = _interopRequireDefault(_typeof2), _keys = __webpack_require__(36), _keys2 = _interopRequireDefault(_keys); + var _typeof2 = __webpack_require__(100), _typeof3 = _interopRequireDefault(_typeof2), _keys = __webpack_require__(41), _keys2 = _interopRequireDefault(_keys); exports.capitalizeFirstLetter = capitalizeFirstLetter, exports.contains = contains, exports.findIndex = findIndex, exports.find = find, exports.createChainedFunction = createChainedFunction; var _warning = __webpack_require__(11), _warning2 = _interopRequireDefault(_warning); @@ -3278,7 +3293,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { var value = getValue(object, key); return baseIsNative(value) ? value : void 0; } - var baseIsNative = __webpack_require__(551), getValue = __webpack_require__(554); + var baseIsNative = __webpack_require__(562), getValue = __webpack_require__(565); module.exports = getNative; }, function(module, __webpack_exports__, __webpack_require__) { "use strict"; @@ -3312,7 +3327,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { } }), superClass && (Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass); } - var _class, _temp2, __WEBPACK_IMPORTED_MODULE_0_lodash_isNil__ = __webpack_require__(20), __WEBPACK_IMPORTED_MODULE_0_lodash_isNil___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_isNil__), __WEBPACK_IMPORTED_MODULE_1_react__ = __webpack_require__(0), __WEBPACK_IMPORTED_MODULE_1_react___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_react__), __WEBPACK_IMPORTED_MODULE_2_prop_types__ = __webpack_require__(1), __WEBPACK_IMPORTED_MODULE_2_prop_types___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_prop_types__), __WEBPACK_IMPORTED_MODULE_3_reduce_css_calc__ = __webpack_require__(676), __WEBPACK_IMPORTED_MODULE_3_reduce_css_calc___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_reduce_css_calc__), __WEBPACK_IMPORTED_MODULE_4_classnames__ = __webpack_require__(3), __WEBPACK_IMPORTED_MODULE_4_classnames___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_classnames__), __WEBPACK_IMPORTED_MODULE_5__util_DataUtils__ = __webpack_require__(9), __WEBPACK_IMPORTED_MODULE_6__util_ReactUtils__ = __webpack_require__(4), __WEBPACK_IMPORTED_MODULE_7__util_DOMUtils__ = __webpack_require__(185), _extends = Object.assign || function(target) { + var _class, _temp2, __WEBPACK_IMPORTED_MODULE_0_lodash_isNil__ = __webpack_require__(20), __WEBPACK_IMPORTED_MODULE_0_lodash_isNil___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_isNil__), __WEBPACK_IMPORTED_MODULE_1_react__ = __webpack_require__(0), __WEBPACK_IMPORTED_MODULE_1_react___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_react__), __WEBPACK_IMPORTED_MODULE_2_prop_types__ = __webpack_require__(1), __WEBPACK_IMPORTED_MODULE_2_prop_types___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_prop_types__), __WEBPACK_IMPORTED_MODULE_3_reduce_css_calc__ = __webpack_require__(686), __WEBPACK_IMPORTED_MODULE_3_reduce_css_calc___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_reduce_css_calc__), __WEBPACK_IMPORTED_MODULE_4_classnames__ = __webpack_require__(3), __WEBPACK_IMPORTED_MODULE_4_classnames___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_classnames__), __WEBPACK_IMPORTED_MODULE_5__util_DataUtils__ = __webpack_require__(9), __WEBPACK_IMPORTED_MODULE_6__util_ReactUtils__ = __webpack_require__(4), __WEBPACK_IMPORTED_MODULE_7__util_DOMUtils__ = __webpack_require__(184), _extends = Object.assign || function(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) Object.prototype.hasOwnProperty.call(source, key) && (target[key] = source[key]); @@ -3532,12 +3547,12 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, _class = _temp)) || _class; __webpack_exports__.a = Dot; }, function(module, exports, __webpack_require__) { - var IObject = __webpack_require__(134), defined = __webpack_require__(136); + var IObject = __webpack_require__(135), defined = __webpack_require__(137); module.exports = function(it) { return IObject(defined(it)); }; }, function(module, exports, __webpack_require__) { - var defined = __webpack_require__(136); + var defined = __webpack_require__(137); module.exports = function(it) { return Object(defined(it)); }; @@ -3576,7 +3591,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { return protoProps && defineProperties(Constructor.prototype, protoProps), staticProps && defineProperties(Constructor, staticProps), Constructor; }; - }(), _warning = __webpack_require__(11), _warning2 = _interopRequireDefault(_warning), _toCss = __webpack_require__(152), _toCss2 = _interopRequireDefault(_toCss), _toCssValue = __webpack_require__(153), _toCssValue2 = _interopRequireDefault(_toCssValue), StyleRule = function() { + }(), _warning = __webpack_require__(11), _warning2 = _interopRequireDefault(_warning), _toCss = __webpack_require__(153), _toCss2 = _interopRequireDefault(_toCss), _toCssValue = __webpack_require__(105), _toCssValue2 = _interopRequireDefault(_toCssValue), StyleRule = function() { function StyleRule(key, style, options) { _classCallCheck(this, StyleRule), this.type = "style", this.isProcessed = !1; var sheet = options.sheet, Renderer = options.Renderer, selector = options.selector; @@ -3638,11 +3653,69 @@ var _bundleJs = []byte((((((((((`!function(modules) { } ]), StyleRule; }(); exports.default = StyleRule; +}, function(module, exports, __webpack_require__) { + "use strict"; + Object.defineProperty(exports, "__esModule", { + value: !0 + }); + var _extends = Object.assign || function(target) { + for (var i = 1; i < arguments.length; i++) { + var source = arguments[i]; + for (var key in source) Object.prototype.hasOwnProperty.call(source, key) && (target[key] = source[key]); + } + return target; + }, menuSkeletons = [ { + id: "home", + menu: { + title: "Home", + icon: "home" + } + }, { + id: "chain", + menu: { + title: "Chain", + icon: "link" + } + }, { + id: "txpool", + menu: { + title: "TxPool", + icon: "credit-card" + } + }, { + id: "network", + menu: { + title: "Network", + icon: "globe" + } + }, { + id: "system", + menu: { + title: "System", + icon: "tachometer" + } + }, { + id: "logs", + menu: { + title: "Logs", + icon: "list" + } + } ]; + exports.MENU = new Map(menuSkeletons.map(function(_ref) { + var id = _ref.id, menu = _ref.menu; + return [ id, _extends({ + id: id + }, menu) ]; + })), exports.DURATION = 200, exports.styles = { + light: { + color: "rgba(255, 255, 255, 0.54)" + } + }; }, function(module, exports, __webpack_require__) { function isSymbol(value) { return "symbol" == typeof value || isObjectLike(value) && baseGetTag(value) == symbolTag; } - var baseGetTag = __webpack_require__(42), isObjectLike = __webpack_require__(37), symbolTag = "[object Symbol]"; + var baseGetTag = __webpack_require__(42), isObjectLike = __webpack_require__(36), symbolTag = "[object Symbol]"; module.exports = isSymbol; }, function(module, exports) { function identity(value) { @@ -3719,12 +3792,12 @@ var _bundleJs = []byte((((((((((`!function(modules) { return _inherits(Rectangle, _Component), _createClass(Rectangle, [ { key: "componentDidMount", value: function() { - if (this.node && this.node.getTotalLength) { + if (this.node && this.node.getTotalLength) try { var totalLength = this.node.getTotalLength(); totalLength && this.setState({ totalLength: totalLength }); - } + } catch (err) {} } }, { key: "render", @@ -4167,7 +4240,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }; }; }, function(module, exports, __webpack_require__) { - var $keys = __webpack_require__(210), enumBugKeys = __webpack_require__(140); + var $keys = __webpack_require__(209), enumBugKeys = __webpack_require__(141); module.exports = Object.keys || function(O) { return $keys(O, enumBugKeys); }; @@ -4185,7 +4258,8 @@ var _bundleJs = []byte((((((((((`!function(modules) { return "@media (min-width:" + ("number" == typeof values[key] ? values[key] : key) + unit + ")"; } function down(key) { - return "@media (max-width:" + (("number" == typeof values[key] ? values[key] : key) - step / 100) + unit + ")"; + var endIndex = keys.indexOf(key) + 1, upperbound = values[keys[endIndex]]; + return endIndex === keys.length ? up("xs") : "@media (max-width:" + (("number" == typeof upperbound && endIndex > 0 ? upperbound : key) - step / 100) + unit + ")"; } function between(start, end) { var endIndex = keys.indexOf(end) + 1; @@ -4224,7 +4298,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, function(module, exports, __webpack_require__) { "use strict"; exports.__esModule = !0; - var _getDisplayName = __webpack_require__(227), _getDisplayName2 = function(obj) { + var _getDisplayName = __webpack_require__(226), _getDisplayName2 = function(obj) { return obj && obj.__esModule ? obj : { default: obj }; @@ -4263,7 +4337,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { return protoProps && defineProperties(Constructor.prototype, protoProps), staticProps && defineProperties(Constructor, staticProps), Constructor; }; - }(), _createRule = __webpack_require__(104), _createRule2 = _interopRequireDefault(_createRule), _linkRule = __webpack_require__(232), _linkRule2 = _interopRequireDefault(_linkRule), _StyleRule = __webpack_require__(60), _StyleRule2 = _interopRequireDefault(_StyleRule), _escape = __webpack_require__(429), _escape2 = _interopRequireDefault(_escape), RuleList = function() { + }(), _createRule = __webpack_require__(106), _createRule2 = _interopRequireDefault(_createRule), _linkRule = __webpack_require__(231), _linkRule2 = _interopRequireDefault(_linkRule), _StyleRule = __webpack_require__(60), _StyleRule2 = _interopRequireDefault(_StyleRule), _escape = __webpack_require__(424), _escape2 = _interopRequireDefault(_escape), RuleList = function() { function RuleList(options) { _classCallCheck(this, RuleList), this.map = {}, this.raw = {}, this.index = [], this.options = options, this.classes = options.classes; @@ -4349,6 +4423,9 @@ var _bundleJs = []byte((((((((((`!function(modules) { } ]), RuleList; }(); exports.default = RuleList; +}, function(module, exports, __webpack_require__) { + var root = __webpack_require__(32), Symbol = root.Symbol; + module.exports = Symbol; }, function(module, __webpack_exports__, __webpack_require__) { "use strict"; function _objectWithoutProperties(obj, keys) { @@ -4392,12 +4469,9 @@ var _bundleJs = []byte((((((((((`!function(modules) { children: __WEBPACK_IMPORTED_MODULE_1_prop_types___default.a.oneOfType([ __WEBPACK_IMPORTED_MODULE_1_prop_types___default.a.arrayOf(__WEBPACK_IMPORTED_MODULE_1_prop_types___default.a.node), __WEBPACK_IMPORTED_MODULE_1_prop_types___default.a.node ]) }; Surface.propTypes = propTypes, __webpack_exports__.a = Surface; -}, function(module, exports, __webpack_require__) { - var root = __webpack_require__(31), Symbol = root.Symbol; - module.exports = Symbol; }, function(module, __webpack_exports__, __webpack_require__) { "use strict"; - var __WEBPACK_IMPORTED_MODULE_0__src_path__ = __webpack_require__(573); + var __WEBPACK_IMPORTED_MODULE_0__src_path__ = __webpack_require__(584); __webpack_require__.d(__webpack_exports__, "a", function() { return __WEBPACK_IMPORTED_MODULE_0__src_path__.a; }); @@ -4455,7 +4529,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { function baseIteratee(value) { return "function" == typeof value ? value : null == value ? identity : "object" == typeof value ? isArray(value) ? baseMatchesProperty(value[0], value[1]) : baseMatches(value) : property(value); } - var baseMatches = __webpack_require__(658), baseMatchesProperty = __webpack_require__(661), identity = __webpack_require__(62), isArray = __webpack_require__(12), property = __webpack_require__(665); + var baseMatches = __webpack_require__(669), baseMatchesProperty = __webpack_require__(672), identity = __webpack_require__(63), isArray = __webpack_require__(12), property = __webpack_require__(676); module.exports = baseIteratee; }, function(module, __webpack_exports__, __webpack_require__) { "use strict"; @@ -4506,29 +4580,29 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, linearish(scale); } __webpack_exports__.b = linearish, __webpack_exports__.a = linear; - var __WEBPACK_IMPORTED_MODULE_0_d3_array__ = __webpack_require__(38), __WEBPACK_IMPORTED_MODULE_1_d3_interpolate__ = __webpack_require__(87), __WEBPACK_IMPORTED_MODULE_2__continuous__ = __webpack_require__(125), __WEBPACK_IMPORTED_MODULE_3__tickFormat__ = __webpack_require__(732); + var __WEBPACK_IMPORTED_MODULE_0_d3_array__ = __webpack_require__(37), __WEBPACK_IMPORTED_MODULE_1_d3_interpolate__ = __webpack_require__(88), __WEBPACK_IMPORTED_MODULE_2__continuous__ = __webpack_require__(126), __WEBPACK_IMPORTED_MODULE_3__tickFormat__ = __webpack_require__(742); }, function(module, __webpack_exports__, __webpack_require__) { "use strict"; - var __WEBPACK_IMPORTED_MODULE_0__src_value__ = __webpack_require__(188); + var __WEBPACK_IMPORTED_MODULE_0__src_value__ = __webpack_require__(187); __webpack_require__.d(__webpack_exports__, "a", function() { return __WEBPACK_IMPORTED_MODULE_0__src_value__.a; }); - var __WEBPACK_IMPORTED_MODULE_5__src_number__ = (__webpack_require__(309), __webpack_require__(191), - __webpack_require__(307), __webpack_require__(310), __webpack_require__(124)); + var __WEBPACK_IMPORTED_MODULE_5__src_number__ = (__webpack_require__(305), __webpack_require__(190), + __webpack_require__(303), __webpack_require__(306), __webpack_require__(125)); __webpack_require__.d(__webpack_exports__, "c", function() { return __WEBPACK_IMPORTED_MODULE_5__src_number__.a; }); - var __WEBPACK_IMPORTED_MODULE_7__src_round__ = (__webpack_require__(311), __webpack_require__(722)); + var __WEBPACK_IMPORTED_MODULE_7__src_round__ = (__webpack_require__(307), __webpack_require__(732)); __webpack_require__.d(__webpack_exports__, "d", function() { return __WEBPACK_IMPORTED_MODULE_7__src_round__.a; }); - var __WEBPACK_IMPORTED_MODULE_15__src_cubehelix__ = (__webpack_require__(312), __webpack_require__(723), - __webpack_require__(726), __webpack_require__(306), __webpack_require__(727), __webpack_require__(728), - __webpack_require__(729), __webpack_require__(730)); + var __WEBPACK_IMPORTED_MODULE_15__src_cubehelix__ = (__webpack_require__(308), __webpack_require__(733), + __webpack_require__(736), __webpack_require__(302), __webpack_require__(737), __webpack_require__(738), + __webpack_require__(739), __webpack_require__(740)); __webpack_require__.d(__webpack_exports__, "b", function() { return __WEBPACK_IMPORTED_MODULE_15__src_cubehelix__.a; }); - __webpack_require__(731); + __webpack_require__(741); }, function(module, __webpack_exports__, __webpack_require__) { "use strict"; function linear(a, d) { @@ -4555,7 +4629,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { return d ? linear(a, d) : Object(__WEBPACK_IMPORTED_MODULE_0__constant__.a)(isNaN(a) ? b : a); } __webpack_exports__.c = hue, __webpack_exports__.b = gamma, __webpack_exports__.a = nogamma; - var __WEBPACK_IMPORTED_MODULE_0__constant__ = __webpack_require__(308); + var __WEBPACK_IMPORTED_MODULE_0__constant__ = __webpack_require__(304); }, function(module, __webpack_exports__, __webpack_require__) { "use strict"; __webpack_exports__.a = function(s) { @@ -4750,7 +4824,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, function(module, exports, __webpack_require__) { "use strict"; (function(process) { - var emptyFunction = __webpack_require__(40), warning = emptyFunction; + var emptyFunction = __webpack_require__(39), warning = emptyFunction; if ("production" !== process.env.NODE_ENV) { var printWarning = function(format) { for (var _len = arguments.length, args = Array(_len > 1 ? _len - 1 : 0), _key = 1; _key < _len; _key++) args[_key - 1] = arguments[_key]; @@ -4785,7 +4859,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { } } } - "production" === process.env.NODE_ENV ? (checkDCE(), module.exports = __webpack_require__(339)) : module.exports = __webpack_require__(342); + "production" === process.env.NODE_ENV ? (checkDCE(), module.exports = __webpack_require__(334)) : module.exports = __webpack_require__(337); }).call(exports, __webpack_require__(2)); }, function(module, exports, __webpack_require__) { "use strict"; @@ -4803,7 +4877,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { var hasOwnProperty = Object.prototype.hasOwnProperty; module.exports = shallowEqual; }, function(module, exports, __webpack_require__) { - var toInteger = __webpack_require__(137), min = Math.min; + var toInteger = __webpack_require__(138), min = Math.min; module.exports = function(it) { return it > 0 ? min(toInteger(it), 9007199254740991) : 0; }; @@ -4822,7 +4896,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }; } exports.__esModule = !0; - var _iterator = __webpack_require__(357), _iterator2 = _interopRequireDefault(_iterator), _symbol = __webpack_require__(365), _symbol2 = _interopRequireDefault(_symbol), _typeof = "function" == typeof _symbol2.default && "symbol" == typeof _iterator2.default ? function(obj) { + var _iterator = __webpack_require__(352), _iterator2 = _interopRequireDefault(_iterator), _symbol = __webpack_require__(360), _symbol2 = _interopRequireDefault(_symbol), _typeof = "function" == typeof _symbol2.default && "symbol" == typeof _iterator2.default ? function(obj) { return typeof obj; } : function(obj) { return obj && "function" == typeof _symbol2.default && obj.constructor === _symbol2.default && obj !== _symbol2.default.prototype ? "symbol" : typeof obj; @@ -4833,9 +4907,9 @@ var _bundleJs = []byte((((((((((`!function(modules) { return obj && "function" == typeof _symbol2.default && obj.constructor === _symbol2.default && obj !== _symbol2.default.prototype ? "symbol" : void 0 === obj ? "undefined" : _typeof(obj); }; }, function(module, exports, __webpack_require__) { - var anObject = __webpack_require__(48), dPs = __webpack_require__(361), enumBugKeys = __webpack_require__(140), IE_PROTO = __webpack_require__(138)("IE_PROTO"), Empty = function() {}, createDict = function() { - var iframeDocument, iframe = __webpack_require__(209)("iframe"), i = enumBugKeys.length; - for (iframe.style.display = "none", __webpack_require__(362).appendChild(iframe), + var anObject = __webpack_require__(48), dPs = __webpack_require__(356), enumBugKeys = __webpack_require__(141), IE_PROTO = __webpack_require__(139)("IE_PROTO"), Empty = function() {}, createDict = function() { + var iframeDocument, iframe = __webpack_require__(208)("iframe"), i = enumBugKeys.length; + for (iframe.style.display = "none", __webpack_require__(357).appendChild(iframe), iframe.src = "javascript:", iframeDocument = iframe.contentWindow.document, iframeDocument.open(), iframeDocument.write("