From 6bff0b3f7f55ee2d971eabe65b72ee16eab41dff Mon Sep 17 00:00:00 2001 From: zelig Date: Sat, 4 Jun 2016 15:38:25 +0100 Subject: [PATCH 01/24] swarm/api/config: fix test for new callinterval value --- swarm/api/config_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/api/config_test.go b/swarm/api/config_test.go index 136967153a..c772d6aa88 100644 --- a/swarm/api/config_test.go +++ b/swarm/api/config_test.go @@ -22,7 +22,7 @@ var ( "Hash": "SHA256", "JoinTimeout": 120, "SplitTimeout": 120, - "CallInterval": 10000000000, + "CallInterval": 3000000000, "KadDbPath": "` + filepath.Join("TMPDIR", "0d2f62485607cf38d9d795d93682a517661e513e", "bzz-peers.json") + `", "MaxProx": 8, "ProxBinSize": 4, From 74cc29789a8743a548f11b1e967ca64191cbac3c Mon Sep 17 00:00:00 2001 From: zelig Date: Mon, 6 Jun 2016 23:39:44 +0100 Subject: [PATCH 02/24] swarm/storage: * optimise splitter * simplify reader * benchmarks, tests improved * simplify IO, remove chunkreader.go and io.SectionReader clone * simplify error/timeout handling * port pyramid splitter by karalabe, adapt to Splitter/DPA interface, rework params etc --- swarm/api/api.go | 12 +- swarm/api/config_test.go | 2 - swarm/api/filesystem.go | 5 +- swarm/api/http/server.go | 5 +- swarm/api/manifest.go | 5 +- swarm/cmd/bzzhash/bzzhash.go | 11 +- swarm/cmd/swarm/swarm.sh | 2 + swarm/storage/chunker.go | 311 +++++++++++++++++++--------------- swarm/storage/chunker_test.go | 264 ++++++++++++++++++----------- swarm/storage/chunkreader.go | 194 --------------------- swarm/storage/common_test.go | 64 +++---- swarm/storage/dpa.go | 24 +-- swarm/storage/dpa_test.go | 8 +- swarm/storage/pyramid.go | 165 ++++++++++++++++++ swarm/storage/types.go | 61 ++++++- 15 files changed, 608 insertions(+), 525 deletions(-) delete mode 100644 swarm/storage/chunkreader.go create mode 100644 swarm/storage/pyramid.go diff --git a/swarm/api/api.go b/swarm/api/api.go index 883f226cf2..cc76b36ae5 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -47,8 +47,8 @@ func (self *Api) Retrieve(key storage.Key) storage.SectionReader { return self.dpa.Retrieve(key) } -func (self *Api) Store(data storage.SectionReader, wg *sync.WaitGroup) (key storage.Key, err error) { - return self.dpa.Store(data, wg) +func (self *Api) Store(data io.Reader, size int64, wg *sync.WaitGroup) (key storage.Key, err error) { + return self.dpa.Store(data, size, wg) } type ErrResolve error @@ -105,15 +105,15 @@ func (self *Api) parseAndResolve(uri string, nameresolver bool) (contentHash sto // Put provides singleton manifest creation on top of dpa store func (self *Api) Put(content, contentType string) (string, error) { - sr := io.NewSectionReader(strings.NewReader(content), 0, int64(len(content))) + r := strings.NewReader(content) wg := &sync.WaitGroup{} - key, err := self.dpa.Store(sr, wg) + key, err := self.dpa.Store(r, int64(len(content)), wg) if err != nil { return "", err } manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType) - sr = io.NewSectionReader(strings.NewReader(manifest), 0, int64(len(manifest))) - key, err = self.dpa.Store(sr, wg) + r = strings.NewReader(manifest) + key, err = self.dpa.Store(r, int64(len(manifest)), wg) if err != nil { return "", err } diff --git a/swarm/api/config_test.go b/swarm/api/config_test.go index c772d6aa88..48fbfacb20 100644 --- a/swarm/api/config_test.go +++ b/swarm/api/config_test.go @@ -20,8 +20,6 @@ var ( "Radius": 0, "Branches": 128, "Hash": "SHA256", - "JoinTimeout": 120, - "SplitTimeout": 120, "CallInterval": 3000000000, "KadDbPath": "` + filepath.Join("TMPDIR", "0d2f62485607cf38d9d795d93682a517661e513e", "bzz-peers.json") + `", "MaxProx": 8, diff --git a/swarm/api/filesystem.go b/swarm/api/filesystem.go index 002bb95273..34f2a8f0c8 100644 --- a/swarm/api/filesystem.go +++ b/swarm/api/filesystem.go @@ -96,17 +96,16 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) { f, err := os.Open(entry.Path) if err == nil { stat, _ := f.Stat() - sr := io.NewSectionReader(f, 0, stat.Size()) wg := &sync.WaitGroup{} var hash storage.Key - hash, err = self.api.dpa.Store(sr, wg) + hash, err = self.api.dpa.Store(f, stat.Size(), wg) if hash != nil { list[i].Hash = hash.String() } wg.Wait() if err == nil { first512 := make([]byte, 512) - fread, _ := sr.ReadAt(first512, 0) + fread, _ := f.ReadAt(first512, 0) if fread > 0 { mimeType := http.DetectContentType(first512[:fread]) if filepath.Ext(entry.Path) == ".css" { diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 6ac2b113e6..945fc4f6e0 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -96,10 +96,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { switch { case r.Method == "POST" || r.Method == "PUT": - key, err := a.Store(io.NewSectionReader(&sequentialReader{ - reader: r.Body, - ahead: make(map[int64]chan bool), - }, 0, r.ContentLength), nil) + key, err := a.Store(r.Body, r.ContentLength, nil) if err == nil { glog.V(logger.Debug).Infof("[BZZ] Swarm: Content for %v stored", key.Log()) } else { diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index 0337382ff0..ab74a35bb0 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/json" "fmt" - "io" "sync" "github.com/ethereum/go-ethereum/common" @@ -198,9 +197,9 @@ func (self *manifestTrie) recalcAndStore() error { return err } - sr := io.NewSectionReader(bytes.NewReader(manifest), 0, int64(len(manifest))) + sr := bytes.NewReader(manifest) wg := &sync.WaitGroup{} - key, err2 := self.dpa.Store(sr, wg) + key, err2 := self.dpa.Store(sr, int64(len(manifest)), wg) wg.Wait() self.hash = key return err2 diff --git a/swarm/cmd/bzzhash/bzzhash.go b/swarm/cmd/bzzhash/bzzhash.go index d4df5fb50b..37df224bfd 100644 --- a/swarm/cmd/bzzhash/bzzhash.go +++ b/swarm/cmd/bzzhash/bzzhash.go @@ -3,7 +3,6 @@ package main import ( "fmt" - "io" "os" "runtime" @@ -24,15 +23,11 @@ func main() { } stat, _ := f.Stat() - sr := io.NewSectionReader(f, 0, stat.Size()) chunker := storage.NewTreeChunker(storage.NewChunkerParams()) - hash := make([]byte, chunker.KeySize()) - errC := chunker.Split(hash, sr, nil, nil) - err, ok := <-errC + key, err := chunker.Split(f, stat.Size(), nil, nil, nil) if err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) - } - if !ok { - fmt.Printf("%064x\n", hash) + } else { + fmt.Printf("%v\n", key) } } diff --git a/swarm/cmd/swarm/swarm.sh b/swarm/cmd/swarm/swarm.sh index 8546e1b8d8..8e3e2bb9c1 100644 --- a/swarm/cmd/swarm/swarm.sh +++ b/swarm/cmd/swarm/swarm.sh @@ -336,6 +336,8 @@ case $cmd in up $*;; "down" ) down $*;; + "download" ) + download $*;; "init" ) init $*;; "start" ) diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index c19bb622ec..156b64d7e0 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -2,10 +2,11 @@ package storage import ( "encoding/binary" + "errors" "fmt" + "hash" "io" "sync" - "time" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" @@ -37,11 +38,9 @@ data_{i} := size(subtree_{i}) || key_{j} || key_{j+1} .... || key_{j+n-1} */ const ( - // defaultHash = "SHA3" // http://golang.org/pkg/hash/#Hash + // defaultHash = "SHA3" // http://golang.org/pkg/hash/#Hash defaultHash = "SHA256" // http://golang.org/pkg/hash/#Hash defaultBranches int64 = 128 - joinTimeout = 120 // second - splitTimeout = 120 // second // hashSize int64 = hasherfunc.New().Size() // hasher knows about its own length in bytes // chunksize int64 = branches * hashSize // chunk is defined as this ) @@ -55,130 +54,107 @@ The hashing itself does use extra copies and allocation though, since it does ne */ type ChunkerParams struct { - Branches int64 - Hash string - JoinTimeout time.Duration - SplitTimeout time.Duration + Branches int64 + Hash string } func NewChunkerParams() *ChunkerParams { return &ChunkerParams{ - Branches: defaultBranches, - Hash: defaultHash, - JoinTimeout: joinTimeout, - SplitTimeout: splitTimeout, + Branches: defaultBranches, + Hash: defaultHash, } } type TreeChunker struct { - branches int64 - hashFunc Hasher - joinTimeout time.Duration - splitTimeout time.Duration + branches int64 + hashFunc Hasher // calculated - hashSize int64 // self.hashFunc.New().Size() - chunkSize int64 // hashSize* branches + hashSize int64 // self.hashFunc.New().Size() + chunkSize int64 // hashSize* branches + workerCount int } func NewTreeChunker(params *ChunkerParams) (self *TreeChunker) { self = &TreeChunker{} self.hashFunc = MakeHashFunc(params.Hash) self.branches = params.Branches - self.joinTimeout = params.JoinTimeout * time.Second - self.splitTimeout = params.SplitTimeout * time.Second self.hashSize = int64(self.hashFunc().Size()) self.chunkSize = self.hashSize * self.branches + self.workerCount = 1 return } -func (self *TreeChunker) KeySize() int64 { - return self.hashSize -} +// func (self *TreeChunker) KeySize() int64 { +// return self.hashSize +// } // String() for pretty printing func (self *Chunk) String() string { return fmt.Sprintf("Key: %v TreeSize: %v Chunksize: %v", self.Key.Log(), self.Size, len(self.SData)) } -// The treeChunkers own Hash hashes together -// - the size (of the subtree encoded in the Chunk) -// - the Chunk, ie. the contents read from the input reader -func (self *TreeChunker) Hash(input []byte) []byte { - hasher := self.hashFunc() - hasher.Write(input) - return hasher.Sum(nil) +type hashJob struct { + key Key + chunk []byte + size int64 + parentWg *sync.WaitGroup } -func (self *TreeChunker) Split(key Key, data SectionReader, chunkC chan *Chunk, swg *sync.WaitGroup) (errC chan error) { - - if swg != nil { - swg.Add(1) - defer swg.Done() - } +func (self *TreeChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, swg, wwg *sync.WaitGroup) (Key, error) { if self.chunkSize <= 0 { panic("chunker must be initialised") } - if int64(len(key)) != self.hashSize { - panic(fmt.Sprintf("root key buffer must be allocated byte slice of length %d", self.hashSize)) + jobC := make(chan *hashJob, 2*processors) + wg := &sync.WaitGroup{} + errC := make(chan error) + + // wwg = workers waitgroup keeps track of hashworkers spawned by this split call + if wwg != nil { + wwg.Add(1) + } + go self.hashWorker(jobC, chunkC, errC, swg, wwg) + + depth := 0 + treeSize := self.chunkSize + + // takes lowest depth such that chunksize*HashCount^(depth+1) > size + // power series, will find the order of magnitude of the data size in base hashCount or numbers of levels of branching in the resulting tree. + for ; treeSize < size; treeSize *= self.branches { + depth++ } - wg := &sync.WaitGroup{} - errC = make(chan error) - rerrC := make(chan error) - timeout := time.After(self.splitTimeout) - + key := make([]byte, self.hashFunc().Size()) + glog.V(logger.Detail).Infof("[BZZ] split request received for data (%v bytes, depth: %v)", size, depth) + // this waitgroup member is released after the root hash is calculated wg.Add(1) - go func() { - - depth := 0 - treeSize := self.chunkSize - size := data.Size() - // takes lowest depth such that chunksize*HashCount^(depth+1) > size - // power series, will find the order of magnitude of the data size in base hashCount or numbers of levels of branching in the resulting tree. - - for ; treeSize < size; treeSize *= self.branches { - depth++ - } - - // glog.V(logger.Detail).Infof("[BZZ] split request received for data (%v bytes, depth: %v)", size, depth) - - //launch actual recursive function passing the workgroup - self.split(depth, treeSize/self.branches, key, data, chunkC, rerrC, wg, swg) - }() + //launch actual recursive function passing the workgroup + go self.split(depth, treeSize/self.branches, key, data, size, jobC, chunkC, errC, wg, swg, wwg) // closes internal error channel if all subprocesses in the workgroup finished go func() { + // waiting for all threads to finish wg.Wait() - close(rerrC) - - }() - - // waiting for request to end with wg finishing, error, or timeout - go func() { - select { - case err := <-rerrC: - if err != nil { - errC <- err - } // otherwise splitting is complete - case <-timeout: - errC <- fmt.Errorf("split time out") + // if storage waitgroup is non-nil, we wait for storage to finish too + if swg != nil { + // glog.V(logger.Detail).Infof("Waiting for storage to finish") + swg.Wait() } close(errC) }() - return + select { + case err := <-errC: + if err != nil { + return nil, err + } + // + } + return key, nil } -func (self *TreeChunker) split(depth int, treeSize int64, key Key, data SectionReader, chunkC chan *Chunk, errc chan error, parentWg *sync.WaitGroup, swg *sync.WaitGroup) { - - defer parentWg.Done() - - size := data.Size() - var newChunk *Chunk - var hash Key - // glog.V(logger.Detail).Infof("[BZZ] depth: %v, max subtree size: %v, data size: %v", depth, treeSize, size) +func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reader, size int64, jobC chan *hashJob, chunkC chan *Chunk, errC chan error, parentWg, swg, wwg *sync.WaitGroup) { for depth > 0 && size < treeSize { treeSize /= self.branches @@ -187,71 +163,110 @@ func (self *TreeChunker) split(depth int, treeSize int64, key Key, data SectionR if depth == 0 { // leaf nodes -> content chunks - chunkData := make([]byte, data.Size()+8) + chunkData := make([]byte, size+8) binary.LittleEndian.PutUint64(chunkData[0:8], uint64(size)) - data.ReadAt(chunkData[8:], 0) - hash = self.Hash(chunkData) - // glog.V(logger.Detail).Infof("[BZZ] content chunk: max subtree size: %v, data size: %v", treeSize, size) - newChunk = &Chunk{ - Key: hash, - SData: chunkData, - Size: size, + data.Read(chunkData[8:]) + select { + case jobC <- &hashJob{key, chunkData, size, parentWg}: + case <-errC: } - } else { - // intermediate chunk containing child nodes hashes - branchCnt := int64((size + treeSize - 1) / treeSize) - // glog.V(logger.Detail).Infof("[BZZ] intermediate node: setting branches: %v, depth: %v, max subtree size: %v, data size: %v", branches, depth, treeSize, size) + // glog.V(logger.Detail).Infof("[BZZ] read %v", size) + return + } + // intermediate chunk containing child nodes hashes + branchCnt := int64((size + treeSize - 1) / treeSize) + // glog.V(logger.Detail).Infof("[BZZ] intermediate node: setting branches: %v, depth: %v, max subtree size: %v, data size: %v", branches, depth, treeSize, size) - var chunk []byte = make([]byte, branchCnt*self.hashSize+8) - var pos, i int64 + var chunk []byte = make([]byte, branchCnt*self.hashSize+8) + var pos, i int64 - binary.LittleEndian.PutUint64(chunk[0:8], uint64(size)) + binary.LittleEndian.PutUint64(chunk[0:8], uint64(size)) - childrenWg := &sync.WaitGroup{} - var secSize int64 - for i < branchCnt { - // the last item can have shorter data - if size-pos < treeSize { - secSize = size - pos - } else { - secSize = treeSize + childrenWg := &sync.WaitGroup{} + var secSize int64 + for i < branchCnt { + // the last item can have shorter data + if size-pos < treeSize { + secSize = size - pos + } else { + secSize = treeSize + } + // the hash of that data + subTreeKey := chunk[8+i*self.hashSize : 8+(i+1)*self.hashSize] + + childrenWg.Add(1) + self.split(depth-1, treeSize/self.branches, subTreeKey, data, secSize, jobC, chunkC, errC, childrenWg, swg, wwg) + + i++ + pos += treeSize + } + // wait for all the children to complete calculating their hashes and copying them onto sections of the chunk + // parentWg.Add(1) + // go func() { + childrenWg.Wait() + if len(jobC) > self.workerCount && self.workerCount < processors { + if wwg != nil { + wwg.Add(1) + } + self.workerCount++ + go self.hashWorker(jobC, chunkC, errC, swg, wwg) + } + select { + case jobC <- &hashJob{key, chunk, size, parentWg}: + case <-errC: + } +} + +func (self *TreeChunker) hashWorker(jobC chan *hashJob, chunkC chan *Chunk, errC chan error, swg, wwg *sync.WaitGroup) { + hasher := self.hashFunc() + if wwg != nil { + defer wwg.Done() + } + for { + select { + + case job, ok := <-jobC: + if !ok { + return } - // take the section of the data encoded in the subTree - subTreeData := NewChunkReader(data, pos, secSize) - // the hash of that data - subTreeKey := chunk[8+i*self.hashSize : 8+(i+1)*self.hashSize] - - childrenWg.Add(1) - go self.split(depth-1, treeSize/self.branches, subTreeKey, subTreeData, chunkC, errc, childrenWg, swg) - - i++ - pos += treeSize - } - // wait for all the children to complete calculating their hashes and copying them onto sections of the chunk - childrenWg.Wait() - // now we got the hashes in the chunk, then hash the chunks - hash = self.Hash(chunk) - newChunk = &Chunk{ - Key: hash, - SData: chunk, - Size: size, - wg: swg, + // now we got the hashes in the chunk, then hash the chunks + hasher.Reset() + self.hashChunk(hasher, job, chunkC, swg) + // glog.V(logger.Detail).Infof("[BZZ] hash chunk (%v)", job.size) + case <-errC: + return } + } +} +// The treeChunkers own Hash hashes together +// - the size (of the subtree encoded in the Chunk) +// - the Chunk, ie. the contents read from the input reader +func (self *TreeChunker) hashChunk(hasher hash.Hash, job *hashJob, chunkC chan *Chunk, swg *sync.WaitGroup) { + hasher.Write(job.chunk) + h := hasher.Sum(nil) + newChunk := &Chunk{ + Key: h, + SData: job.chunk, + Size: job.size, + wg: swg, + } + + // report hash of this chunk one level up (keys corresponds to the proper subslice of the parent chunk) + copy(job.key, h) + // send off new chunk to storage + if chunkC != nil { if swg != nil { swg.Add(1) } } - - // send off new chunk to storage + job.parentWg.Done() if chunkC != nil { chunkC <- newChunk } - // report hash of this chunk one level up (keys corresponds to the proper subslice of the parent chunk)x - copy(key, hash) - } +// implements the Joiner interface func (self *TreeChunker) Join(key Key, chunkC chan *Chunk) SectionReader { return &LazyChunkReader{ @@ -263,7 +278,7 @@ func (self *TreeChunker) Join(key Key, chunkC chan *Chunk) SectionReader { } } -// LazyChunkReader implements LazySectionReader +// LazyChunkReader implements Lazy.SectionReader type LazyChunkReader struct { key Key // root key chunkC chan *Chunk // chunk channel to send retrieve requests on @@ -282,7 +297,7 @@ func (self *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) { C: make(chan bool), // close channel to signal data delivery } self.chunkC <- chunk // submit retrieval request, someone should be listening on the other side (or we will time out globally) - glog.V(logger.Detail).Infof("[BZZ] readAt: reading %v into %d bytes at offset %d.", chunk.Key.Log(), len(b), off) + // glog.V(logger.Detail).Infof("[BZZ] readAt: reading %v into %d bytes at offset %d.", chunk.Key.Log(), len(b), off) // waiting for the chunk retrieval select { @@ -409,3 +424,35 @@ func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, tr } //for wg.Wait() } + +func (self *LazyChunkReader) Size() (n int64) { + self.ReadAt(nil, 0) + return self.size +} + +func (self *LazyChunkReader) Read(b []byte) (read int, err error) { + read, err = self.ReadAt(b, self.off) + self.off += int64(read) + return +} + +var errWhence = errors.New("Seek: invalid whence") +var errOffset = errors.New("Seek: invalid offset") + +func (s *LazyChunkReader) Seek(offset int64, whence int) (int64, error) { + switch whence { + default: + return 0, errWhence + case 0: + offset += 0 + case 1: + offset += s.off + case 2: + offset += s.size + } + if offset < 0 { + return 0, errOffset + } + s.off = offset + return offset, nil +} diff --git a/swarm/storage/chunker_test.go b/swarm/storage/chunker_test.go index 7dd301fff5..fbcf210588 100644 --- a/swarm/storage/chunker_test.go +++ b/swarm/storage/chunker_test.go @@ -2,20 +2,33 @@ package storage import ( "bytes" - // "fmt" + "fmt" "io" + "runtime" + "sync" "testing" "time" + + "github.com/ethereum/go-ethereum/logger" + "github.com/ethereum/go-ethereum/logger/glog" ) +func init() { + glog.SetV(logger.Info) + glog.SetToStderr(true) +} + /* Tests TreeChunker by splitting and joining a random byte slice */ +type test interface { + Fatalf(string, ...interface{}) +} + type chunkerTester struct { - errors []error - chunks []*Chunk - timeout bool + chunks []*Chunk + t test } func (self *chunkerTester) checkChunks(t *testing.T, want int) { @@ -25,77 +38,70 @@ func (self *chunkerTester) checkChunks(t *testing.T, want int) { } } -func (self *chunkerTester) Split(chunker *TreeChunker, l int) (key Key, input []byte) { +func (self *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, chunkC chan *Chunk, swg *sync.WaitGroup) (key Key) { // reset - self.errors = nil self.chunks = nil - self.timeout = false - - data, slice := testDataReader(l) - input = slice - key = make([]byte, 32) - chunkC := make(chan *Chunk, 1000) - errC := chunker.Split(key, data, chunkC, nil) quitC := make(chan bool) timeout := time.After(600 * time.Second) + if chunkC != nil { + go func() { + for { + select { + case <-timeout: + self.t.Fatalf("Join timeout error") - go func() { - LOOP: - for { - select { - case <-timeout: - self.timeout = true - break LOOP - - case chunk := <-chunkC: - if chunk != nil { + case chunk, ok := <-chunkC: + if !ok { + // glog.V(logger.Info).Infof("chunkC closed quitting") + close(quitC) + return + } + // glog.V(logger.Info).Infof("chunk %v received", len(self.chunks)) self.chunks = append(self.chunks, chunk) - } else { - break LOOP - } - - case err, ok := <-errC: - if err != nil { - self.errors = append(self.errors, err) - } - // fmt.Printf("err %v", err) - if !ok { - close(chunkC) - errC = nil + if chunk.wg != nil { + chunk.wg.Done() + } } } + }() + } + key, err := chunker.Split(data, size, chunkC, swg, nil) + if err != nil { + self.t.Fatalf("Split error: %v", err) + } + if chunkC != nil { + if swg != nil { + // glog.V(logger.Info).Infof("Waiting for storage to finish") + swg.Wait() + // glog.V(logger.Info).Infof("St orage finished") } - close(quitC) - }() - <-quitC // waiting for it to finish + close(chunkC) + } + if chunkC != nil { + <-quitC + } return } -func (self *chunkerTester) Join(chunker *TreeChunker, key Key, c int) SectionReader { +func (self *chunkerTester) Join(chunker *TreeChunker, key Key, c int, chunkC chan *Chunk, quitC chan bool) SectionReader { // reset but not the chunks - self.errors = nil - self.timeout = false - chunkC := make(chan *Chunk, 1000) reader := chunker.Join(key, chunkC) - quitC := make(chan bool) timeout := time.After(600 * time.Second) i := 0 go func() { - LOOP: for { select { - case <-quitC: - break LOOP - case <-timeout: - self.timeout = true - break LOOP + self.t.Fatalf("Join timeout error") - case chunk := <-chunkC: + case chunk, ok := <-chunkC: + if !ok { + close(quitC) + return + } i++ - // dpaLogger.DebugDetailf("TESTER: chunk request %x", chunk.Key[:4]) // this just mocks the behaviour of a chunk store retrieval var found bool for _, ch := range self.chunks { @@ -106,53 +112,55 @@ func (self *chunkerTester) Join(chunker *TreeChunker, key Key, c int) SectionRea } } if !found { - // fmt.Printf("TESTER: chunk unknown for %x", chunk.Key[:4]) + self.t.Fatalf("not found ") } close(chunk.C) - // dpaLogger.DebugDetailf("TESTER: chunk request served %x", chunk.Key[:4]) } } }() return reader } -func testRandomData(chunker *TreeChunker, tester *chunkerTester, n int, chunks int, t *testing.T) { - key, input := tester.Split(chunker, n) +func testRandomData(n int, chunks int, t *testing.T) { + chunker := NewTreeChunker(&ChunkerParams{ + Branches: 128, + Hash: "SHA3", + }) + tester := &chunkerTester{t: t} + data, input := testDataReaderAndSlice(n) - t.Logf(" Key = %x\n", key) + chunkC := make(chan *Chunk, 1000) + swg := &sync.WaitGroup{} - tester.checkChunks(t, chunks) - time.Sleep(100 * time.Millisecond) + splitter := chunker + key := tester.Split(splitter, data, int64(n), chunkC, swg) - reader := tester.Join(chunker, key, 0) + // t.Logf(" Key = %v\n", key) + + // tester.checkChunks(t, chunks) + chunkC = make(chan *Chunk, 1000) + quitC := make(chan bool) + + reader := tester.Join(chunker, key, 0, chunkC, quitC) output := make([]byte, n) r, err := reader.Read(output) if r != n || err != io.EOF { - t.Errorf("read error read: %v n = %v err = %v\n", r, n, err) + t.Fatalf("read error read: %v n = %v err = %v\n", r, n, err) } - // t.Logf(" IN: %x\nOUT: %x\n", input, output) - if !bytes.Equal(output, input) { - t.Errorf("input and output mismatch\n IN: %x\nOUT: %x\n", input, output) + if input != nil { + if !bytes.Equal(output, input) { + t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", input, output) + } } + close(chunkC) + <-quitC } func TestRandomData(t *testing.T) { - chunker, tester := chunkerAndTester() - testRandomData(chunker, tester, 60, 1, t) - testRandomData(chunker, tester, 179, 5, t) - testRandomData(chunker, tester, 253, 7, t) - // t.Logf("chunks %v", tester.chunks) -} - -func chunkerAndTester() (chunker *TreeChunker, tester *chunkerTester) { - chunker = NewTreeChunker(&ChunkerParams{ - Branches: 2, - Hash: "SHA256", - SplitTimeout: 10, - JoinTimeout: 10, - }) - tester = &chunkerTester{} - return + testRandomData(60, 1, t) + testRandomData(83, 3, t) + testRandomData(179, 5, t) + testRandomData(253, 7, t) } func readAll(reader SectionReader, result []byte) { @@ -177,38 +185,90 @@ func benchReadAll(reader SectionReader) { } } -func benchmarkJoinRandomData(n int, chunks int, t *testing.B) { - t.StopTimer() +func benchmarkJoin(n int, t *testing.B) { for i := 0; i < t.N; i++ { - // fmt.Printf("round %v\n", i) - chunker, tester := chunkerAndTester() - key, _ := tester.Split(chunker, n) - // fmt.Printf("split done %v, joining...\n", i) + chunker := NewTreeChunker(&ChunkerParams{ + Branches: 128, + Hash: "SHA3", + }) + tester := &chunkerTester{t: t} + data := testDataReader(n) + + chunkC := make(chan *Chunk, 1000) + swg := &sync.WaitGroup{} + + key := tester.Split(chunker, data, int64(n), chunkC, swg) t.StartTimer() - reader := tester.Join(chunker, key, i) - // fmt.Printf("join done %v, reading...\n", i) + chunkC = make(chan *Chunk, 1000) + quitC := make(chan bool) + reader := tester.Join(chunker, key, i, chunkC, quitC) + t.StopTimer() benchReadAll(reader) + close(chunkC) + <-quitC } } -func benchmarkSplitRandomData(n int, chunks int, t *testing.B) { +func benchmarkSplitTree(n int, t *testing.B) { + t.ReportAllocs() for i := 0; i < t.N; i++ { - chunker, tester := chunkerAndTester() - tester.Split(chunker, n) + chunker := NewTreeChunker(&ChunkerParams{ + Branches: 128, + Hash: "SHA3", + }) + tester := &chunkerTester{t: t} + data := testDataReader(n) + // glog.V(logger.Info).Infof("splitting data of length %v", n) + tester.Split(chunker, data, int64(n), nil, nil) } + stats := new(runtime.MemStats) + runtime.ReadMemStats(stats) + fmt.Println(stats.Sys) } -func BenchmarkJoinRandomData_100_2(t *testing.B) { benchmarkJoinRandomData(100, 3, t) } -func BenchmarkJoinRandomData_1000_2(t *testing.B) { benchmarkJoinRandomData(1000, 3, t) } -func BenchmarkJoinRandomData_10000_2(t *testing.B) { benchmarkJoinRandomData(10000, 3, t) } -func BenchmarkJoinRandomData_100000_2(t *testing.B) { benchmarkJoinRandomData(100000, 3, t) } -func BenchmarkJoinRandomData_1000000_2(t *testing.B) { benchmarkJoinRandomData(1000000, 3, t) } +func benchmarkSplitPyramid(n int, t *testing.B) { + t.ReportAllocs() + for i := 0; i < t.N; i++ { + splitter := NewPyramidChunker(&ChunkerParams{ + Branches: 128, + Hash: "SHA3", + }) + tester := &chunkerTester{t: t} + data := testDataReader(n) + // glog.V(logger.Info).Infof("splitting data of length %v", n) + tester.Split(splitter, data, int64(n), nil, nil) + } + stats := new(runtime.MemStats) + runtime.ReadMemStats(stats) + fmt.Println(stats.Sys) +} -func BenchmarkSplitRandomData_100_2(t *testing.B) { benchmarkSplitRandomData(100, 3, t) } -func BenchmarkSplitRandomData_1000_2(t *testing.B) { benchmarkSplitRandomData(1000, 3, t) } -func BenchmarkSplitRandomData_10000_2(t *testing.B) { benchmarkSplitRandomData(10000, 3, t) } -func BenchmarkSplitRandomData_100000_2(t *testing.B) { benchmarkSplitRandomData(100000, 3, t) } -func BenchmarkSplitRandomData_1000000_2(t *testing.B) { benchmarkSplitRandomData(1000000, 3, t) } -func BenchmarkSplitRandomData_10000000_2(t *testing.B) { benchmarkSplitRandomData(10000000, 3, t) } +func BenchmarkJoin_100_2(t *testing.B) { benchmarkJoin(100, t) } +func BenchmarkJoin_1000_2(t *testing.B) { benchmarkJoin(1000, t) } +func BenchmarkJoin_10000_2(t *testing.B) { benchmarkJoin(10000, t) } +func BenchmarkJoin_100000_2(t *testing.B) { benchmarkJoin(100000, t) } +func BenchmarkJoin_1000000_2(t *testing.B) { benchmarkJoin(1000000, t) } -// go test -bench ./bzz -cpuprofile cpu.out -memprofile mem.out +func BenchmarkSplitTree_2(t *testing.B) { benchmarkSplitTree(100, t) } +func BenchmarkSplitTree_2h(t *testing.B) { benchmarkSplitTree(500, t) } +func BenchmarkSplitTree_3(t *testing.B) { benchmarkSplitTree(1000, t) } +func BenchmarkSplitTree_3h(t *testing.B) { benchmarkSplitTree(5000, t) } +func BenchmarkSplitTree_4(t *testing.B) { benchmarkSplitTree(10000, t) } +func BenchmarkSplitTree_4h(t *testing.B) { benchmarkSplitTree(50000, t) } +func BenchmarkSplitTree_5(t *testing.B) { benchmarkSplitTree(100000, t) } +func BenchmarkSplitTree_6(t *testing.B) { benchmarkSplitTree(1000000, t) } +func BenchmarkSplitTree_7(t *testing.B) { benchmarkSplitTree(10000000, t) } +func BenchmarkSplitTree_8(t *testing.B) { benchmarkSplitTree(100000000, t) } + +func BenchmarkSplitPyramid_2(t *testing.B) { benchmarkSplitPyramid(100, t) } +func BenchmarkSplitPyramid_2h(t *testing.B) { benchmarkSplitPyramid(500, t) } +func BenchmarkSplitPyramid_3(t *testing.B) { benchmarkSplitPyramid(1000, t) } +func BenchmarkSplitPyramid_3h(t *testing.B) { benchmarkSplitPyramid(5000, t) } +func BenchmarkSplitPyramid_4(t *testing.B) { benchmarkSplitPyramid(10000, t) } +func BenchmarkSplitPyramid_4h(t *testing.B) { benchmarkSplitPyramid(50000, t) } +func BenchmarkSplitPyramid_5(t *testing.B) { benchmarkSplitPyramid(100000, t) } +func BenchmarkSplitPyramid_6(t *testing.B) { benchmarkSplitPyramid(1000000, t) } +func BenchmarkSplitPyramid_7(t *testing.B) { benchmarkSplitPyramid(10000000, t) } +func BenchmarkSplitPyramid_8(t *testing.B) { benchmarkSplitPyramid(100000000, t) } + +// godep go test -bench ./swarm/storage -cpuprofile cpu.out -memprofile mem.out diff --git a/swarm/storage/chunkreader.go b/swarm/storage/chunkreader.go deleted file mode 100644 index b147c85bf8..0000000000 --- a/swarm/storage/chunkreader.go +++ /dev/null @@ -1,194 +0,0 @@ -package storage - -import ( - "bytes" - "errors" - "io" -) - -type Bounded interface { - Size() int64 -} - -type Sliced interface { - Slice(int64, int64) (b []byte, err error) -} - -// Size, Seek, Read, ReadAt -type SectionReader interface { - Bounded - io.Seeker - io.Reader - io.ReaderAt -} - -// ChunkReader implements SectionReader on a section -// of an underlying ReaderAt. -type ChunkReader struct { - r io.ReaderAt - base int64 - off int64 - limit int64 -} - -// NewChunkReader returns a ChunkReader that reads from r -// starting at offset off and stops with EOF after n bytes. -func NewChunkReader(r io.ReaderAt, off int64, n int64) *ChunkReader { - return &ChunkReader{r: r, base: off, off: off, limit: off + n} -} - -// ByteSliceReader just extends byte.Reader to make base slice accessible -type ByteSliceReader struct { - *bytes.Reader - base []byte -} - -func NewByteSliceReader(b []byte) *ByteSliceReader { - return &ByteSliceReader{ - base: b, - Reader: bytes.NewReader(b), - } -} - -// ByteSliceReader implements the Sliced interface -func (self *ByteSliceReader) Slice(from, to int64) (b []byte, err error) { - if from < 0 || to >= int64(self.Len()) { - err = io.EOF - } else { - b = self.base[from:to] - } - return -} - -// NewChunkReaderFromBytes is a convenience shortcut to get a SectionReader over a byte slice -func NewChunkReaderFromBytes(b []byte) *ChunkReader { - return NewChunkReader(NewByteSliceReader(b), 0, int64(len(b))) -} - -/* -The following is adapted from io.SectionReader -*/ - -func (s *ChunkReader) Size() int64 { - return s.limit - s.base -} - -var errWhence = errors.New("Seek: invalid whence") -var errOffset = errors.New("Seek: invalid offset") - -func (s *ChunkReader) Seek(offset int64, whence int) (int64, error) { - switch whence { - default: - return 0, errWhence - case 0: - offset += s.base - case 1: - offset += s.off - case 2: - offset += s.limit - } - if offset < s.base { - return 0, errOffset - } - s.off = offset - return offset - s.base, nil -} - -func (s *ChunkReader) Read(p []byte) (n int, err error) { - if s.off >= s.limit { - return 0, io.EOF - } - if max := s.limit - s.off; int64(len(p)) > max { - p = p[0:max] - } - n, err = s.r.ReadAt(p, s.off) - s.off += int64(n) - return -} - -func (s *ChunkReader) ReadAt(p []byte, off int64) (n int, err error) { - if off < 0 || off >= s.limit-s.base { - return 0, io.EOF - } - off += s.base - if max := s.limit - off; int64(len(p)) > max { - p = p[0:max] - n, err = s.r.ReadAt(p, off) - if err == nil { - err = io.EOF - } - return n, err - } - n, err = s.r.ReadAt(p, off) - return -} - -// added methods to that ChunkReader implements the Sliced interface -func (s *ChunkReader) Slice(from, to int64) (b []byte, err error) { - if from < 0 || to >= s.Size() { - err = io.EOF - } else { - if sl, ok := s.r.(Sliced); ok { - b, err = sl.Slice(s.base+from, s.base+to) - } else { - err = errors.New("not sliceable base") - } - } - return -} - -// added method so that ChunkReader implements the io.WriterTo interface -// WriteTo method is used by io.Copy -// This is so that we avoid one extra step of allocation (if the underlying initial Reader implements Sliced -func (r *ChunkReader) WriteTo(w io.Writer) (n int64, err error) { - var b []byte - var m int - // if b, _ := r.Slice(r.off-r.base, r.limit-r.base); b == nil { - // if slices not available we do it with extra allocation - b = make([]byte, r.limit-r.off) - m, err = r.Read(b) - if err != nil { - return - } - // } - m, err = w.Write(b) - if m > len(b) { - panic("bytes.Reader.WriteTo: invalid Write count") - } - r.off = r.base + int64(m) - n = int64(m) - if m != len(b) && err == nil { - err = io.ErrShortWrite - } - // w - return -} - -func (self *LazyChunkReader) Size() (n int64) { - self.ReadAt(nil, 0) - return self.size -} - -func (self *LazyChunkReader) Read(b []byte) (read int, err error) { - read, err = self.ReadAt(b, self.off) - self.off += int64(read) - return -} - -func (s *LazyChunkReader) Seek(offset int64, whence int) (int64, error) { - switch whence { - default: - return 0, errWhence - case 0: - offset += 0 - case 1: - offset += s.off - case 2: - offset += s.size - } - if offset < 0 { - return 0, errOffset - } - s.off = offset - return offset, nil -} diff --git a/swarm/storage/common_test.go b/swarm/storage/common_test.go index 40dc35fc69..b3fdc027b7 100644 --- a/swarm/storage/common_test.go +++ b/swarm/storage/common_test.go @@ -1,6 +1,7 @@ package storage import ( + "bytes" "crypto/rand" "io" "sync" @@ -10,61 +11,40 @@ import ( "github.com/ethereum/go-ethereum/logger/glog" ) -func testDataReader(l int) (r *ChunkReader, slice []byte) { +func testDataReader(l int) (r io.Reader) { + return io.LimitReader(rand.Reader, int64(l)) +} + +func testDataReaderAndSlice(l int) (r io.Reader, slice []byte) { slice = make([]byte, l) if _, err := rand.Read(slice); err != nil { panic("rand error") } - r = NewChunkReaderFromBytes(slice) - return -} - -func randomChunks(l int64, branches int64, chunkC chan *Chunk) (key Key, errC chan error) { - chunker := NewTreeChunker(&ChunkerParams{ - Branches: branches, - Hash: defaultHash, - SplitTimeout: splitTimeout, - }) - key = make([]byte, 32) - b := make([]byte, l) - _, err := rand.Read(b) - if err != nil { - panic("no rand") - } - wg := &sync.WaitGroup{} - errC = chunker.Split(key, NewChunkReaderFromBytes(b), chunkC, wg) - wg.Wait() + r = bytes.NewReader(slice) return } func testStore(m ChunkStore, l int64, branches int64, t *testing.T) { chunkC := make(chan *Chunk) - key, errC := randomChunks(l, branches, chunkC) - -SPLIT: - for { - select { - case chunk := <-chunkC: + go func() { + for chunk := range chunkC { m.Put(chunk) - case err, ok := <-errC: - if err != nil { - t.Errorf("Chunker error: %v", err) - return - } - if !ok { - break SPLIT + if chunk.wg != nil { + chunk.wg.Done() } } - } + }() chunker := NewTreeChunker(&ChunkerParams{ - Branches: branches, - Hash: defaultHash, - SplitTimeout: splitTimeout, + Branches: branches, + Hash: defaultHash, }) + swg := &sync.WaitGroup{} + key, err := chunker.Split(rand.Reader, l, chunkC, swg, nil) + swg.Wait() + close(chunkC) chunkC = make(chan *Chunk) - var r SectionReader - r = chunker.Join(key, chunkC) + r := chunker.Join(key, chunkC) quit := make(chan bool) @@ -83,12 +63,14 @@ SPLIT: close(chunk.C) }(ch) } + close(quit) }() b := make([]byte, l) n, err := r.ReadAt(b, 0) if err != io.EOF { - t.Errorf("read error (%v/%v) %v", n, l, err) - close(quit) + t.Fatalf("read error (%v/%v) %v", n, l, err) } + close(chunkC) + <-quit } diff --git a/swarm/storage/dpa.go b/swarm/storage/dpa.go index 922ff3bec2..ba8e0e2c56 100644 --- a/swarm/storage/dpa.go +++ b/swarm/storage/dpa.go @@ -2,6 +2,7 @@ package storage import ( "errors" + "io" "sync" "time" @@ -78,27 +79,8 @@ func (self *DPA) Retrieve(key Key) SectionReader { // Public API. Main entry point for document storage directly. Used by the // FS-aware API and httpaccess -func (self *DPA) Store(data SectionReader, wg *sync.WaitGroup) (key Key, err error) { - key = make([]byte, self.Chunker.KeySize()) - errC := self.Chunker.Split(key, data, self.storeC, wg) - -SPLIT: - for { - select { - case err, ok := <-errC: - if err != nil { - glog.V(logger.Error).Infof("[BZZ] chunker split error: %v", err) - } - if !ok { - break SPLIT - } - - case <-self.quitC: - break SPLIT - } - } - return - +func (self *DPA) Store(data io.Reader, size int64, wg *sync.WaitGroup) (key Key, err error) { + return self.Chunker.Split(data, size, self.storeC, nil, wg) } func (self *DPA) Start() { diff --git a/swarm/storage/dpa_test.go b/swarm/storage/dpa_test.go index a4400783fb..4c50a7214f 100644 --- a/swarm/storage/dpa_test.go +++ b/swarm/storage/dpa_test.go @@ -29,9 +29,9 @@ func TestDPArandom(t *testing.T) { ChunkStore: localStore, } dpa.Start() - reader, slice := testDataReader(testDataSize) + reader, slice := testDataReaderAndSlice(testDataSize) wg := &sync.WaitGroup{} - key, err := dpa.Store(reader, wg) + key, err := dpa.Store(reader, testDataSize, wg) if err != nil { t.Errorf("Store error: %v", err) } @@ -85,9 +85,9 @@ func TestDPA_capacity(t *testing.T) { ChunkStore: localStore, } dpa.Start() - reader, slice := testDataReader(testDataSize) + reader, slice := testDataReaderAndSlice(testDataSize) wg := &sync.WaitGroup{} - key, err := dpa.Store(reader, wg) + key, err := dpa.Store(reader, testDataSize, wg) if err != nil { t.Errorf("Store error: %v", err) } diff --git a/swarm/storage/pyramid.go b/swarm/storage/pyramid.go new file mode 100644 index 0000000000..507dc7b768 --- /dev/null +++ b/swarm/storage/pyramid.go @@ -0,0 +1,165 @@ +package storage + +import ( + "io" + "math" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/logger" + "github.com/ethereum/go-ethereum/logger/glog" +) + +const ( + processors = 8 +) + +type Tree struct { + Chunks int64 + Levels []map[int64]*Node + Lock sync.RWMutex +} + +type Node struct { + Pending int64 + Children []common.Hash + Last bool +} + +type Task struct { + Index int64 // Index of the chunk being processed + Data []byte // Binary blob of the chunk + Last bool +} + +type PyramidChunker struct { + hashFunc Hasher + chunkSize int64 + hashSize int64 + branches int64 + workerCount int +} + +func NewPyramidChunker(params *ChunkerParams) (self *PyramidChunker) { + self = &PyramidChunker{} + self.hashFunc = MakeHashFunc(params.Hash) + self.branches = params.Branches + self.hashSize = int64(self.hashFunc().Size()) + self.chunkSize = self.hashSize * self.branches + self.workerCount = 1 + return +} + +func (self *PyramidChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, swg, wwg *sync.WaitGroup) (Key, error) { + + chunks := (size + self.chunkSize - 1) / self.chunkSize + depth := int(math.Ceil(math.Log(float64(chunks))/math.Log(float64(self.branches)))) + 1 + glog.V(logger.Detail).Infof("chunks: %v, depth: %v", chunks, depth) + + results := Tree{ + Chunks: chunks, + Levels: make([]map[int64]*Node, depth), + } + for i := 0; i < depth; i++ { + results.Levels[i] = make(map[int64]*Node) + } + // Create a pool of workers to crunch through the file + tasks := make(chan *Task, 2*processors) + pend := new(sync.WaitGroup) + abortC := make(chan bool) + for i := 0; i < processors; i++ { + pend.Add(1) + go self.processor(pend, tasks, &results) + } + // Feed the chunks into the task pool + for index := 0; ; index++ { + buffer := make([]byte, self.chunkSize+8) + n, err := io.ReadFull(data, buffer) + last := err == io.ErrUnexpectedEOF + if err != nil && !last { + glog.V(logger.Info).Infof("error: %v", err) + + close(abortC) + } + pend.Add(1) + // glog.V(logger.Info).Infof("-> task %v (%v)", index, n) + select { + case tasks <- &Task{Index: int64(index), Data: buffer[:n+8], Last: last}: + case <-abortC: + return nil, err + } + if last { + // glog.V(logger.Info).Infof("last task %v (%v)", index, n) + break + } + } + // Wait for the workers and return + close(tasks) + pend.Wait() + + // glog.V(logger.Info).Infof("len: %v", results.Levels[0][0]) + key := results.Levels[0][0].Children[0][:] + return key, nil +} + +func (self *PyramidChunker) processor(pend *sync.WaitGroup, tasks chan *Task, results *Tree) { + defer pend.Done() + + // glog.V(logger.Info).Infof("processor started") + // Start processing leaf chunks ad infinitum + hasher := self.hashFunc() + for task := range tasks { + depth, pow := len(results.Levels)-1, self.branches + // glog.V(logger.Info).Infof("task: %v, last: %v", task.Index, task.Last) + + var node *Node + for depth >= 0 { + // New chunk received, reset the hasher and start processing + hasher.Reset() + + if node == nil { // Leaf node, hash the data chunk + hasher.Write(task.Data) + } else { // Internal node, hash the children + for _, hash := range node.Children { + hasher.Write(hash[:]) + } + } + hash := hasher.Sum(nil) + last := task.Last || (node != nil) && node.Last + // Insert the subresult into the memoization tree + results.Lock.Lock() + if node = results.Levels[depth][task.Index/pow]; node == nil { + // Figure out the pending tasks + pending := self.branches + if task.Index/pow == results.Chunks/pow { + pending = (results.Chunks + pow/self.branches - 1) / (pow / self.branches) % self.branches + } + node = &Node{pending, make([]common.Hash, pending), last} + results.Levels[depth][task.Index/pow] = node + } + node.Pending-- + i := task.Index / (pow / self.branches) % self.branches + if last { + node.Pending -= self.branches - i + node.Children = node.Children[:i+1] + node.Last = true + } + copy(node.Children[i][:], hash) + left := node.Pending + + if depth+1 < len(results.Levels) { + delete(results.Levels[depth+1], task.Index/(pow/self.branches)) + } + results.Lock.Unlock() + // If there's more work to be done, leave for others + // glog.V(logger.Info).Infof("left %v", left) + if left > 0 { + break + } + // We're the last ones in this batch, merge the children together + depth-- + pow *= self.branches + } + pend.Done() + } +} diff --git a/swarm/storage/types.go b/swarm/storage/types.go index 124a56a085..0a40fb4614 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -5,6 +5,7 @@ import ( "crypto" "fmt" "hash" + "io" "sync" "github.com/ethereum/go-ethereum/common" @@ -17,6 +18,41 @@ type Peer interface{} type Key []byte +func (x Key) Size() uint { + return uint(len(x)) +} + +func (x Key) isEqual(y Key) bool { + return bytes.Compare(x, y) == 0 +} + +func (h Key) bits(i, j uint) uint { + ii := i >> 3 + jj := i & 7 + if ii >= h.Size() { + return 0 + } + + if jj+j <= 8 { + return uint((h[ii] >> jj) & ((1 << j) - 1)) + } + + res := uint(h[ii] >> jj) + jj = 8 - jj + j -= jj + for j != 0 { + ii++ + if j < 8 { + res += uint(h[ii]&((1< Date: Sat, 11 Jun 2016 17:10:08 +0100 Subject: [PATCH 03/24] swarm/network/kademlia: fix index out of range when deleting last idle peer from kaddb --- swarm/network/kademlia/kaddb.go | 8 ++++---- swarm/storage/memstore.go | 36 --------------------------------- 2 files changed, 4 insertions(+), 40 deletions(-) diff --git a/swarm/network/kademlia/kaddb.go b/swarm/network/kademlia/kaddb.go index 22adb2cbb8..37900a141a 100644 --- a/swarm/network/kademlia/kaddb.go +++ b/swarm/network/kademlia/kaddb.go @@ -247,14 +247,14 @@ func (self *KadDb) delete(row int, indexes ...int) { dbrow := self.Nodes[row] for _, next := range indexes { // need to adjust dbcursor + if next <= self.cursors[row] { + self.cursors[row]-- + } if next > 0 { - if next <= self.cursors[row] { - self.cursors[row]-- - } nodes = append(nodes, dbrow[prev:next]...) } - prev = next + 1 delete(self.index, dbrow[next].Addr) + prev = next + 1 } self.Nodes[row] = append(nodes, dbrow[prev:]...) } diff --git a/swarm/storage/memstore.go b/swarm/storage/memstore.go index f415bdfa59..c5e0f6227f 100644 --- a/swarm/storage/memstore.go +++ b/swarm/storage/memstore.go @@ -3,7 +3,6 @@ package storage import ( - "bytes" "sync" ) @@ -44,41 +43,6 @@ func NewMemStore(d *DbStore, capacity uint) (m *MemStore) { return } -func (x Key) Size() uint { - return uint(len(x)) -} - -func (x Key) isEqual(y Key) bool { - return bytes.Compare(x, y) == 0 -} - -func (h Key) bits(i, j uint) uint { - ii := i >> 3 - jj := i & 7 - if ii >= h.Size() { - return 0 - } - - if jj+j <= 8 { - return uint((h[ii] >> jj) & ((1 << j) - 1)) - } - - res := uint(h[ii] >> jj) - jj = 8 - jj - j -= jj - for j != 0 { - ii++ - if j < 8 { - res += uint(h[ii]&((1< Date: Tue, 14 Jun 2016 11:45:57 +0100 Subject: [PATCH 04/24] swarm/network: log node address consistently in syncdb --- swarm/network/syncdb.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/swarm/network/syncdb.go b/swarm/network/syncdb.go index f91ed9a50d..aabc2d9b03 100644 --- a/swarm/network/syncdb.go +++ b/swarm/network/syncdb.go @@ -126,7 +126,7 @@ LOOP: // if syncdb is stopped. In this case we need to save the item to the db more = deliver(req, self.quit) if !more { - glog.V(logger.Debug).Infof("[BZZ] syncDb[%v] quit: switching to db. session tally (db/total): %v/%v", self.priority, self.dbTotal, self.total) + glog.V(logger.Debug).Infof("[BZZ] syncDb[%v/%v] quit: switching to db. session tally (db/total): %v/%v", self.key.Log(), self.priority, self.dbTotal, self.total) // received quit signal, save request currently waiting delivery // by switching to db mode and closing the buffer buffer = nil @@ -136,12 +136,12 @@ LOOP: break // break from select, this item will be written to the db } self.total++ - glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] deliver (db/total): %v/%v", self.priority, self.dbTotal, self.total) + glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] deliver (db/total): %v/%v", self.key.Log(), self.priority, self.dbTotal, self.total) // by the time deliver returns, there were new writes to the buffer // if buffer contention is detected, switch to db mode which drains // the buffer so no process will block on pushing store requests if len(buffer) == cap(buffer) { - glog.V(logger.Debug).Infof("[BZZ] syncDb[%v] buffer full %v: switching to db. session tally (db/total): %v/%v", self.priority, cap(buffer), self.dbTotal, self.total) + glog.V(logger.Debug).Infof("[BZZ] syncDb[%v/%v] buffer full %v: switching to db. session tally (db/total): %v/%v", self.key.Log(), self.priority, cap(buffer), self.dbTotal, self.total) buffer = nil db = self.buffer } @@ -154,18 +154,18 @@ LOOP: binary.BigEndian.PutUint64(counterValue, counter) batch.Put(self.counterKey, counterValue) // persist counter in batch self.writeSyncBatch(batch) // save batch - glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] quitting: save current batch to db", self.priority) + glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] quitting: save current batch to db", self.key.Log(), self.priority) break LOOP } self.dbTotal++ self.total++ - // otherwise break after selec + // otherwise break after select case dbSize = <-self.batch: // explicit request for batch if inBatch == 0 && quit != nil { // there was no writes since the last batch so db depleted // switch to buffer mode - glog.V(logger.Debug).Infof("[BZZ] syncDb[%v] empty db: switching to buffer", self.priority) + glog.V(logger.Debug).Infof("[BZZ] syncDb[%v/%v] empty db: switching to buffer", self.key.Log(), self.priority) db = nil buffer = self.buffer dbSize <- 0 // indicates to 'caller' that batch has been written @@ -174,7 +174,7 @@ LOOP: } binary.BigEndian.PutUint64(counterValue, counter) batch.Put(self.counterKey, counterValue) - glog.V(logger.Debug).Infof("[BZZ] syncDb[%v] write batch %v/%v - %x - %x", self.priority, inBatch, counter, self.counterKey, counterValue) + glog.V(logger.Debug).Infof("[BZZ] syncDb[%v/%v] write batch %v/%v - %x - %x", self.key.Log(), self.priority, inBatch, counter, self.counterKey, counterValue) batch = self.writeSyncBatch(batch) dbSize <- inBatch // indicates to 'caller' that batch has been written inBatch = 0 @@ -186,7 +186,7 @@ LOOP: db = self.buffer buffer = nil quit = nil - glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] quitting: save buffer to db", self.priority) + glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] quitting: save buffer to db", self.key.Log(), self.priority) close(db) continue LOOP } @@ -194,15 +194,15 @@ LOOP: // only get here if we put req into db entry, err = self.newSyncDbEntry(req, counter) if err != nil { - glog.V(logger.Warn).Infof("[BZZ] syncDb[%v] saving request %v (#%v/%v) failed: %v", self.priority, req, inBatch, inDb, err) + glog.V(logger.Warn).Infof("[BZZ] syncDb[%v/%v] saving request %v (#%v/%v) failed: %v", self.key.Log(), self.priority, req, inBatch, inDb, err) continue LOOP } batch.Put(entry.key, entry.val) - glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] to batch %v '%v' (#%v/%v/%v)", self.priority, req, entry, inBatch, inDb, counter) + glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] to batch %v '%v' (#%v/%v/%v)", self.key.Log(), self.priority, req, entry, inBatch, inDb, counter) // if just switched to db mode and not quitting, then launch dbRead // in a parallel go routine to send deliveries from db if inDb == 0 && quit != nil { - glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] start dbRead") + glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] start dbRead") go self.dbRead(true, counter, deliver) } inDb++ @@ -221,7 +221,7 @@ LOOP: func (self *syncDb) writeSyncBatch(batch *leveldb.Batch) *leveldb.Batch { err := self.db.Write(batch) if err != nil { - glog.V(logger.Warn).Infof("[BZZ] syncDb[%v] saving batch to db failed: %v", self.priority, err) + glog.V(logger.Warn).Infof("[BZZ] syncDb[%v/%v] saving batch to db failed: %v", self.key.Log(), self.priority, err) return batch } return new(leveldb.Batch) @@ -295,7 +295,7 @@ func (self *syncDb) dbRead(useBatches bool, counter uint64, fun func(interface{} continue } del = new(leveldb.Batch) - glog.V(logger.Detail).Infof("[BZZ] syncDb[%v]: new iterator: %x (batch %v, count %v)", self.priority, key, batches, cnt) + glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v]: new iterator: %x (batch %v, count %v)", self.key.Log(), self.priority, key, batches, cnt) for n = 0; !useBatches || n < cnt; it.Next() { copy(key, it.Key()) @@ -307,11 +307,11 @@ func (self *syncDb) dbRead(useBatches bool, counter uint64, fun func(interface{} val := make([]byte, 40) copy(val, it.Value()) entry = &syncDbEntry{key, val} - // glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] - %v, batches: %v, total: %v, session total from db: %v/%v", self.priority, self.key.Log(), batches, total, self.dbTotal, self.total) + // glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] - %v, batches: %v, total: %v, session total from db: %v/%v", self.key.Log(), self.priority, self.key.Log(), batches, total, self.dbTotal, self.total) more = fun(entry, self.quit) if !more { // quit received when waiting to deliver entry, the entry will not be deleted - glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] batch %v quit after %v/%v items", self.priority, batches, n, cnt) + glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] batch %v quit after %v/%v items", self.key.Log(), self.priority, batches, n, cnt) break } // since subsequent batches of the same db session are indexed incrementally From fb83c1fcecc2ff8551d25f2061a02821e82f2dcf Mon Sep 17 00:00:00 2001 From: zelig Date: Tue, 14 Jun 2016 11:47:39 +0100 Subject: [PATCH 05/24] swarm/storage: chunker join fix process leak and keep parallelisation limited to depth 1 --- swarm/storage/chunker.go | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index 156b64d7e0..c3c4997b4f 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -336,17 +336,14 @@ func (self *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) { wg.Wait() close(self.errC) }() - select { - case err = <-self.errC: - // glog.V(logger.Detail).Infof("[BZZ] ReadAt received %v", err) - read = len(b) - if off+int64(read) == self.size { - err = io.EOF - } - // glog.V(logger.Detail).Infof("[BZZ] ReadAt returning at %d: %v", read, err) - case <-self.quitC: - // glog.V(logger.Detail).Infof("[BZZ] ReadAt aborted at %d: %v", read, err) + + err = <-self.errC + // glog.V(logger.Detail).Infof("[BZZ] ReadAt received %v", err) + read = len(b) + if off+int64(read) == self.size { + err = io.EOF } + // glog.V(logger.Detail).Infof("[BZZ] ReadAt returning at %d: %v", read, err) return } @@ -374,10 +371,10 @@ func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, tr return // simply give back the chunks reader for content chunks } - // subtree index + // subtree start := off / treeSize end := (eoff + treeSize - 1) / treeSize - wg := sync.WaitGroup{} + wg := &sync.WaitGroup{} for i := start; i < end; i++ { @@ -391,7 +388,9 @@ func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, tr if seoff > eoff { seoff = eoff } - + if depth > 1 { + wg.Wait() + } wg.Add(1) go func(j int64) { childKey := chunk.SData[8+j*self.chunker.hashSize : 8+(j+1)*self.chunker.hashSize] @@ -416,10 +415,13 @@ func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, tr soff = off } if len(ch.SData) == 0 { - self.errC <- fmt.Errorf("chunk %v-%v not found", off, off+treeSize) + select { + case self.errC <- fmt.Errorf("chunk %v-%v not found", off, off+treeSize): + case <-self.quitC: + } return } - self.join(b[soff-off:seoff-off], soff-roff, seoff-roff, depth-1, treeSize/self.chunker.branches, ch, &wg) + self.join(b[soff-off:seoff-off], soff-roff, seoff-roff, depth-1, treeSize/self.chunker.branches, ch, wg) }(i) } //for wg.Wait() From df5c8f3ab5a158f1124f20a88929f1a3a903494f Mon Sep 17 00:00:00 2001 From: zelig Date: Tue, 14 Jun 2016 12:43:04 +0100 Subject: [PATCH 06/24] swarm/network/kadenlia: simplify and fix code that deletes expired/unconnectable nodes --- swarm/network/kademlia/kaddb.go | 37 ++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/swarm/network/kademlia/kaddb.go b/swarm/network/kademlia/kaddb.go index 37900a141a..6f3643a48f 100644 --- a/swarm/network/kademlia/kaddb.go +++ b/swarm/network/kademlia/kaddb.go @@ -162,7 +162,7 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe var interval time.Duration var found bool var count int - var purge []int + var purge []bool var delta time.Duration var cursor int var after time.Time @@ -182,6 +182,7 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe need = true } cursor = self.cursors[po] + purge = make([]bool, len(dbrow)) // there is a missing slot - finding a node to connect to // select a node record from the relavant kaddb row (of identical prox order) @@ -208,7 +209,7 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe } if delta > self.purgeInterval { // remove node - purge = append(purge, cursor) + purge[cursor] = true glog.V(logger.Debug).Infof("[KΛÐ]: kaddb record %v (PO%03d:%d) unreachable since %v. Removed", node.Addr, po, cursor, node.Seen) continue ROW } @@ -225,7 +226,7 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe break ROW } // ROW self.cursors[po] = cursor - self.delete(po, purge...) + self.delete(po, purge) if found { return node, true, proxLimit } @@ -241,22 +242,23 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe // deletes the noderecords of a kaddb row corresponding to the indexes // caller must hold the dblock // the call is unsafe, no index checks -func (self *KadDb) delete(row int, indexes ...int) { - var prev int +func (self *KadDb) delete(row int, purge []bool) { var nodes []*NodeRecord dbrow := self.Nodes[row] - for _, next := range indexes { - // need to adjust dbcursor - if next <= self.cursors[row] { - self.cursors[row]-- + for i, del := range purge { + if i == self.cursors[row] { + //reset cursor + self.cursors[row] = len(nodes) } - if next > 0 { - nodes = append(nodes, dbrow[prev:next]...) + // delete the entry to be purged + if del { + delete(self.index, dbrow[i].Addr) + continue } - delete(self.index, dbrow[next].Addr) - prev = next + 1 + // otherwise append to new list + nodes = append(nodes, dbrow[i]) } - self.Nodes[row] = append(nodes, dbrow[prev:]...) + self.Nodes[row] = nodes } // save persists kaddb on disk (written to file on path in json format. @@ -306,14 +308,15 @@ func (self *KadDb) load(path string, cb func(*NodeRecord, Node) error) (err erro return } var n int - var purge []int + var purge []bool for po, b := range self.Nodes { + purge = make([]bool, len(b)) ROW: for i, node := range b { if cb != nil { err = cb(node, node.node) if err != nil { - purge = append(purge, i) + purge[i] = true continue ROW } } @@ -323,7 +326,7 @@ func (self *KadDb) load(path string, cb func(*NodeRecord, Node) error) (err erro } self.index[node.Addr] = node } - self.delete(po, purge...) + self.delete(po, purge) } glog.V(logger.Info).Infof("[KΛÐ] loaded kaddb with %v nodes from %v", n, path) From f72d3b26a2ddc902f467b5c8299f9244ae20b8f8 Mon Sep 17 00:00:00 2001 From: zelig Date: Wed, 15 Jun 2016 10:50:55 +0100 Subject: [PATCH 07/24] swarm/network: kaddb.findBest does not get stuck on empty row but finds an actual missing node --- swarm/network/hive.go | 5 +++-- swarm/network/kademlia/kaddb.go | 7 ++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/swarm/network/hive.go b/swarm/network/hive.go index cb38787b63..9f21440724 100644 --- a/swarm/network/hive.go +++ b/swarm/network/hive.go @@ -131,7 +131,8 @@ func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPee // enode or any lower level connection address is unnecessary in future // discovery table is used to look it up. connectPeer(node.Url) - } else if need { + } + if need { // a random peer is taken from the table peers := self.kad.FindClosest(kademlia.RandomAddressAt(self.addr, rand.Intn(self.kad.MaxProx)), 1) if len(peers) > 0 { @@ -342,7 +343,7 @@ func (self *Hive) peers(req *retrieveRequestMsgData) { for _, peer := range self.getPeers(key, int(req.MaxPeers)) { addrs = append(addrs, peer.remoteAddr) } - glog.V(logger.Debug).Infof("[BZZ] Hive sending %d peer addresses to %v. req.Id: %v, req.Key: %x", len(addrs), req.from, req.Id, req.Key.Log()) + glog.V(logger.Debug).Infof("[BZZ] Hive sending %d peer addresses to %v. req.Id: %v, req.Key: %v", len(addrs), req.from, req.Id, req.Key.Log()) peersData := &peersMsgData{ Peers: addrs, diff --git a/swarm/network/kademlia/kaddb.go b/swarm/network/kademlia/kaddb.go index 6f3643a48f..959a70beab 100644 --- a/swarm/network/kademlia/kaddb.go +++ b/swarm/network/kademlia/kaddb.go @@ -228,15 +228,12 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe self.cursors[po] = cursor self.delete(po, purge) if found { - return node, true, proxLimit + return node, need, proxLimit } } // ROUND - if need { - return nil, true, proxLimit - } } // ROUNDS - return nil, false, proxLimit + return nil, need, proxLimit } // deletes the noderecords of a kaddb row corresponding to the indexes From babb3a752687a87385bb3acdb9ab8b53c26b2d74 Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 16 Jun 2016 11:26:00 +0100 Subject: [PATCH 08/24] swarm/network/kademlia+hive: unwanted peers (due to full kad bucket) are now properly dropped with ErrUnwanted --- swarm/network/hive.go | 19 +++++++++++++------ swarm/network/kademlia/kademlia.go | 10 +++++----- swarm/network/protocol.go | 10 +++++++--- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/swarm/network/hive.go b/swarm/network/hive.go index 9f21440724..dcf666d0b9 100644 --- a/swarm/network/hive.go +++ b/swarm/network/hive.go @@ -198,9 +198,18 @@ func (self *Hive) Stop() error { } // called at the end of a successful protocol handshake -func (self *Hive) addPeer(p *peer) { +func (self *Hive) addPeer(p *peer) error { + defer func() { + select { + case self.more <- true: + default: + } + }() glog.V(logger.Detail).Infof("[BZZ] KΛÐΞMLIΛ hive: hi new bee %v", p) - self.kad.On(p, loadSync) + err := self.kad.On(p, loadSync) + if err != nil { + return err + } // self lookup (can be encoded as nil/zero key since peers addr known) + no id () // the most common way of saying hi in bzz is initiation of gossip // let me know about anyone new from my hood , here is the storageradius @@ -208,10 +217,8 @@ func (self *Hive) addPeer(p *peer) { // we do not record as request or forward it, just reply with peers p.retrieve(&retrieveRequestMsgData{}) glog.V(logger.Detail).Infof("[BZZ] KΛÐΞMLIΛ hive: 'whatsup wheresdaparty' sent to %v", p) - select { - case self.more <- true: - default: - } + + return nil } // called after peer disconnected diff --git a/swarm/network/kademlia/kademlia.go b/swarm/network/kademlia/kademlia.go index 50fb6b6f2c..e635c042c2 100644 --- a/swarm/network/kademlia/kademlia.go +++ b/swarm/network/kademlia/kademlia.go @@ -136,13 +136,14 @@ func (self *Kademlia) On(node Node, cb func(*NodeRecord, Node) error) (err error } if replaced != nil { glog.V(logger.Debug).Infof("[KΛÐ]: node %v replaced by %v ", replaced, node) - return + replaced.Drop() + return nil } // new node added glog.V(logger.Info).Infof("[KΛÐ]: add node %v to table", node) self.count++ self.setProxLimit(index, false) - return + return nil } // is the entrypoint called when a node is taken offline @@ -161,7 +162,8 @@ func (self *Kademlia) Off(node Node, cb func(*NodeRecord, Node)) (err error) { } if !found { - return + // gracefully return without error if peer already offline + return nil } glog.V(logger.Info).Infof("[KΛÐ]: remove node %v from table", node) @@ -215,8 +217,6 @@ func (self *Kademlia) setProxLimit(r int, off bool) { self.proxLimit++ glog.V(logger.Detail).Infof("[KΛÐ]: proxbin contraction (size: %v, limit: %v, bin: %v, off: %v)", self.proxSize, self.proxLimit, r, off) } - // glog.V(logger.Detail).Infof("%v", self) - } /* diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index 6f27f68b26..e4e247d458 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -49,6 +49,7 @@ const ( ErrExtraStatusMsg ErrSwap ErrSync + ErrUnwanted ) var errorToString = map[int]string{ @@ -61,6 +62,7 @@ var errorToString = map[int]string{ ErrExtraStatusMsg: "Extra status message", ErrSwap: "SWAP error", ErrSync: "Sync error", + ErrUnwanted: "Unwanted peer", } // bzz represents the swarm wire protocol @@ -254,7 +256,7 @@ func (self *bzz) handle() error { return self.protoError(ErrDecode, "<- %v: %v", msg, err) } req.from = &peer{bzz: self} - glog.V(logger.Debug).Infof("[BZZ] <- peer addresses: %v", req) + glog.V(logger.Detail).Infof("[BZZ] <- peer addresses: %v", req) self.hive.HandlePeersMsg(&req, &peer{bzz: self}) case syncRequestMsg: @@ -366,7 +368,10 @@ func (self *bzz) handleStatus() (err error) { } glog.V(logger.Info).Infof("[BZZ] Peer %08x is [bzz] capable (%d/%d)", self.remoteAddr.Addr[:4], status.Version, status.NetworkId) - self.hive.addPeer(&peer{bzz: self}) + err = self.hive.addPeer(&peer{bzz: self}) + if err != nil { + return self.protoError(ErrUnwanted, "%v", err) + } // hive sets syncstate so sync should start after node added glog.V(logger.Info).Infof("[BZZ] syncronisation request sent with %v", self.syncState) @@ -516,7 +521,6 @@ func (self *bzz) send(msg uint64, data interface{}) error { if self.hive.blockWrite { return fmt.Errorf("network write blocked") } - // self.messages = append(self.messages, "") glog.V(logger.Detail).Infof("[BZZ] -> %v: %v (%T) to %v", msg, data, data, self) err := p2p.Send(self.rw, msg, data) if err != nil { From 0e23886184c2cbbc669288fa9fc28f0d3ac20520 Mon Sep 17 00:00:00 2001 From: zelig Date: Wed, 22 Jun 2016 14:44:13 +0200 Subject: [PATCH 09/24] improve logging and minor reorg in kademlia --- swarm/network/hive.go | 13 ++++--- swarm/network/kademlia/kademlia.go | 2 +- swarm/network/kademlia/kademlia_test.go | 2 +- swarm/network/syncer.go | 46 ++++++++++++++----------- 4 files changed, 34 insertions(+), 29 deletions(-) diff --git a/swarm/network/hive.go b/swarm/network/hive.go index dcf666d0b9..bb01365a09 100644 --- a/swarm/network/hive.go +++ b/swarm/network/hive.go @@ -123,11 +123,10 @@ func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPee // to attempt to write to more (remove Peer when shutting down) return } - node, need, proxLimit := self.kad.FindBest() + node, need, proxLimit := self.kad.Suggest() - glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: select candidate peer") if node != nil && len(node.Url) > 0 { - glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: call for bee %v", node.Url) + glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: call known bee %v", node.Url) // enode or any lower level connection address is unnecessary in future // discovery table is used to look it up. connectPeer(node.Url) @@ -141,15 +140,15 @@ func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPee req := &retrieveRequestMsgData{ Key: storage.Key(randAddr[:]), } - glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: call any bee in area %v messenger bee %v", randAddr, peers[0]) + glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: call any bee near %v (PO%03d) - messenger bee: %v", randAddr, proxLimit, peers[0]) peers[0].(*peer).retrieve(req) } else { - glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: no peer") + glog.V(logger.Warn).Infof("[BZZ] KΛÐΞMLIΛ hive: no peer") } self.toggle <- true - glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: buzz kept alive") + glog.V(logger.Detail).Infof("[BZZ] KΛÐΞMLIΛ hive: buzz kept alive") } else { - glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: no need for more bees") + glog.V(logger.Info).Infof("[BZZ] KΛÐΞMLIΛ hive: no need for more bees") self.toggle <- false } glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: queen's address: %v, population: %d (%d)", self.addr, self.kad.Count(), self.kad.DBCount()) diff --git a/swarm/network/kademlia/kademlia.go b/swarm/network/kademlia/kademlia.go index e635c042c2..6df255e4a1 100644 --- a/swarm/network/kademlia/kademlia.go +++ b/swarm/network/kademlia/kademlia.go @@ -279,7 +279,7 @@ func (self *Kademlia) binsize(p int) int { return len(b.nodes) } -func (self *Kademlia) FindBest() (*NodeRecord, bool, int) { +func (self *Kademlia) Suggest() (*NodeRecord, bool, int) { return self.db.findBest(self.BucketSize, self.binsize) } diff --git a/swarm/network/kademlia/kademlia_test.go b/swarm/network/kademlia/kademlia_test.go index ca05e1ded5..de906cabe6 100644 --- a/swarm/network/kademlia/kademlia_test.go +++ b/swarm/network/kademlia/kademlia_test.go @@ -82,7 +82,7 @@ func TestBootstrap(t *testing.T) { t.Fatalf("backend not accepting node: %v", err) } - record, need, _ := kad.FindBest() + record, need, _ := kad.Suggest() if !need { break } diff --git a/swarm/network/syncer.go b/swarm/network/syncer.go index 80893336b7..f778773001 100644 --- a/swarm/network/syncer.go +++ b/swarm/network/syncer.go @@ -298,6 +298,7 @@ func (self *syncer) sync() { if state.LastSeenAt < state.SessionAt { state.Last = state.SessionAt glog.V(logger.Debug).Infof("[BZZ] syncer[%v]: start syncronising history since last disconnect at %v up until session start at %v: %v", self.key.Log(), state.LastSeenAt, state.SessionAt, state) + // blocks until state syncing is finished self.syncState(state) } glog.V(logger.Info).Infof("[BZZ] syncer[%v]: syncing all history complete", self.key.Log()) @@ -358,19 +359,18 @@ func (self *syncer) syncHistory(state *syncState) chan interface{} { IT: for { key := it.Next() - if key != nil { - select { - // blocking until history channel is read from - case history <- storage.Key(key): - n++ - glog.V(logger.Detail).Infof("[BZZ] syncer[%v]: history: %v (%v keys)", self.key.Log(), key.Log(), n) - state.Latest = key - case <-self.quit: - return - } - } else { + if key == nil { break IT } + select { + // blocking until history channel is read from + case history <- storage.Key(key): + n++ + glog.V(logger.Detail).Infof("[BZZ] syncer[%v]: history: %v (%v keys)", self.key.Log(), key.Log(), n) + state.Latest = key + case <-self.quit: + return + } } glog.V(logger.Debug).Infof("[BZZ] syncer[%v]: finished syncing history between %v - %v for chunk addresses %v - %v (at %v) (chunks = %v)", self.key.Log(), state.First, state.Last, state.Start, state.Stop, state.Latest, n) }() @@ -416,25 +416,31 @@ LOOP: // are checked first - integrity can only be guaranteed if writing // is locked while selecting if priority != High || len(keys) == 0 { + // selection is not needed if the High priority queue has items keys = nil + PRIORITIES: for priority = High; priority >= 0; priority-- { + // the first priority channel that is non-empty will be assigned to keys if len(self.keys[priority]) > 0 { glog.V(logger.Detail).Infof("[BZZ] syncer[%v]: reading request with priority %v", self.key.Log(), priority) keys = self.keys[priority] - break + break PRIORITIES } + glog.V(logger.Debug).Infof("[BZZ] syncer[%v/%v]: queue: [%v, %v, %v]", self.key.Log(), priority, len(self.keys[High]), len(self.keys[High]), len(self.keys[High])) + // if the input queue is empty on this level, resort to history if there is any if uint(priority) == histPrior && history != nil { glog.V(logger.Detail).Infof("[BZZ] syncer[%v]: reading history for %v", self.key.Log(), self.key) keys = history - break + break PRIORITIES } } - // if peer ready to receive but nothing to send - if keys == nil && deliveryRequest == nil { - // if no items left and switch to waiting mode - glog.V(logger.Detail).Infof("[BZZ] syncer[%v]: buffers consumed. Waiting", self.key.Log()) - newUnsyncedKeys = self.newUnsyncedKeys - } + } + + // if peer ready to receive but nothing to send + if keys == nil && deliveryRequest == nil { + // if no items left and switch to waiting mode + glog.V(logger.Detail).Infof("[BZZ] syncer[%v]: buffers consumed. Waiting", self.key.Log()) + newUnsyncedKeys = self.newUnsyncedKeys } // send msg iff @@ -447,7 +453,7 @@ LOOP: len(unsynced) > 0 && keys == nil || len(unsynced) == int(self.SyncBatchSize)) { justSynced = false - // listen to requests again + // listen to requests deliveryRequest = self.deliveryRequest newUnsyncedKeys = nil // not care about data until next req comes in // set sync to current counter From c6205b224467c003958a867b84319e0af9edf80a Mon Sep 17 00:00:00 2001 From: zelig Date: Wed, 22 Jun 2016 15:07:26 +0200 Subject: [PATCH 10/24] swarm/cmd: add cleanlog, cleanbzz, update-src, remote-update-scripts, remote-update-bin, remote-run --- swarm/cmd/swarm/env.sh | 9 +++++ swarm/cmd/swarm/gethup.sh | 15 +++----- swarm/cmd/swarm/swarm | 2 ++ swarm/cmd/swarm/swarm.sh | 75 +++++++++++++++++++++++++++++++++++++-- 4 files changed, 87 insertions(+), 14 deletions(-) create mode 100644 swarm/cmd/swarm/env.sh create mode 100755 swarm/cmd/swarm/swarm diff --git a/swarm/cmd/swarm/env.sh b/swarm/cmd/swarm/env.sh new file mode 100644 index 0000000000..634f316fb7 --- /dev/null +++ b/swarm/cmd/swarm/env.sh @@ -0,0 +1,9 @@ +export GOPATH=~/go +export PATH=~/bin:$GOPATH/bin:$PATH + +if [ -f ~/.bash_aliases ]; then + . ~/.bash_aliases +fi + +export NVM_DIR="/home/ubuntu/.nvm" +[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" # This loads nvm \ No newline at end of file diff --git a/swarm/cmd/swarm/gethup.sh b/swarm/cmd/swarm/gethup.sh index dcf2391d95..8d8925cd62 100644 --- a/swarm/cmd/swarm/gethup.sh +++ b/swarm/cmd/swarm/gethup.sh @@ -17,13 +17,13 @@ shift # ls -l $GETH # geth CLI params e.g., (dd=04, run=09) -datetag=`date "+%c%y%m%d-%H%M%S"|cut -d ' ' -f 5` +datetag=`date "+%Y-%m-%d-%H:%M:%S"` datadir=$root/data/$id # /tmp/eth/04 log=$root/log/$id.$datetag.log # /tmp/eth/04.09.log linklog=$root/log/$id.log # /tmp/eth/04.09.log password=$id # 04 port=303$id # 34504 -bzzport=322$id # 32204 +bzzport=322$id # 3 2204 rpcport=302$id # 3204 mkdir -p $root/data @@ -43,8 +43,8 @@ if [ ! -d "$keystoredir" ]; then # note that the account key will be stored also separately outside # datadir # this way you can safely clear the data directory and still keep your key - # under `/keystore/dd - # LS=`ls $datadir/keystore` + # under /keystore/dd + # LS=$(ls $datadir/keystore) # echo $LS while [ ! -d "$keystoredir" ]; do echo "." @@ -57,13 +57,6 @@ if [ ! -d "$keystoredir" ]; then cp -R "$datadir/keystore/" $root/keystore/$id fi -# # mkdir -p $datadir/keystore -# if [ ! -d "$datadir/keystore" ]; then -# echo "copying keys $root/keystore/$id $datadir/keystore" -# cp -R $root/keystore/$id/keystore/ $datadir/keystore/ -# fi - - # query node's enode url if [ $ip_addr="" ]; then pattern='\d+\.\d+\.\d+\.\d+' diff --git a/swarm/cmd/swarm/swarm b/swarm/cmd/swarm/swarm new file mode 100755 index 0000000000..5289aba5a3 --- /dev/null +++ b/swarm/cmd/swarm/swarm @@ -0,0 +1,2 @@ +#!/bin/bash +bash ~/bin/swarm.sh $SWARM_DIR $SWARM_NETWORK_ID $* \ No newline at end of file diff --git a/swarm/cmd/swarm/swarm.sh b/swarm/cmd/swarm/swarm.sh index 8e3e2bb9c1..4ca75c82a5 100644 --- a/swarm/cmd/swarm/swarm.sh +++ b/swarm/cmd/swarm/swarm.sh @@ -44,8 +44,8 @@ function attach { id=$1 shift echo "attaching console to instance $id" - cmd="$GETH $* attach ipc:$root/$network_id/data/$id/geth.ipc" - # echo $cmd + cmd="$GETH $* attach ipc:$root/$network_id/data/$id/geth.ipc" + echo $cmd eval $cmd } @@ -58,6 +58,30 @@ function log { eval $cmd } +function cleanlog { + id=$1 + shift + if [ $id = "all" ]; then + echo "remove logs for all instances" + rm -rf "$root/$network_id/log/" + else + echo "remove logs for instance $id" + rm -rf $root/$network_id/log/$id* + fi +} + +function cleanbzz { + id=$1 + shift + if [ $id = "all" ]; then + echo "remove bzz data for all instances" + rm -rf $root/$network_id/data/*/bzz + else + echo "remove bzz data for instance $id" + rm -rf "$root/$network_id/data/$id" + fi +} + function less { id=$1 shift @@ -319,6 +343,37 @@ function netstatconf { echo "]" >> $conf } +function remote-update-scripts { + scriptdir=$1 + remotes=$2 + cd $GETH_DIR + for remote in `cat $remotes|grep -v '^#'`; do echo "updating scripts on $remote..."; ssh $remote mkdir -p bin && scp -r $scriptdir/* $remote:bin/; done +} + +function remote-update-bin { + remote-update-scripts ~/bin $remotes + for remote in `cat $remotes|grep -v '^#'`; do echo "updating binary on $remote..."; scp -r $GETH_DIR/geth $remote:bin/; done +} + +function remote-run { + remotes=$1 + shift + for remote in `cat $remotes|grep -v '^#'`; do echo "running on $remote..."; ssh $remote ". ~/bin/env.sh; $*"; done +} + +function update-src { + branch=$1 + echo "cd $GETH_DIR && git remote update && git reset --hard $branch" + (cd $GETH_DIR && git remote update && git reset --hard $branch) +} + +function netstatrun { + cd ~/eth-net-intelligence-api + pm2 kill + pm2 start $root/$network_id/*.netstat.json +} + + case $cmd in "info" ) info $*;; @@ -352,11 +407,25 @@ case $cmd in cluster $*;; "attach" ) attach $*;; + "cleanbzz" ) + cleanbzz $*;; + "cleanlog" ) + cleanlog $*;; "log" ) log $*;; "less" ) less $*;; - "netstatconf" ) + "remote-update-scripts" ) + remote-update-scripts $*;; + "remote-update-bin" ) + remote-update-bin $*;; + "update-src" ) + update-src $*;; + "remote-run" ) + remote-run $*;; + "netstatconf" ) netstatconf $*;; + "netstatrun" ) + netstatrun $*;; esac From cb7e6ccd5bc83cfb1573eb42f93009dd22d1e875 Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 23 Jun 2016 18:56:35 +0200 Subject: [PATCH 11/24] swarm/storage: make NetStore.Put -> cloud.Store synchronous --- swarm/storage/netstore.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/swarm/storage/netstore.go b/swarm/storage/netstore.go index 9d61feb45b..eb8cfec893 100644 --- a/swarm/storage/netstore.go +++ b/swarm/storage/netstore.go @@ -91,7 +91,8 @@ func (self *NetStore) Put(entry *Chunk) { } else { glog.V(logger.Detail).Infof("[BZZ] NetStore.Put: localStore.Put %v stored locally", entry.Key.Log()) // handle propagating store requests - go self.cloud.Store(entry) + // go self.cloud.Store(entry) + self.cloud.Store(entry) } } From e01ae3cded0e4443575b16c4f0a6a1546eda6fc9 Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 23 Jun 2016 18:57:36 +0200 Subject: [PATCH 12/24] swarm/network: syncer logs queue cardinalities properly in syncUnSyncedKeys loop --- swarm/network/syncer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/network/syncer.go b/swarm/network/syncer.go index f778773001..3bbb1ed7d3 100644 --- a/swarm/network/syncer.go +++ b/swarm/network/syncer.go @@ -426,7 +426,7 @@ LOOP: keys = self.keys[priority] break PRIORITIES } - glog.V(logger.Debug).Infof("[BZZ] syncer[%v/%v]: queue: [%v, %v, %v]", self.key.Log(), priority, len(self.keys[High]), len(self.keys[High]), len(self.keys[High])) + glog.V(logger.Debug).Infof("[BZZ] syncer[%v/%v]: queue: [%v, %v, %v]", self.key.Log(), priority, len(self.keys[High]), len(self.keys[Medium]), len(self.keys[Low])) // if the input queue is empty on this level, resort to history if there is any if uint(priority) == histPrior && history != nil { glog.V(logger.Detail).Infof("[BZZ] syncer[%v]: reading history for %v", self.key.Log(), self.key) From 1e9b3c1583ccc7e9342626a03dc73286fc603428 Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 23 Jun 2016 19:09:06 +0200 Subject: [PATCH 13/24] swarm/cmd: for remote binary update, take scripts from swarm/cmd/swarm --- swarm/cmd/swarm/swarm.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/swarm/cmd/swarm/swarm.sh b/swarm/cmd/swarm/swarm.sh index 4ca75c82a5..34dafda44c 100644 --- a/swarm/cmd/swarm/swarm.sh +++ b/swarm/cmd/swarm/swarm.sh @@ -351,7 +351,8 @@ function remote-update-scripts { } function remote-update-bin { - remote-update-scripts ~/bin $remotes + remotes=$1 + remote-update-scripts $GETH_DIR/swarm/cmd/swarm/ $remotes for remote in `cat $remotes|grep -v '^#'`; do echo "updating binary on $remote..."; scp -r $GETH_DIR/geth $remote:bin/; done } From 2b1c9501931cdfc662e2a79cfc983c095a6a1396 Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 23 Jun 2016 20:22:48 +0200 Subject: [PATCH 14/24] swarm/network: must replace node if bucket is full otherwise nodes will get stuck on empty rows --- swarm/network/kademlia/kademlia.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/swarm/network/kademlia/kademlia.go b/swarm/network/kademlia/kademlia.go index 6df255e4a1..6b7e8066ac 100644 --- a/swarm/network/kademlia/kademlia.go +++ b/swarm/network/kademlia/kademlia.go @@ -340,7 +340,11 @@ func (self *bucket) insert(node Node) (replaced Node, err error) { if len(self.nodes) >= self.size { // >= allows us to add peers beyond the bucketsize limitation // dev p2p kicks out nodes idle for > 30 s, so here we never replace nodes if // bucket is full - return nil, fmt.Errorf("bucket full") + // update, it seems we need to replace nodes + // return nil, fmt.Errorf("bucket full") + replaced := self.nodes[0] + self.nodes = append(self.nodes[1:], node) + return replaced, nil } self.nodes = append(self.nodes, node) return From 38184bf18da60174b43b2b5e4ba032ef80a992dc Mon Sep 17 00:00:00 2001 From: zelig Date: Fri, 24 Jun 2016 14:48:09 +0200 Subject: [PATCH 15/24] swarm/api/http: server handler: check protocol substring length to fix slice out of bounds crash --- swarm/api/http/server.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 945fc4f6e0..29945a9f77 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -86,8 +86,10 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { return } } - raw = proto[1:5] == "bzzr" - nameresolver = proto[1:5] != "bzzi" + if len(proto) > 4 { + raw = proto[1:5] == "bzzr" + nameresolver = proto[1:5] != "bzzi" + } glog.V(logger.Debug).Infof( "[BZZ] Swarm: %s request over protocol %s '%s' received.", From 7aee835b36d7bf8fb68e29a8a93610d0104ca3fb Mon Sep 17 00:00:00 2001 From: zelig Date: Fri, 24 Jun 2016 14:48:46 +0200 Subject: [PATCH 16/24] swarm/cmd/swarm: raise default maxpeers to 40 --- swarm/cmd/swarm/swarm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/cmd/swarm/swarm.sh b/swarm/cmd/swarm/swarm.sh index 34dafda44c..dde74d6b71 100644 --- a/swarm/cmd/swarm/swarm.sh +++ b/swarm/cmd/swarm/swarm.sh @@ -37,7 +37,7 @@ shift # ip_addr=`curl ipecho.net/plain 2>/dev/null;echo ` # echo "external IP: $ip_addr" -swarmoptions='--dev --maxpeers=20 --shh=false --nodiscover' +swarmoptions='--dev --maxpeers=40 --shh=false --nodiscover' tmpdir=/tmp function attach { From 2371ac726573e4f5c914bda227f8603b9b2129a3 Mon Sep 17 00:00:00 2001 From: zelig Date: Sun, 26 Jun 2016 12:07:46 +0200 Subject: [PATCH 17/24] swarm/storage: reset the base hash to SHA3. !!hard fork = no backward compatibility :) --- swarm/storage/chunker.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index c3c4997b4f..d58eed6381 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -38,8 +38,8 @@ data_{i} := size(subtree_{i}) || key_{j} || key_{j+1} .... || key_{j+n-1} */ const ( - // defaultHash = "SHA3" // http://golang.org/pkg/hash/#Hash - defaultHash = "SHA256" // http://golang.org/pkg/hash/#Hash + defaultHash = "SHA3" // http://golang.org/pkg/hash/#Hash + // defaultHash = "SHA256" // http://golang.org/pkg/hash/#Hash defaultBranches int64 = 128 // hashSize int64 = hasherfunc.New().Size() // hasher knows about its own length in bytes // chunksize int64 = branches * hashSize // chunk is defined as this @@ -364,7 +364,8 @@ func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, tr // glog.V(logger.Detail).Infof("[BZZ] depth: %v, len(b): %v, off: %v, eoff: %v, chunk.Size: %v, treeSize: %v", depth, len(b), off, eoff, chunk.Size, treeSize) if int64(len(b)) != eoff-off { //fmt.Printf("len(b) = %v off = %v eoff = %v", len(b), off, eoff) - panic("len(b) does not match") + msg := fmt.Sprintf("len(b) = %v =?= %v = eoff-off", len(b), eoff-off) + panic(msg) } copy(b, chunk.SData[8+off:8+eoff]) From 58397953bc6394bccd8295bf0bea12e3cc4b27eb Mon Sep 17 00:00:00 2001 From: zelig Date: Sun, 26 Jun 2016 12:09:07 +0200 Subject: [PATCH 18/24] swarm/network: fix filesystem API - upload/Modify tests --- swarm/api/api_test.go | 16 ++++++++++------ swarm/api/config_test.go | 2 +- swarm/api/filesystem_test.go | 19 ++++++++++++++----- swarm/api/storage.go | 2 ++ 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index 0ee3ca8df7..7db25c1fda 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -1,7 +1,7 @@ package api import ( - // "bytes" + "io" "io/ioutil" "os" "testing" @@ -51,10 +51,8 @@ func checkResponse(t *testing.T, resp *testResponse, exp *Response) { resp.Content = string(content) } if resp.Content != exp.Content { - // if !bytes.Equal(resp.Content, exp.Content) { - t.Errorf("incorrect content. expected '%s...', got '%s...'", string(exp.Content), string(resp.Content)) - } -} + // if !bytes.Equal(resp.Content, exp.Content) + t.Errorf("incorrect content. expected '%s...', got '%s...'", string(exp.Content), string(resp.Con} // func expResponse(content []byte, mimeType string, status int) *Response { func expResponse(content string, mimeType string, status int) *Response { @@ -67,7 +65,13 @@ func testGet(t *testing.T, api *Api, bzzhash string) *testResponse { if err != nil { t.Fatalf("unexpected error: %v", err) } - return &testResponse{reader, &Response{mimeType, status, reader.Size(), ""}} + + s := make([]byte, reader.Size()) + _, err = reader.Read(s) + if err != io.EOF { + t.Fatalf("unexpected error: %v", err) + } + return &testResponse{reader, &Response{mimeType, status, reader.Size(), string(s)}} // return &testResponse{reader, &Response{mimeType, status, reader.Size(), nil}} } diff --git a/swarm/api/config_test.go b/swarm/api/config_test.go index 48fbfacb20..c394383e5a 100644 --- a/swarm/api/config_test.go +++ b/swarm/api/config_test.go @@ -19,7 +19,7 @@ var ( "CacheCapacity": 5000, "Radius": 0, "Branches": 128, - "Hash": "SHA256", + "Hash": "SHA3", "CallInterval": 3000000000, "KadDbPath": "` + filepath.Join("TMPDIR", "0d2f62485607cf38d9d795d93682a517661e513e", "bzz-peers.json") + `", "MaxProx": 8, diff --git a/swarm/api/filesystem_test.go b/swarm/api/filesystem_test.go index e0bb8915a9..c6be66e68f 100644 --- a/swarm/api/filesystem_test.go +++ b/swarm/api/filesystem_test.go @@ -1,12 +1,16 @@ package api import ( + "io" "io/ioutil" "os" "path" "runtime" "testing" -) + +"github.com/ethereum/go-ethereum/logger" + "github.com/ethereum/go-ethereum/logger/glog" + ) var ( testDir string @@ -17,8 +21,8 @@ func init() { _, filename, _, _ := runtime.Caller(1) testDir = path.Join(path.Dir(filename), "../test") testDownloadDir, _ = ioutil.TempDir(os.TempDir(), "bzz-test") -} - + glog.SetV(logger.Detail) + glog.SetV = nil func testFileSystem(t *testing.T, f func(*FileSystem)) { testApi(t, func(api *Api) { f(NewFileSystem(api)) @@ -29,6 +33,7 @@ func readPath(t *testing.T, parts ...string) string { // func readPath(t *testing.T, parts ...string) []byte { file := path.Join(parts...) content, err := ioutil.ReadFile(file) + if err != nil { t.Fatalf("unexpected error reading '%v': %v", file, err) } @@ -45,13 +50,17 @@ func TestApiDirUpload0(t *testing.T) { } content := readPath(t, testDir, "test0", "index.html") + t.Logf("content (%v): %v ", len(content), content) resp := testGet(t, api, bzzhash+"/index.html") - exp := expResponse(content, "text/html; charset=utf-8", 0) + exp := expRes ponse(content, "text/html; charset=utf-8", 0) + t.Logf("index.html (size%v=?=%v)", resp.Size, exp.Size) checkResponse(t, resp, exp) + t.FailNow() content = readPath(t, testDir, "test0", "index.css") resp = testGet(t, api, bzzhash+"/index.css") exp = expResponse(content, "text/css", 0) + t.Logf("index.css (size%v=?=%v)", resp.Size, exp.Size) checkResponse(t, resp, exp) content = readPath(t, testDir, "test0", "img", "logo.png") @@ -163,7 +172,7 @@ func TestApiFileUploadWithRootFile(t *testing.T) { testFileSystem(t, func(fs *FileSystem) { api := fs.api bzzhash, err := fs.Upload(path.Join(testDir, "test0", "index.html"), "index.html") - if err != nil { + if err != io.EOF { t.Errorf("unexpected error: %v", err) return } diff --git a/swarm/api/storage.go b/swarm/api/storage.go index 97b5590fdf..1fd3fa6008 100644 --- a/swarm/api/storage.go +++ b/swarm/api/storage.go @@ -43,6 +43,8 @@ func (self *Storage) Get(bzzpath string) (*Response, error) { return &Response{mimeType, status, expsize, string(body[:size])}, err } +// Modify(rootHash, path, contentHash, contentType) takes th e manifest trie rooted in rootHash, +// and merge on to it. creating an entry w conentType (mime) func (self *Storage) Modify(rootHash, path, contentHash, contentType string) (newRootHash string, err error) { return self.api.Modify(rootHash+"/"+path, contentHash, contentType, true) } From 0de16bd17e3ad01905898ffb786a2ebf0b1d8927 Mon Sep 17 00:00:00 2001 From: zelig Date: Tue, 28 Jun 2016 22:35:32 +0200 Subject: [PATCH 19/24] swarm/storage, swarm/api: fix chunker joiner and fs api * SectionReader -> LazySectionReader: Size method signature change * introduce abort channel to joiner to fix process leak * add abort channel context to all manifest retrieval methods * global waitgroup fixes intermittent upload test failures due to unfinished storage of chunks * add back final slash to paths in manifest matching * simplify downloader code and fix process leak * fix filesystem api tests * streamline joiner logic, contexts now unique to each readAt call * LazyChunkReader seeker complains about missing size only if whence=2 --- swarm/api/api.go | 15 +-- swarm/api/api_test.go | 21 +++- swarm/api/filesystem.go | 120 ++++++++++--------- swarm/api/filesystem_test.go | 46 ++++--- swarm/api/http/server.go | 10 +- swarm/api/manifest.go | 72 ++++++----- swarm/api/manifest_test.go | 7 +- swarm/api/storage.go | 6 +- swarm/storage/chunker.go | 219 +++++++++++++++++++--------------- swarm/storage/chunker_test.go | 8 +- swarm/storage/common_test.go | 9 +- swarm/storage/dpa.go | 4 +- swarm/storage/localstore.go | 5 + swarm/storage/types.go | 16 ++- 14 files changed, 318 insertions(+), 240 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index cc76b36ae5..321837bda8 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -43,7 +43,7 @@ func NewApi(dpa *storage.DPA, dns Resolver) (self *Api) { } // DPA reader API -func (self *Api) Retrieve(key storage.Key) storage.SectionReader { +func (self *Api) Retrieve(key storage.Key) storage.LazySectionReader { return self.dpa.Retrieve(key) } @@ -124,11 +124,11 @@ func (self *Api) Put(content, contentType string) (string, error) { // Get uses iterative manifest retrieval and prefix matching // to resolve path to content using dpa retrieve // it returns a section reader, mimeType, status and an error -func (self *Api) Get(uri string, nameresolver bool) (reader storage.SectionReader, mimeType string, status int, err error) { +func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionReader, mimeType string, status int, err error) { key, _, path, err := self.parseAndResolve(uri, nameresolver) - - trie, err := loadManifest(self.dpa, key) + quitC := make(chan bool) + trie, err := loadManifest(self.dpa, key, quitC) if err != nil { glog.V(logger.Warn).Infof("[BZZ] Swarm: loadManifestTrie error: %v", err) return @@ -151,7 +151,8 @@ func (self *Api) Get(uri string, nameresolver bool) (reader storage.SectionReade func (self *Api) Modify(uri, contentHash, contentType string, nameresolver bool) (newRootHash string, err error) { root, _, path, err := self.parseAndResolve(uri, nameresolver) - trie, err := loadManifest(self.dpa, root) + quitC := make(chan bool) + trie, err := loadManifest(self.dpa, root, quitC) if err != nil { return } @@ -162,9 +163,9 @@ func (self *Api) Modify(uri, contentHash, contentType string, nameresolver bool) Hash: contentHash, ContentType: contentType, } - trie.addEntry(entry) + trie.addEntry(entry, quitC) } else { - trie.deleteEntry(path) + trie.deleteEntry(path, quitC) } err = trie.recalcAndStore() diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index 7db25c1fda..0fb50e5793 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -6,6 +6,8 @@ import ( "os" "testing" + "github.com/ethereum/go-ethereum/logger" + "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/swarm/storage" ) @@ -27,7 +29,7 @@ func testApi(t *testing.T, f func(*Api)) { } type testResponse struct { - reader storage.SectionReader + reader storage.LazySectionReader *Response } @@ -52,10 +54,13 @@ func checkResponse(t *testing.T, resp *testResponse, exp *Response) { } if resp.Content != exp.Content { // if !bytes.Equal(resp.Content, exp.Content) - t.Errorf("incorrect content. expected '%s...', got '%s...'", string(exp.Content), string(resp.Con} + t.Errorf("incorrect content. expected '%s...', got '%s...'", string(exp.Content), string(resp.Content)) + } +} // func expResponse(content []byte, mimeType string, status int) *Response { func expResponse(content string, mimeType string, status int) *Response { + glog.V(logger.Detail).Infof("expected content (%v): %v ", len(content), content) return &Response{mimeType, status, int64(len(content)), content} } @@ -65,13 +70,19 @@ func testGet(t *testing.T, api *Api, bzzhash string) *testResponse { if err != nil { t.Fatalf("unexpected error: %v", err) } - - s := make([]byte, reader.Size()) + quitC := make(chan bool) + size, err := reader.Size(quitC) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + glog.V(logger.Detail).Infof("reader size: %v ", size) + s := make([]byte, size) _, err = reader.Read(s) if err != io.EOF { t.Fatalf("unexpected error: %v", err) } - return &testResponse{reader, &Response{mimeType, status, reader.Size(), string(s)}} + reader.Seek(0, 0) + return &testResponse{reader, &Response{mimeType, status, size, string(s)}} // return &testResponse{reader, &Response{mimeType, status, reader.Size(), nil}} } diff --git a/swarm/api/filesystem.go b/swarm/api/filesystem.go index 34f2a8f0c8..cc8fd2c818 100644 --- a/swarm/api/filesystem.go +++ b/swarm/api/filesystem.go @@ -86,6 +86,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) { errors := make([]error, cnt) done := make(chan bool, maxParallelFiles) dcnt := 0 + wg := &sync.WaitGroup{} for i, entry := range list { if i >= dcnt+maxParallelFiles { @@ -96,7 +97,6 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) { f, err := os.Open(entry.Path) if err == nil { stat, _ := f.Stat() - wg := &sync.WaitGroup{} var hash storage.Key hash, err = self.api.dpa.Store(f, stat.Size(), wg) if hash != nil { @@ -128,6 +128,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) { trie := &manifestTrie{ dpa: self.api.dpa, } + quitC := make(chan bool) for i, entry := range list { if errors[i] != nil { return "", errors[i] @@ -139,9 +140,9 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) { Hash: entry.Hash, ContentType: entry.ContentType, } - trie.addEntry(ientry) + trie.addEntry(ientry, quitC) } - trie.addEntry(entry) + trie.addEntry(entry, quitC) } err2 := trie.recalcAndStore() @@ -149,6 +150,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) { if err2 == nil { hs = trie.hash.String() } + wg.Wait() return hs, err2 } @@ -169,11 +171,13 @@ func (self *FileSystem) Download(bzzpath, localpath string) error { if err != nil { return err } - // if len(path) > 0 { - // path += "/" - // } - trie, err := loadManifest(self.api.dpa, key) + if len(path) > 0 { + path += "/" + } + + quitC := make(chan bool) + trie, err := loadManifest(self.api.dpa, key, quitC) if err != nil { glog.V(logger.Warn).Infof("[BZZ] fs.Download: loadManifestTrie error: %v", err) return err @@ -185,72 +189,76 @@ func (self *FileSystem) Download(bzzpath, localpath string) error { } var list []*downloadListEntry - var mde, mderr error + var mde error prevPath := lpath - err = trie.listWithPrefix(path, func(entry *manifestTrieEntry, suffix string) { // TODO: paralellize + err = trie.listWithPrefix(path, quitC, func(entry *manifestTrieEntry, suffix string) { glog.V(logger.Detail).Infof("[BZZ] fs.Download: %#v", entry) - key := common.Hex2Bytes(entry.Hash) + key = common.Hex2Bytes(entry.Hash) path := lpath + "/" + suffix dir := filepath.Dir(path) if dir != prevPath { mde = os.MkdirAll(dir, os.ModePerm) - if mde != nil { - mderr = mde - } prevPath = dir } if (mde == nil) && (path != dir+"/") { list = append(list, &downloadListEntry{key: key, path: path}) } }) - if err == nil { - err = mderr - } - - cnt := len(list) - errors := make([]error, cnt) - done := make(chan bool, maxParallelFiles) - dcnt := 0 - - for i, entry := range list { - if i >= dcnt+maxParallelFiles { - <-done - dcnt++ - } - go func(i int, entry *downloadListEntry, done chan bool) { - f, err := os.Create(entry.path) // TODO: path separators - if err == nil { - reader := self.api.dpa.Retrieve(entry.key) - writer := bufio.NewWriter(f) - _, err = io.CopyN(writer, reader, reader.Size()) // TODO: handle errors - err2 := writer.Flush() - if err == nil { - err = err2 - } - err2 = f.Close() - if err == nil { - err = err2 - } - } - - errors[i] = err - done <- true - }(i, entry, done) - } - for dcnt < cnt { - <-done - dcnt++ - } - if err != nil { return err } - for i, _ := range list { - if errors[i] != nil { - return errors[i] + + wg := sync.WaitGroup{} + errC := make(chan error) + done := make(chan bool, maxParallelFiles) + for i, entry := range list { + select { + case done <- true: + wg.Add(1) + case <-quitC: + return fmt.Errorf("aborted") } + go func(i int, entry *downloadListEntry) { + defer wg.Done() + f, err := os.Create(entry.path) // TODO: path separators + if err == nil { + + reader := self.api.dpa.Retrieve(entry.key) + writer := bufio.NewWriter(f) + size, err := reader.Size(quitC) + if err == nil { + _, err = io.CopyN(writer, reader, size) // TODO: handle errors + err2 := writer.Flush() + if err == nil { + err = err2 + } + err2 = f.Close() + if err == nil { + err = err2 + } + } + } + if err != nil { + select { + case errC <- err: + case <-quitC: + } + return + } + <-done + }(i, entry) } - return err + go func() { + wg.Wait() + close(errC) + }() + select { + case err = <-errC: + return err + case <-quitC: + return fmt.Errorf("aborted") + } + } diff --git a/swarm/api/filesystem_test.go b/swarm/api/filesystem_test.go index c6be66e68f..f26c5df0ab 100644 --- a/swarm/api/filesystem_test.go +++ b/swarm/api/filesystem_test.go @@ -1,16 +1,14 @@ package api import ( - "io" + "bytes" "io/ioutil" "os" "path" "runtime" + "sync" "testing" - -"github.com/ethereum/go-ethereum/logger" - "github.com/ethereum/go-ethereum/logger/glog" - ) +) var ( testDir string @@ -21,8 +19,8 @@ func init() { _, filename, _, _ := runtime.Caller(1) testDir = path.Join(path.Dir(filename), "../test") testDownloadDir, _ = ioutil.TempDir(os.TempDir(), "bzz-test") - glog.SetV(logger.Detail) - glog.SetV = nil +} + func testFileSystem(t *testing.T, f func(*FileSystem)) { testApi(t, func(api *Api) { f(NewFileSystem(api)) @@ -30,7 +28,6 @@ func testFileSystem(t *testing.T, f func(*FileSystem)) { } func readPath(t *testing.T, parts ...string) string { - // func readPath(t *testing.T, parts ...string) []byte { file := path.Join(parts...) content, err := ioutil.ReadFile(file) @@ -41,39 +38,28 @@ func readPath(t *testing.T, parts ...string) string { } func TestApiDirUpload0(t *testing.T) { - // t.Skip("FIXME") testFileSystem(t, func(fs *FileSystem) { api := fs.api bzzhash, err := fs.Upload(path.Join(testDir, "test0"), "") if err != nil { t.Fatalf("unexpected error: %v", err) } - content := readPath(t, testDir, "test0", "index.html") - t.Logf("content (%v): %v ", len(content), content) resp := testGet(t, api, bzzhash+"/index.html") - exp := expRes ponse(content, "text/html; charset=utf-8", 0) - t.Logf("index.html (size%v=?=%v)", resp.Size, exp.Size) + exp := expResponse(content, "text/html; charset=utf-8", 0) checkResponse(t, resp, exp) - t.FailNow() content = readPath(t, testDir, "test0", "index.css") resp = testGet(t, api, bzzhash+"/index.css") exp = expResponse(content, "text/css", 0) - t.Logf("index.css (size%v=?=%v)", resp.Size, exp.Size) checkResponse(t, resp, exp) - content = readPath(t, testDir, "test0", "img", "logo.png") - resp = testGet(t, api, bzzhash+"/img/logo.png") - exp = expResponse(content, "image/png", 0) - _, _, _, err = api.Get(bzzhash, true) if err == nil { t.Fatalf("expected error: %v", err) } downloadDir := path.Join(testDownloadDir, "test0") - os.RemoveAll(downloadDir) defer os.RemoveAll(downloadDir) err = fs.Download(bzzhash, downloadDir) if err != nil { @@ -86,12 +72,10 @@ func TestApiDirUpload0(t *testing.T) { if bzzhash != newbzzhash { t.Fatalf("download %v reuploaded has incorrect hash, expected %v, got %v", downloadDir, bzzhash, newbzzhash) } - }) } func TestApiDirUploadModify(t *testing.T) { - // t.Skip("FIXME") testFileSystem(t, func(fs *FileSystem) { api := fs.api bzzhash, err := fs.Upload(path.Join(testDir, "test0"), "") @@ -105,12 +89,24 @@ func TestApiDirUploadModify(t *testing.T) { t.Errorf("unexpected error: %v", err) return } - bzzhash, err = api.Modify(bzzhash+"/index2.html", "9ea1f60ebd80786d6005f6b256376bdb494a82496cd86fe8c307cdfb23c99e71", "text/html; charset=utf-8", true) + index, err := ioutil.ReadFile(path.Join(testDir, "test0", "index.html")) if err != nil { t.Errorf("unexpected error: %v", err) return } - bzzhash, err = api.Modify(bzzhash+"/img/logo.png", "9ea1f60ebd80786d6005f6b256376bdb494a82496cd86fe8c307cdfb23c99e71", "text/html; charset=utf-8", true) + wg := &sync.WaitGroup{} + hash, err := api.Store(bytes.NewReader(index), int64(len(index)), wg) + wg.Wait() + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + bzzhash, err = api.Modify(bzzhash+"/index2.html", hash.Hex(), "text/html; charset=utf-8", true) + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + bzzhash, err = api.Modify(bzzhash+"/img/logo.png", hash.Hex(), "text/html; charset=utf-8", true) if err != nil { t.Errorf("unexpected error: %v", err) return @@ -172,7 +168,7 @@ func TestApiFileUploadWithRootFile(t *testing.T) { testFileSystem(t, func(fs *FileSystem) { api := fs.api bzzhash, err := fs.Upload(path.Join(testDir, "test0", "index.html"), "index.html") - if err != io.EOF { + if err != nil { t.Errorf("unexpected error: %v", err) return } diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 29945a9f77..f5810c5c0a 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -164,7 +164,9 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { // retrieving content reader := a.Retrieve(key) - glog.V(logger.Debug).Infof("[BZZ] Swarm: Reading %d bytes.", reader.Size()) + quitC := make(chan bool) + size, err := reader.Size(quitC) + glog.V(logger.Debug).Infof("[BZZ] Swarm: Reading %d bytes.", size) // setting mime type qv := requestURL.Query() @@ -175,7 +177,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { w.Header().Set("Content-Type", mimeType) http.ServeContent(w, r, uri, forever(), reader) - glog.V(logger.Debug).Infof("[BZZ] Swarm: Serve raw content '%s' (%d bytes) as '%s'", uri, reader.Size(), mimeType) + glog.V(logger.Debug).Infof("[BZZ] Swarm: Serve raw content '%s' (%d bytes) as '%s'", uri, size, mimeType) // retrieve path via manifest } else { @@ -202,7 +204,9 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { } else { status = 200 } - glog.V(logger.Debug).Infof("[BZZ] Swarm: Served '%s' (%d bytes) as '%s' (status code: %v)", uri, reader.Size(), mimeType, status) + quitC := make(chan bool) + size, err := reader.Size(quitC) + glog.V(logger.Debug).Infof("[BZZ] Swarm: Served '%s' (%d bytes) as '%s' (status code: %v)", uri, size, mimeType, status) http.ServeContent(w, r, path, forever(), reader) diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index ab74a35bb0..a78d86f485 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -34,24 +34,24 @@ type manifestTrieEntry struct { subtrie *manifestTrie } -func loadManifest(dpa *storage.DPA, hash storage.Key) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand +func loadManifest(dpa *storage.DPA, hash storage.Key, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand glog.V(logger.Detail).Infof("[BZZ] manifest lookup key: '%v'.", hash.Log()) // retrieve manifest via DPA manifestReader := dpa.Retrieve(hash) - return readManifest(manifestReader, hash, dpa) + return readManifest(manifestReader, hash, dpa, quitC) } -func readManifest(manifestReader storage.SectionReader, hash storage.Key, dpa *storage.DPA) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand +func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dpa *storage.DPA, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand // TODO check size for oversized manifests - manifestData := make([]byte, manifestReader.Size()) - var size int - size, err = manifestReader.Read(manifestData) - if int64(size) < manifestReader.Size() { + size, err := manifestReader.Size(quitC) + manifestData := make([]byte, size) + read, err := manifestReader.Read(manifestData) + if int64(read) < size { glog.V(logger.Detail).Infof("[BZZ] Manifest %v not found.", hash.Log()) if err == nil { - err = fmt.Errorf("Manifest retrieval cut short: read %v, expect %v", size, manifestReader.Size()) + err = fmt.Errorf("Manifest retrieval cut short: read %v, expect %v", read, size) } return } @@ -71,12 +71,12 @@ func readManifest(manifestReader storage.SectionReader, hash storage.Key, dpa *s dpa: dpa, } for _, entry := range man.Entries { - trie.addEntry(entry) + trie.addEntry(entry, quitC) } return } -func (self *manifestTrie) addEntry(entry *manifestTrieEntry) { +func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) { self.hash = nil // trie modified, hash needs to be re-calculated on demand if len(entry.Path) == 0 { @@ -97,11 +97,11 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry) { } if (oldentry.ContentType == manifestType) && (cpl == len(oldentry.Path)) { - if self.loadSubTrie(oldentry) != nil { + if self.loadSubTrie(oldentry, quitC) != nil { return } entry.Path = entry.Path[cpl:] - oldentry.subtrie.addEntry(entry) + oldentry.subtrie.addEntry(entry, quitC) oldentry.Hash = "" return } @@ -113,8 +113,8 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry) { } entry.Path = entry.Path[cpl:] oldentry.Path = oldentry.Path[cpl:] - subtrie.addEntry(entry) - subtrie.addEntry(oldentry) + subtrie.addEntry(entry, quitC) + subtrie.addEntry(oldentry, quitC) self.entries[b] = &manifestTrieEntry{ Path: commonPrefix, @@ -134,7 +134,7 @@ func (self *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) { return } -func (self *manifestTrie) deleteEntry(path string) { +func (self *manifestTrie) deleteEntry(path string, quitC chan bool) { self.hash = nil // trie modified, hash needs to be re-calculated on demand if len(path) == 0 { @@ -154,10 +154,10 @@ func (self *manifestTrie) deleteEntry(path string) { epl := len(entry.Path) if (entry.ContentType == manifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) { - if self.loadSubTrie(entry) != nil { + if self.loadSubTrie(entry, quitC) != nil { return } - entry.subtrie.deleteEntry(path[epl:]) + entry.subtrie.deleteEntry(path[epl:], quitC) entry.Hash = "" // remove subtree if it has less than 2 elements cnt, lastentry := entry.subtrie.getCountLast() @@ -205,16 +205,16 @@ func (self *manifestTrie) recalcAndStore() error { return err2 } -func (self *manifestTrie) loadSubTrie(entry *manifestTrieEntry) (err error) { +func (self *manifestTrie) loadSubTrie(entry *manifestTrieEntry, quitC chan bool) (err error) { if entry.subtrie == nil { hash := common.Hex2Bytes(entry.Hash) - entry.subtrie, err = loadManifest(self.dpa, hash) + entry.subtrie, err = loadManifest(self.dpa, hash, quitC) entry.Hash = "" // might not match, should be recalculated } return } -func (self *manifestTrie) listWithPrefixInt(prefix, rp string, cb func(entry *manifestTrieEntry, suffix string)) (err error) { +func (self *manifestTrie) listWithPrefixInt(prefix, rp string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) error { plen := len(prefix) var start, stop int if plen == 0 { @@ -226,6 +226,11 @@ func (self *manifestTrie) listWithPrefixInt(prefix, rp string, cb func(entry *ma } for i := start; i <= stop; i++ { + select { + case <-quitC: + return fmt.Errorf("aborted") + default: + } entry := self.entries[i] if entry != nil { epl := len(entry.Path) @@ -235,11 +240,13 @@ func (self *manifestTrie) listWithPrefixInt(prefix, rp string, cb func(entry *ma l = epl } if prefix[:l] == entry.Path[:l] { - sterr := self.loadSubTrie(entry) - if sterr == nil { - entry.subtrie.listWithPrefixInt(prefix[l:], rp+entry.Path[l:], cb) - } else { - err = sterr + err := self.loadSubTrie(entry, quitC) + if err != nil { + return err + } + err = entry.subtrie.listWithPrefixInt(prefix[l:], rp+entry.Path[l:], quitC, cb) + if err != nil { + return err } } } else { @@ -249,14 +256,14 @@ func (self *manifestTrie) listWithPrefixInt(prefix, rp string, cb func(entry *ma } } } - return + return nil } -func (self *manifestTrie) listWithPrefix(prefix string, cb func(entry *manifestTrieEntry, suffix string)) (err error) { - return self.listWithPrefixInt(prefix, "", cb) +func (self *manifestTrie) listWithPrefix(prefix string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) (err error) { + return self.listWithPrefixInt(prefix, "", quitC, cb) } -func (self *manifestTrie) findPrefixOf(path string) (entry *manifestTrieEntry, pos int) { +func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *manifestTrieEntry, pos int) { glog.V(logger.Detail).Infof("[BZZ] findPrefixOf(%s)", path) @@ -274,10 +281,10 @@ func (self *manifestTrie) findPrefixOf(path string) (entry *manifestTrieEntry, p if (len(path) >= epl) && (path[:epl] == entry.Path) { glog.V(logger.Detail).Infof("[BZZ] entry.ContentType = %v", entry.ContentType) if entry.ContentType == manifestType { - if self.loadSubTrie(entry) != nil { + if self.loadSubTrie(entry, quitC) != nil { return nil, 0 } - entry, pos = entry.subtrie.findPrefixOf(path[epl:]) + entry, pos = entry.subtrie.findPrefixOf(path[epl:], quitC) if entry != nil { pos += epl } @@ -307,6 +314,7 @@ func RegularSlashes(path string) (res string) { func (self *manifestTrie) getEntry(spath string) (entry *manifestTrieEntry, fullpath string) { path := RegularSlashes(spath) var pos int - entry, pos = self.findPrefixOf(path) + quitC := make(chan bool) + entry, pos = self.findPrefixOf(path, quitC) return entry, path[:pos] } diff --git a/swarm/api/manifest_test.go b/swarm/api/manifest_test.go index 558bdfb51b..3143e5eaaf 100644 --- a/swarm/api/manifest_test.go +++ b/swarm/api/manifest_test.go @@ -10,18 +10,19 @@ import ( "github.com/ethereum/go-ethereum/swarm/storage" ) -func manifest(paths ...string) (manifestReader storage.SectionReader) { +func manifest(paths ...string) (manifestReader storage.LazySectionReader) { var entries []string for _, path := range paths { entry := fmt.Sprintf(`{"path":"%s"}`, path) entries = append(entries, entry) } manifest := fmt.Sprintf(`{"entries":[%s]}`, strings.Join(entries, ",")) - return io.NewSectionReader(strings.NewReader(manifest), 0, int64(len(manifest))) + return &storage.LazyTestSectionReader{io.NewSectionReader(strings.NewReader(manifest), 0, int64(len(manifest)))} } func testGetEntry(t *testing.T, path, match string, paths ...string) *manifestTrie { - trie, err := readManifest(manifest(paths...), nil, nil) + quitC := make(chan bool) + trie, err := readManifest(manifest(paths...), nil, nil, quitC) if err != nil { t.Errorf("unexpected error making manifest: %v", err) } diff --git a/swarm/api/storage.go b/swarm/api/storage.go index 1fd3fa6008..3dfc37f1c2 100644 --- a/swarm/api/storage.go +++ b/swarm/api/storage.go @@ -34,7 +34,11 @@ func (self *Storage) Get(bzzpath string) (*Response, error) { if err != nil { return nil, err } - expsize := reader.Size() + quitC := make(chan bool) + expsize, err := reader.Size(quitC) + if err != nil { + return nil, err + } body := make([]byte, expsize) size, err := reader.Read(body) if int64(size) == expsize { diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index d58eed6381..490f5b432e 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -126,10 +126,10 @@ func (self *TreeChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, s } key := make([]byte, self.hashFunc().Size()) - glog.V(logger.Detail).Infof("[BZZ] split request received for data (%v bytes, depth: %v)", size, depth) + // glog.V(logger.Detail).Infof("[BZZ] split request received for data (%v bytes, depth: %v)", size, depth) // this waitgroup member is released after the root hash is calculated wg.Add(1) - //launch actual recursive function passing the workgroup + //launch actual recursive function passing the waitgroups go self.split(depth, treeSize/self.branches, key, data, size, jobC, chunkC, errC, wg, swg, wwg) // closes internal error channel if all subprocesses in the workgroup finished @@ -266,108 +266,117 @@ func (self *TreeChunker) hashChunk(hasher hash.Hash, job *hashJob, chunkC chan * } } +// LazyChunkReader implements LazySectionReader +type LazyChunkReader struct { + key Key // root key + chunkC chan *Chunk // chunk channel to send retrieve requests on + chunk *Chunk // size of the entire subtree + off int64 // offset + chunkSize int64 // inherit from chunker + branches int64 // inherit from chunker + hashSize int64 // inherit from chunker +} + // implements the Joiner interface -func (self *TreeChunker) Join(key Key, chunkC chan *Chunk) SectionReader { +func (self *TreeChunker) Join(key Key, chunkC chan *Chunk) LazySectionReader { return &LazyChunkReader{ - key: key, - chunkC: chunkC, - quitC: make(chan bool), - errC: make(chan error), - chunker: self, + key: key, + chunkC: chunkC, + chunkSize: self.chunkSize, + branches: self.branches, + hashSize: self.hashSize, } } -// LazyChunkReader implements Lazy.SectionReader -type LazyChunkReader struct { - key Key // root key - chunkC chan *Chunk // chunk channel to send retrieve requests on - size int64 // size of the entire subtree - off int64 // offset - quitC chan bool // channel to abort retrieval - errC chan error // error channel to monitor retrieve errors - chunker *TreeChunker // needs TreeChunker params TODO: should just take - // the chunkSize, branches etc as params +// Size is meant to be called on the LazySectionReader +func (self *LazyChunkReader) Size(quitC chan bool) (n int64, err error) { + if self.chunk != nil { + return self.chunk.Size, nil + } + chunk := retrieve(self.key, self.chunkC, quitC) + if chunk == nil { + select { + case <-quitC: + return 0, errors.New("aborted") + default: + return 0, fmt.Errorf("root chunk not found for %v", self.key.Hex()) + } + } + self.chunk = chunk + return chunk.Size, nil } +// read at can be called numerous times +// concurrent reads are allowed +// Size() needs to be called synchronously on the LazyChunkReader first func (self *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) { - self.errC = make(chan error) - chunk := &Chunk{ - Key: self.key, - C: make(chan bool), // close channel to signal data delivery - } - self.chunkC <- chunk // submit retrieval request, someone should be listening on the other side (or we will time out globally) - // glog.V(logger.Detail).Infof("[BZZ] readAt: reading %v into %d bytes at offset %d.", chunk.Key.Log(), len(b), off) - - // waiting for the chunk retrieval - select { - case <-self.quitC: - // this is how we control process leakage (quitC is closed once join is finished (after timeout)) - // glog.V(logger.Detail).Infof("[BZZ] quit") - return - case <-chunk.C: // bells are ringing, data have been delivered - // glog.V(logger.Detail).Infof("[BZZ] chunk data received for %v", chunk.Key.Log()) - } - if len(chunk.SData) == 0 { - // glog.V(logger.Detail).Infof("[BZZ] No payload in %v", chunk.Key.Log()) - return 0, notFound - } - chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) - self.size = chunk.Size - if b == nil { + // this is correct, a swarm doc cannot be zero length, so no EOF is expected + if len(b) == 0 { // glog.V(logger.Detail).Infof("[BZZ] Size query for %v", chunk.Key.Log()) - return + return 0, nil } - want := int64(len(b)) - if off+want > self.size { - want = self.size - off + quitC := make(chan bool) + size, err := self.Size(quitC) + if err != nil { + return 0, err } + glog.V(logger.Detail).Infof("readAt: len(b): %v, off: %v, size: %v ", len(b), off, size) + + errC := make(chan error) + // glog.V(logger.Detail).Infof("[BZZ] readAt: reading %v into %d bytes at offset %d.", self.chunk.Key.Log(), len(b), off) + + // } + // glog.V(logger.Detail).Infof("-> want: %v, off: %v size: %v ", want, off, self.size) var treeSize int64 var depth int // calculate depth and max treeSize - treeSize = self.chunker.chunkSize - for ; treeSize < chunk.Size; treeSize *= self.chunker.branches { + treeSize = self.chunkSize + for ; treeSize < size; treeSize *= self.branches { depth++ } wg := sync.WaitGroup{} wg.Add(1) - go self.join(b, off, off+want, depth, treeSize/self.chunker.branches, chunk, &wg) + go self.join(b, off, off+int64(len(b)), depth, treeSize/self.branches, self.chunk, &wg, errC, quitC) go func() { wg.Wait() - close(self.errC) + close(errC) }() - err = <-self.errC + err = <-errC + if err != nil { + close(quitC) + + return 0, err + } // glog.V(logger.Detail).Infof("[BZZ] ReadAt received %v", err) - read = len(b) - if off+int64(read) == self.size { - err = io.EOF + glog.V(logger.Detail).Infof("end: len(b): %v, off: %v, size: %v ", len(b), off, size) + if off+int64(len(b)) >= size { + glog.V(logger.Detail).Infof(" len(b): %v EOF", len(b)) + return len(b), io.EOF } // glog.V(logger.Detail).Infof("[BZZ] ReadAt returning at %d: %v", read, err) - return + return len(b), nil } -func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeSize int64, chunk *Chunk, parentWg *sync.WaitGroup) { +func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeSize int64, chunk *Chunk, parentWg *sync.WaitGroup, errC chan error, quitC chan bool) { defer parentWg.Done() + // return NewDPA(&LocalStore{}) + glog.V(logger.Detail).Infof("inh len(b): %v, off: %v eoff: %v ", len(b), off, eoff) // glog.V(logger.Detail).Infof("[BZZ] depth: %v, loff: %v, eoff: %v, chunk.Size: %v, treeSize: %v", depth, off, eoff, chunk.Size, treeSize) - chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) + // chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) // find appropriate block level for chunk.Size < treeSize && depth > 0 { - treeSize /= self.chunker.branches + treeSize /= self.branches depth-- } + // leaf chunk found if depth == 0 { - // glog.V(logger.Detail).Infof("[BZZ] depth: %v, len(b): %v, off: %v, eoff: %v, chunk.Size: %v, treeSize: %v", depth, len(b), off, eoff, chunk.Size, treeSize) - if int64(len(b)) != eoff-off { - //fmt.Printf("len(b) = %v off = %v eoff = %v", len(b), off, eoff) - msg := fmt.Sprintf("len(b) = %v =?= %v = eoff-off", len(b), eoff-off) - panic(msg) - } - + glog.V(logger.Detail).Infof("[BZZ] depth: %v, len(b): %v, off: %v, eoff: %v, chunk.Size: %v, treeSize: %v", depth, len(b), off, eoff, chunk.Size, treeSize) copy(b, chunk.SData[8+off:8+eoff]) return // simply give back the chunks reader for content chunks } @@ -375,10 +384,12 @@ func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, tr // subtree start := off / treeSize end := (eoff + treeSize - 1) / treeSize + wg := &sync.WaitGroup{} + defer wg.Wait() + glog.V(logger.Detail).Infof("[BZZ] start %v,end %v", start, end) for i := start; i < end; i++ { - soff := i * treeSize roff := soff seoff := soff + treeSize @@ -394,51 +405,65 @@ func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, tr } wg.Add(1) go func(j int64) { - childKey := chunk.SData[8+j*self.chunker.hashSize : 8+(j+1)*self.chunker.hashSize] - // glog.V(logger.Detail).Infof("[BZZ] subtree index: %v -> %v", j, childKey.Log()) - - ch := &Chunk{ - Key: childKey, - C: make(chan bool), // close channel to signal data delivery - } - // glog.V(logger.Detail).Infof("[BZZ] chunk data sent for %v (key interval in chunk %v-%v)", ch.Key.Log(), j*self.chunker.hashSize, (j+1)*self.chunker.hashSize) - self.chunkC <- ch // submit retrieval request, someone should be listening on the other side (or we will time out globally) - - // waiting for the chunk retrieval - select { - case <-self.quitC: - // this is how we control process leakage (quitC is closed once join is finished (after timeout)) + childKey := chunk.SData[8+j*self.hashSize : 8+(j+1)*self.hashSize] + // glog.V(logger.Detail).Infof("[BZZ] subtree ind.ex: %v -> %v", j, childKey.Log()) + chunk := retrieve(childKey, self.chunkC, quitC) + if chunk == nil { + select { + case errC <- fmt.Errorf("chunk %v-%v not found", off, off+treeSize): + case <-quitC: + } return - case <-ch.C: // bells are ringing, data have been delivered - // glog.V(logger.Detail).Infof("[BZZ] chunk data received") } if soff < off { soff = off } - if len(ch.SData) == 0 { - select { - case self.errC <- fmt.Errorf("chunk %v-%v not found", off, off+treeSize): - case <-self.quitC: - } - return - } - self.join(b[soff-off:seoff-off], soff-roff, seoff-roff, depth-1, treeSize/self.chunker.branches, ch, wg) + self.join(b[soff-off:seoff-off], soff-roff, seoff-roff, depth-1, treeSize/self.branches, chunk, wg, errC, quitC) }(i) } //for - wg.Wait() } -func (self *LazyChunkReader) Size() (n int64) { - self.ReadAt(nil, 0) - return self.size +// the helper method submits chunks for a key to a oueue (DPA) and +// block until they time out or arrive +// abort if quitC is readable +func retrieve(key Key, chunkC chan *Chunk, quitC chan bool) *Chunk { + chunk := &Chunk{ + Key: key, + C: make(chan bool), // close channel to signal data delivery + } + // glog.V(logger.Detail).Infof("[BZZ] chunk data sent for %v (key interval in chunk %v-%v)", ch.Key.Log(), j*self.chunker.hashSize, (j+1)*self.chunker.hashSize) + // submit chunk for retrieval + select { + case chunkC <- chunk: // submit retrieval request, someone should be listening on the other side (or we will time out globally) + case <-quitC: + return nil + } + // waiting for the chunk retrieval + select { // chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) + + case <-quitC: + // this is how we control process leakage (quitC is closed once join is finished (after timeout)) + return nil + case <-chunk.C: // bells are ringing, data have been delivered + // glog.V(logger.Detail).Infof("[BZZ] chunk data received") + } + if len(chunk.SData) == 0 { + return nil // chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) + + } + return chunk } +// Read keeps a cursor so cannot be called simulateously, see ReadAt func (self *LazyChunkReader) Read(b []byte) (read int, err error) { read, err = self.ReadAt(b, self.off) + glog.V(logger.Detail).Infof("[BZZ] read: %v, off: %v, error: %v", read, self.off, err) + self.off += int64(read) return } +// completely analogous to standard SectionReader implementation var errWhence = errors.New("Seek: invalid whence") var errOffset = errors.New("Seek: invalid offset") @@ -451,8 +476,12 @@ func (s *LazyChunkReader) Seek(offset int64, whence int) (int64, error) { case 1: offset += s.off case 2: - offset += s.size + if s.chunk == nil { + return 0, fmt.Errorf("seek from the end requires rootchunk for size. call Size first") + } + offset += s.chunk.Size } + if offset < 0 { return 0, errOffset } diff --git a/swarm/storage/chunker_test.go b/swarm/storage/chunker_test.go index fbcf210588..cdf549ff45 100644 --- a/swarm/storage/chunker_test.go +++ b/swarm/storage/chunker_test.go @@ -83,7 +83,7 @@ func (self *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, c return } -func (self *chunkerTester) Join(chunker *TreeChunker, key Key, c int, chunkC chan *Chunk, quitC chan bool) SectionReader { +func (self *chunkerTester) Join(chunker *TreeChunker, key Key, c int, chunkC chan *Chunk, quitC chan bool) LazySectionReader { // reset but not the chunks reader := chunker.Join(key, chunkC) @@ -163,7 +163,7 @@ func TestRandomData(t *testing.T) { testRandomData(253, 7, t) } -func readAll(reader SectionReader, result []byte) { +func readAll(reader LazySectionReader, result []byte) { size := int64(len(result)) var end int64 @@ -177,8 +177,8 @@ func readAll(reader SectionReader, result []byte) { } } -func benchReadAll(reader SectionReader) { - size := reader.Size() +func benchReadAll(reader LazySectionReader) { + size, _ := reader.Size(nil) output := make([]byte, 1000) for pos := int64(0); pos < size; pos += 1000 { reader.ReadAt(output, pos) diff --git a/swarm/storage/common_test.go b/swarm/storage/common_test.go index b3fdc027b7..55fcbfd409 100644 --- a/swarm/storage/common_test.go +++ b/swarm/storage/common_test.go @@ -44,7 +44,6 @@ func testStore(m ChunkStore, l int64, branches int64, t *testing.T) { swg.Wait() close(chunkC) chunkC = make(chan *Chunk) - r := chunker.Join(key, chunkC) quit := make(chan bool) @@ -53,18 +52,20 @@ func testStore(m ChunkStore, l int64, branches int64, t *testing.T) { go func(chunk *Chunk) { storedChunk, err := m.Get(chunk.Key) if err == notFound { - glog.V(logger.Detail).Infof("[BZZ] chunk '%x' not found", chunk.Key) + glog.V(logger.Detail).Infof("[BZZ] chunk '%v' not found", chunk.Key.Log()) } else if err != nil { - glog.V(logger.Detail).Infof("[BZZ] error retrieving chunk %x: %v", chunk.Key, err) + glog.V(logger.Detail).Infof("[BZZ] error retrieving chunk %v: %v", chunk.Key.Log(), err) } else { chunk.SData = storedChunk.SData + chunk.Size = storedChunk.Size } - glog.V(logger.Detail).Infof("[BZZ] chunk '%x' not found", chunk.Key[:4]) + glog.V(logger.Detail).Infof("[BZZ] chunk '%v' not found", chunk.Key.Log()) close(chunk.C) }(ch) } close(quit) }() + r := chunker.Join(key, chunkC) b := make([]byte, l) n, err := r.ReadAt(b, 0) diff --git a/swarm/storage/dpa.go b/swarm/storage/dpa.go index ba8e0e2c56..34a4639a6d 100644 --- a/swarm/storage/dpa.go +++ b/swarm/storage/dpa.go @@ -73,7 +73,7 @@ func NewDPA(store ChunkStore, params *ChunkerParams) *DPA { // FS-aware API and httpaccess // Chunk retrieval blocks on netStore requests with a timeout so reader will // report error if retrieval of chunks within requested range time out. -func (self *DPA) Retrieve(key Key) SectionReader { +func (self *DPA) Retrieve(key Key) LazySectionReader { return self.Chunker.Join(key, self.retrieveC) } @@ -146,7 +146,7 @@ func (self *DPA) storeLoop() { go func(chunk *Chunk) { self.Put(chunk) if chunk.wg != nil { - glog.V(logger.Detail).Infof("[BZZ] DPA.storeLoop %v", chunk.Key.Log()) + glog.V(logger.Detail).Infof("[BZZ] dpa: store loop %v", chunk.Key.Log()) chunk.wg.Done() } }(ch) diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index 81d4a6f2d0..68487368f1 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -1,5 +1,9 @@ package storage +import ( + "encoding/binary" +) + // LocalStore is a combination of inmemory db over a disk persisted db // implements a Get/Put with fallback (caching) logic using any 2 ChunkStores type LocalStore struct { @@ -48,6 +52,7 @@ func (self *LocalStore) Get(key Key) (chunk *Chunk, err error) { if err != nil { return } + chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) self.memStore.Put(chunk) return } diff --git a/swarm/storage/types.go b/swarm/storage/types.go index 0a40fb4614..8d7e7fdd38 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -14,6 +14,8 @@ import ( type Hasher func() hash.Hash +// Peer is the recorded as Source on the chunk +// should probably not be here? but network should wrap chunk object type Peer interface{} type Key []byte @@ -187,7 +189,7 @@ type Joiner interface { The chunks are not meant to be validated by the chunker when joining. This is because it is left to the DPA to decide which sources are trusted. */ - Join(key Key, chunkC chan *Chunk) SectionReader + Join(key Key, chunkC chan *Chunk) LazySectionReader } type Chunker interface { @@ -198,9 +200,17 @@ type Chunker interface { } // Size, Seek, Read, ReadAt -type SectionReader interface { - Size() int64 +type LazySectionReader interface { + Size(chan bool) (int64, error) io.Seeker io.Reader io.ReaderAt } + +type LazyTestSectionReader struct { + *io.SectionReader +} + +func (self *LazyTestSectionReader) Size(chan bool) (int64, error) { + return self.SectionReader.Size(), nil +} From 38028fc8e91bcdffc57ed8d709e9d14a22050623 Mon Sep 17 00:00:00 2001 From: zelig Date: Tue, 28 Jun 2016 22:56:48 +0200 Subject: [PATCH 20/24] swarm/cmd: swarm cntrol cli improvements * no more alias, swarm is executable so it can be called via ssh * environment vars now set to default, no need to preconfigure * stop method falls back to kill -9 after 10s * enode method now supplies ip addr via ipecho request * execute, options, rawoptions, setup, create-account addpeers and hive methods and subcommands * local and remote hive monitoring * gethup.sh script now merged into swarm script and nice modularised * simplify bash code and fix e2e tests in swarm/test * update-src -> update * remote-update-scripts, remote-update-bin and remote-run * local and remote monitoring of kademlia * extensive documentation in swarm/cmd/README.md swarm/network: fix hive stop issue leading to send on closed chan + minor logging fixes swarm/network/kademlia: * simplify code * now really fix prox limit adjustment + tests and comments * simplify and make readable findclosest algo code + comments * reset initial time interval settings for kaddb findbest * fix bucket replace scheme to optimise stability and availability * absolute idle peers are disconnected after maxIdleInterval --- swarm/api/config_test.go | 3 +- swarm/cmd/README.md | 218 ++++++++ swarm/cmd/swarm/env.sh | 11 +- swarm/cmd/swarm/gethup.sh | 138 ----- swarm/cmd/swarm/swarm | 664 +++++++++++++++++++++++- swarm/cmd/swarm/swarm.sh | 432 --------------- swarm/cmd/swarm/test.sh | 28 - swarm/network/hive.go | 19 +- swarm/network/kademlia/kaddb.go | 11 +- swarm/network/kademlia/kademlia.go | 265 +++++----- swarm/network/kademlia/kademlia_test.go | 30 +- swarm/network/protocol.go | 5 +- swarm/network/syncer.go | 6 +- swarm/test/connections/00.sh | 30 +- swarm/test/swap/00.sh | 2 +- swarm/test/swap/01.sh | 13 +- swarm/test/syncing/00.sh | 6 +- swarm/test/syncing/01.sh | 2 +- swarm/test/syncing/02.sh | 28 +- swarm/test/test.sh | 20 + 20 files changed, 1100 insertions(+), 831 deletions(-) create mode 100644 swarm/cmd/README.md delete mode 100644 swarm/cmd/swarm/gethup.sh delete mode 100644 swarm/cmd/swarm/swarm.sh delete mode 100644 swarm/cmd/swarm/test.sh create mode 100644 swarm/test/test.sh diff --git a/swarm/api/config_test.go b/swarm/api/config_test.go index c394383e5a..96f112f4bd 100644 --- a/swarm/api/config_test.go +++ b/swarm/api/config_test.go @@ -26,7 +26,8 @@ var ( "ProxBinSize": 4, "BucketSize": 3, "PurgeInterval": 151200000000000, - "InitialRetryInterval": 4200000000, + "InitialRetryInterval": 42000000, + "MaxIdleInterval": 4200000000, "ConnRetryExp": 2, "Swap": { "BuyAt": 20000000000, diff --git a/swarm/cmd/README.md b/swarm/cmd/README.md new file mode 100644 index 0000000000..a28ea84a7b --- /dev/null +++ b/swarm/cmd/README.md @@ -0,0 +1,218 @@ + +# install and setup swarm + +swarm is developed on a branch of the ethereum/go-ethereum repo +at this stage of the project there is no packages or binary distro, you need to a dev environment and compile from source. +[This document spells out a complete server setup on ubuntu linux](https://gist.github.com/zelig/74eb365752ceaacf15e860fb80eacb3e) including git/ssh/screen config, golang and compilation, node/npm and network monitoring (might contain a few bits that are tangential to swarm). + +Assuming you got your setup working, you will use the `swarm` command line tool to control your swarm of instances. +This command line tool is at the moment geared towards developers and testing. +It is likely that it will be replaced by two different tools, one for devel/testing and one for end users + + +The command can be used to update the code + +```shell +swarm update upstream/swarm +``` + +Then compile with + +```shell +godep go build -v ./cmd/geth +``` + +Make sure you have `GOPATH` variable set and also that the `swarm` executable is in your PATH. +These environment variables are relevant and set to the following defaults. +Make sure you are happy with them, otherwise change them, in which case best to put these lines in your `~/.profile`. + +``` +export GETH_DIR=$GOPATH/src/github.com/ethereum/go-ethereum +export GETH=$GETH_DIR/geth +export SWARM_DIR=~/bzz +export SWARM_NETWORK_ID=322 +``` + +* `GETH_DIR` points to your git working copy (given `GOPATH` its standardly under `$GOPATH/src/github.com/ethereum/go-ethereum`) +* `GETH` points to the `geth` executable compiled from the swarm branch. If you have systemwide install or use multiple geths you may need to change this, otherwise it is assumed you compile to the working copy of the repo. +* `SWARM_DIR` is the root directory for all swarm related stuff: logs, configs, as well as geth datadirs, make sure this dir is on a device with sufficient disk space +* `SWARM_NETWORK_ID`: this is by default the network id of the swarm testnet. If you run your own swarm, you need to change it, choose a number that is not likely chosen by others to avoid others joining you. + +# Deploying and remote control + +the swarm command supports remote update and remote control of your instances. +In our setting we assume you want to run a cluster of potentially remote swarm nodes each running a local cluster of instances +The only assumption is that you have (passwordless) ssh access set up to your swarm servers. +Assume `nodes.lst` is a list of nodes in the format of `username@ip` one per line. blank lines and lines commented out with `#` are ignored. + + +This copies the scripts found in `swarm/cmd/swarm` on all remote nodes listed in `nodes.lst` + +``` +swarm remote-update-scripts nodes.lst +``` + +If you just want to deploy a locally compiled binaries to all your remote nodes, this will fail if the remote instances are running, so make sure you stop them beforehand + +```shell +swarm remote-run nodes.lst swarm stop all +swarm remote-update-bin nodes.lst +``` + + + +Once you deployed the executables to the nodes, you can control them all with one command. For instance the following line initialises a cluster of two test swarm instances on each remote node. +Watch out, this will wipe your storage and all swarm related data + +```shell +swarm remote-run nodes.lst swarm init 2 +``` + + +To (re) start a particular instance on a specific remote node with alternative options (for instance mining and different logging verbosity), you can just: + +```shell +swarm remote-run cicada@3.3.0.1 'swarm restart 01 --mine --verbosity=0 --vmodule=swarm/*=5' +``` + +# Logging + +To check logs + +```shell +swarm log 00 # taillog flow +swarm remote-run cicada@3.3.0.1 swarm log 00 +``` + +You can view the log with a pager for an instance with + +``` +swarm viewlog 00 +``` + +Logs are preserved and viewable with the above commands even when nodes are offline +Each new run logs to a different file + +To purge logs + +``` +swarm cleanlog 01 +``` + +To remove all logs on all nodes: + +``` +swarm remote-run nodes.lst swarm cleanlog all +``` + +# upload and dowload + +upload and download via a running local instance + +```shell +swarm up 00 /path/to/file/or/directory +swarm down 01 hash /path/to/destination +``` + +upload via remote swarm proxy or public gateway + +```shell +swarm remote-up gateway-url /path/to/file/or/directory +wget -O- gateway-url/bzz:/swarm-url +``` + + +# Further examples + +```shell + +# display CLI options given to geth used to launch swarm instance 02 +swarm options 02 + +# restart swarm instance 00 with alternatiev options +swarm restart 00 --mine --bzznosync --verbosity=0 --vmodule=swarm/*=6 + +# attach console to a running swarm instance +swarm attach 00 + +# execute a command; e.g., start mining on a running instance +swarm execute 00 'miner.stop(1)' + +# display static info about a instance (even if its offline) +swarm info 00 + +# displays the enode url of a running instance +swarm enode 01 + +# add peers to a running swarm instance +swarm addpeers 00 "enode://1033c1cada...@3.3.0.1:30301" + +# to compile a list of enodes from all instances on all remote nodes: +swarm remote-run nodes.lst 'swarm enode all' > enodes.lst + +# to add all peers to all instances on each node +for node in `cat nodes.lst|grep -v '^#'`; do scp enodes.lst $node:; done +swarm remote-run nodes.lst 'swarm addpeers enodes.lst' + +# stop all running instances on the node +swarm stop all + +# stop all running instances on all remote nodes +swarm remote-run nodes.lst swarm stop all + +# display peer connection table of running instance 00 +swarm hive 00 + +# display peer connection table for a running instance and continually refresh every 4 seconds +swarm monitor 00 4 + +# display peer connection table for all running instance on a remote node and continually refresh every 10 seconds +swarm monitor cicada@3.3.0.1 all 10 + + +# configure eth-net-intelligence-api network monitoring client API for a node (the name argument appears as a prefix for all instances in your cluster) +swarm netstatconf cicada-sworm + +# restart the net monitor client API +swarm netstatun + +# configure eth-net-intelligence-api network monitoring client API and (re)start the monitor tool on all remote nodes +swarm remote-run nodes.lst 'swarm netstatconf cicada-sworm; swarm netstatrun' + + +swarm remote-run-all nodes.lst 'swarm netstatconf cicada-sworm; swarm netstatrun' +``` + + +see also: + +* https://github.com/ethereum/go-ethereum/tree/swarm/swarm/test +* https://github.com/ethereum/go-ethereum/tree/swarm/swarm/cmd + +# ethereum netstats client setup + +## install + +nodejs and npm are prerequisites + +```shell +# MAC +brew install node npm +# ubuntu +sudo apt-get install npm nodejs +``` + +clone the git repo and install: + +``` +git clone git@github.com:cubedro/eth-net-intelligence-api.git +cd eth-net-intelligence-api +npm install +npm install -g pm2 +``` + +## configure and run netstats client for each node + +```shell +swarm remote-run-all nodes.lst 'swarm netstatconf cicada-sworm; swarm netstatrun' +``` diff --git a/swarm/cmd/swarm/env.sh b/swarm/cmd/swarm/env.sh index 634f316fb7..8ceafcafc7 100644 --- a/swarm/cmd/swarm/env.sh +++ b/swarm/cmd/swarm/env.sh @@ -1,9 +1,6 @@ -export GOPATH=~/go -export PATH=~/bin:$GOPATH/bin:$PATH +export PATH=$HOME/bin:$PATH +export GETH_DIR=$HOME/bin +export SWARM_DIR= -if [ -f ~/.bash_aliases ]; then - . ~/.bash_aliases -fi - -export NVM_DIR="/home/ubuntu/.nvm" +export NVM_DIR=$HOME/.nvm [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" # This loads nvm \ No newline at end of file diff --git a/swarm/cmd/swarm/gethup.sh b/swarm/cmd/swarm/gethup.sh deleted file mode 100644 index 8d8925cd62..0000000000 --- a/swarm/cmd/swarm/gethup.sh +++ /dev/null @@ -1,138 +0,0 @@ -#!/bin/bash -# Usage: -# bash /path/to/eth-utils/gethup.sh - -root=$1 # base directory to use for datadir and logs -shift -id=$1 # double digit instance id like 00 01 02 -shift -ip_addr=$1 # ip address to substitute -shift - -# logs are output to a date-tagged file for each run , while a link is -# created to the latest, so that monitoring be easier with the same filename -# TODO: use this if GETH not set -# GETH=geth -# echo "ls -l $GETH" -# ls -l $GETH - -# geth CLI params e.g., (dd=04, run=09) -datetag=`date "+%Y-%m-%d-%H:%M:%S"` -datadir=$root/data/$id # /tmp/eth/04 -log=$root/log/$id.$datetag.log # /tmp/eth/04.09.log -linklog=$root/log/$id.log # /tmp/eth/04.09.log -password=$id # 04 -port=303$id # 34504 -bzzport=322$id # 3 2204 -rpcport=302$id # 3204 - -mkdir -p $root/data -mkdir -p $root/enodes -mkdir -p $root/pids -mkdir -p $root/log -# if we do not have an account, create one -# will not prompt for password, we use the double digit instance id as passwd -# NEVER EVER USE THESE ACCOUNTS FOR INTERACTING WITH A LIVE CHAIN -keystoredir="$datadir/keystore/" -# echo "KeyStore dir: $keystoredir" -if [ ! -d "$keystoredir" ]; then - # echo "create an account with password $id [DO NOT EVER USE THIS ON LIVE]" - # mkdir -p $datadir/keystore - $GETH --datadir $datadir --password <(echo -n $id) account new >/dev/null 2>&1 - # create account with password 00, 01, ... - # note that the account key will be stored also separately outside - # datadir - # this way you can safely clear the data directory and still keep your key - # under /keystore/dd - # LS=$(ls $datadir/keystore) - # echo $LS - while [ ! -d "$keystoredir" ]; do - echo "." - ((i++)) - if ((i>10)); then break; fi - sleep 1 - done - # echo "copying keys $datadir/keystore $root/keystore/$id" - mkdir -p $root/keystore/$id - cp -R "$datadir/keystore/" $root/keystore/$id -fi - -# query node's enode url -if [ $ip_addr="" ]; then - pattern='\d+\.\d+\.\d+\.\d+' - ip_addr="[::]" -else - pattern='\[\:\:\]' -fi - -geth="$GETH --datadir $datadir --port $port" - -# echo -n "enode for instance $id... " -if [ ! "$GETH" = "" ] && [ ! -f $root/enodes/$id.enode ]; then - cmd="$geth js <(echo 'console.log(admin.nodeInfo.enode); exit();') " - # echo $cmd '2>/dev/null |grep enode | perl -pe "s/'$pattern'/'$ip_addr'/g" | perl -pe "s/^/\"/; s/\s*$/\"/;" > '$root/enodes/$id.enode - eval $cmd 2>/dev/null |grep enode | perl -pe "s/$pattern/$ip_addr/g" | perl -pe "s/^/\"/; s/\s*\$/\"/;" > $root/enodes/$id.enode -fi -# cat $root/enodes/$id.enode -echo - -# copy cluster enodes list to node's static node list -# echo "copy cluster enodes list to node's static node list" -if [ -f $root/enodes.all ]; then - cp $root/enodes.all $datadir/static-nodes.json -fi - -if [ ! -f $root/pids/$id.pid ]; then - # bring up node `dd` (double digit) - # - using /
- # - listening on port 303dd, (like 30300, 30301, ...) - # - with the account unlocked - # - launching json-rpc server on port 81dd (like 8100, 8101, 8102, ...) - # echo "BZZKEY=$geth account list|head -n1|perl -ne '/([a-f0-9]{40})/ && print \$1'" - BZZKEY=`$geth account list|head -n1|perl -ne '/([a-f0-9]{40})/ && print \$1'` - echo -n "starting instance $id ($BZZKEY @ $datadir )..." - # echo "$geth \ - # --identity=$id \ - # --bzzaccount=$BZZKEY --bzzport=$bzzport \ - # --unlock=$BZZKEY \ - # --password=<(echo -n $id) \ - # --rpc --rpcport=$rpcport --rpccorsdomain='*' $* \ - # 2>&1 | tee "$stablelog" > "$log" & # comment out if you pipe it to a tty etc. - # " >&2 - - - $GETH --datadir=$datadir \ - --identity=$id \ - --bzzaccount=$BZZKEY --bzzport=$bzzport \ - --port=$port \ - --unlock=$BZZKEY \ - --password=<(echo -n $id) \ - --rpc --rpcport=$rpcport --rpccorsdomain='*' $* \ - > "$log" 2>&1 & # comment out if you pipe it to a tty etc. - - ln -sf "$log" "$linklog" - - # wait until ready - # pid=`ps auxwww|grep geth|grep "ty=$id"|grep -v grep|awk '{print $2}'` - # echo "pid: $pid" - # ps auxwww|grep geth|grep "ty=$id"|grep -v grep - # echo $pid > $root/pids/$id.pid - #echo $! > $root/pids/$id.pid - while true; do - $GETH --exec="net" attach ipc:$datadir/geth.ipc > /dev/null 2>&1 && break - sleep 1 - echo -n "." - if ((i++>10)); then - echo "instance $id failed to start" - exit 1 - fi - done - echo -n "started - " - pid=`ps auxwww|grep geth|grep "ty=$id"|grep -v grep|awk '{print $2}'` - echo "pid: $pid" - # ps auxwww|grep geth|grep "ty=$id"|grep -v grep - echo $pid > $root/pids/$id.pid -fi - -# to bring up logs, uncomment -# tail -f $log diff --git a/swarm/cmd/swarm/swarm b/swarm/cmd/swarm/swarm index 5289aba5a3..6f78060a6f 100755 --- a/swarm/cmd/swarm/swarm +++ b/swarm/cmd/swarm/swarm @@ -1,2 +1,664 @@ #!/bin/bash -bash ~/bin/swarm.sh $SWARM_DIR $SWARM_NETWORK_ID $* \ No newline at end of file +if [ "$GETH_DIR" = "" ]; then + if [ "$GOPATH" = "" ]; then echo "either GETH_DIR or GOPATH environment variable must be set"; exit 1; fi + export GETH_DIR=$GOPATH/src/github.com/ethereum/go-ethereum +fi + +if [ "$GETH" = "" ]; then + export GETH=$GETH_DIR/geth +fi + +if [ "$SWARM_NETWORK_ID" = "" ]; then export SWARM_NETWORK_ID=322; fi + +if [ "$SWARM_DIR" = "" ]; then export SWARM_DIR=$HOME/bzz; fi + +if [ "$IP_ADDR" = "" ]; then + export IP_ADDR=`curl ipecho.net/plain 2>/dev/null;echo ` +fi + +root=$SWARM_DIR +network_id=$SWARM_NETWORK_ID +cmd=$1 +shift + +dir="$root/$network_id" + +tmpdir=/tmp + +# swarm attach 00 brings up a console attached to a running instance +function attach { + id=$1 + shift + echo "attaching console to instance $id" + cmd="$GETH --datadir=$dir/data/$id $* attach ipc:$dir/data/$id/geth.ipc" + # echo $cmd + eval $cmd + } + +# swarm attach 00 brings up a console attached to a running instance +function execute { + id=$1 + shift + # attach $id --exec "'$*' " + cmd="$GETH --datadir=$dir/data/$id --exec '$*' attach ipc:$dir/data/$id/geth.ipc" + # echo $cmd + eval $cmd +} + +# swarm hive 00 displays the kademlia table of the given running instance +function hive { + if [ "$1" = "all" ]; then + N=`ls -1 -d $dir/data/* |wc -l` + for ((i=0;i $log 2>&1 &" + + $GETH $opts --password=<(echo -n $id) > "$log" 2>&1 & # comment out if you pipe it to a tty etc. + ln -sf "$log" "$linklog" + + # wait until ready + + ((j=0)) + while true; do + execute $id "net" > /dev/null 2>&1 && break + sleep 1 + echo -n "." + if ((j++>10)); then + echo "instance $id failed to start" + exit 1 + fi + done + echo -n "started - " + pid=`ps auxwww|grep geth|grep "ty=$id"|grep -v grep|awk '{print $2}'` + echo "pid: $pid" + echo $pid > $dir/pids/$id.pid +} + + +# setup 00 creates the direcories for instance +function setup { + id=$1 + shift + mkdir -p $dir/data/$id + mkdir -p $dir/enodes + mkdir -p $dir/pids + mkdir -p $dir/log +} + +# creates the swarm base account for an instance +function create-account { + id=$1 + datadir=$dir/data/$id + # if we do not have an account, create one + # will not prompt for password, we use the double digit instance id as passwd + # NEVER EVER USE THESE ACCOUNTS FOR INTERACTING WITH A LIVE CHAIN + keystoredir="$datadir/keystore/" + # echo "KeyStore dir: $keystoredir" + if [ ! -d "$keystoredir" ]; then + # echo "create an account with password $id [DO NOT EVER USE THIS ON LIVE]" + # mkdir -p $datadir/keystore + $GETH --datadir=$datadir --password=<(echo -n $id) account new >/dev/null 2>&1 + # create account with password 00, 01, ... + # note that the account key will be stored also separately outside + # datadir + # this way you can safely clear the data directory and still keep your key + # under /keystore/dd + # LS=$(ls $datadir/keystore) + # echo $LS + while [ ! -d "$keystoredir" ]; do + echo "." + ((i++)) + if ((i>10)); then break; fi + sleep 1 + done + # echo "copying keys $datadir/keystore $root/keystore/$id" + mkdir -p $dir/keystore/$id + cp -R "$keystoredir" $dir/keystore/$id + fi +} + +# shuts down a running instance, cleans the pid +function stop { + id=$1 + shift + if [ $id = "all" ]; then + procs=`cat $dir/pids/*.pid 2>/dev/null |perl -pe 's/^\s+//;s/\s+\\$//;s/\s+/\n/g'` + # echo "stopping processes $procs" + for p in $procs; do + shutdown $p + done + rm -rf $dir/pids/* + else + pid=$dir/pids/$id.pid + if [ -f $pid ]; then + echo "stopping instance $id, pid="`cat $pid` + shutdown `cat $pid` + rm $pid + fi + fi +} + +# shutdown kills the node with interrupt 2 - if it resits falls back to -9 after 10s +function shutdown { + echo -n "stopping $1..." + kill -2 $1 + while true; do + ps auxwww|grep geth|grep -v grep|awk '{print $2}'|grep -ql $1 || break + if ((i++>5)); then + echo "not stopping. killing it" + kill -QUIT $1 + break + fi + echo '.' + sleep 1 + done + echo "stopped" +} + +# swarm restart 00 calls stop and start +function restart { + id=$1 + shift + if [ $id = "all" ]; then + stop all + N=`ls -d1 $dir/data/*|wc -l` + cluster $N $* + else + stop $id + start $id $* + fi +} + +# swarm init X sets up and starts a new client instance +########################################################## +# +# IT WIPES THE DATABASE +# +########################################################## +function init { + killall geth + reset all + cluster $* + enode all + connect all +} + +# reset wipes the datadirs of the instance +function reset { + id=$1 + shift + if [ $id = "all" ]; then + rm -rf $dir + else + rm -rf$dir/*/$id* + fi +} + +# enode displays the instance's enode address +# swarm enode all writes all instances' enodes in a file +function enode { + id=$1 + shift + + if [ $id = "all" ]; then + json=$dir/static-nodes.json + enodes=$dir/enodes.lst + cmd=$dir/connect.js + rm -f $enodes $json $cmd + # build a static nodes(-like) list of all enodes of the local cluster + echo "[" >> $json + N=`ls -1 -d $dir/data/* |wc -l` + for ((i=0;i $f + echo -n "admin.addPeer(" >> $cmd + cat "$f" | perl -pe 's/\s*$//' >> $cmd + echo ");" >> $cmd + cat $f |perl -pe 's/"//g'>> $enodes + cat $f >> $json + echo "," >> $json + done + echo "\"\"]" >> $json + cat $enodes + else + # echo "local IP: $ip_addr " + execute $id 'admin.nodeInfo.enode' |perl -pe "s/\[\:\:\]/$IP_ADDR/ " + fi +} + +# connect sources the local or remote set of peers and connects the node to the peers +function connect { + id=$1 + shift + if [ $id = "all" ]; then + N=`ls -1 -d $dir/data/* |wc -l` + for ((i=0;i $key" + download $id $key $dest && cmp --silent $file $target && echo "PASS" || echo "FAIL" +} + +# swarm up 00 file uploads file via instances CLI +function up { #port, file + echo "Upload file '$2' to node $1... " 1>&2 + file=`basename $2` + execute $1 "bzz.upload(\"$2\", \"$file\")"|tail -n1> /tmp/key + cat /tmp/key +} + +# swarm download 00 file download file via instances CLI +function download { + echo "download '$2' from node $1 to '$3'" + execute $1 "bzz.download(\"$2\", \"$3\")" >/dev/null +} + +# swarm down issues bzz.get to download the content 10 attempts +function down { + echo -n "Download hash '$2' from node $1... " + while true; do + execute $1 "bzz.get(\"$2\")" 2> /dev/null |grep -qil "status" && break + sleep 1 + echo -n "." + if ((i++>10)); then + echo "not found" + return + fi + done + echo "found OK" +} + +# static info about an instance (available even if node is off) +function info { + echo "swarm node information" + echo "ROOTDIR: $root" + echo "DATADIR: $dir/data/$1" + echo "LOGFILE: $dir/log/$1.log" + echo "HTTPAPI: http://localhost:322$1" + echo "ETHPORT: 303$1" + echo "RPCPORT: 302$1" + echo "ACCOUNT:" 0x`ls -1 $dir/data/$1/bzz` + echo "CHEQUEB:" `cat $dir/data/$1/bzz/*/config.json|grep Contract|awk -F\" '{print $4}'` + echo "ROOTDIR: $root" + echo "DATADIR: $dir/data/$1" + echo "LOGFILE: $dir/log/$1.log" +} + +# live into about an instance +function status { + echo -n "account balance: " + execute $1 'eth.getBalance(eth.accounts[0])' + echo -n "swap contract balance: " + execute $1 "eth.getBalance(bzz.info.Swap.Contract)" + echo -n "chequebook balance: " + execute $1 "chequebook.balance" + echo -n "peer count: " + execute $1 'net.peerCount' + echo -n "latest block number: " + execute $1 "eth.blockNumber" +} + +# display peers for an instance +function peers { + execute $1 'admin.peers' +} + +# add peers into an instance (connection not guaranteed) +function addpeers { + id=$1 + peers=$2 + if [ $id = "all" ]; then + N=`ls -1 -d $dir/data/* |wc -l` + for ((i=0;i/dev/null;echo ` + ws_server="ws://146.185.130.117:3000" + ws_secret=BZZ322 + conf="$dir/$group-$ip.netstat.json" + + echo "writing netstat conf for cluster $group-$ip ($N instances) -> $conf" + + echo -e "[" > $conf + + for ((i=0;i<$N;++i)); do + id=`printf "%02d" $i` + single_template=" {\n \"name\" : \"$group-$ip-$i\",\n \"cwd\" : \".\",\n \"script\" : \"app.js\",\n \"log_date_format\" : \"YYYY-MM-DD HH:mm Z\",\n \"merge_logs\" : false,\n \"watch\" : false,\n \"exec_interpreter\" : \"node\",\n \"exec_mode\" : \"fork_mode\",\n \"env\":\n {\n \"NODE_ENV\" : \"production\",\n \"RPC_HOST\" : \"localhost\",\n \"RPC_PORT\" : \"302$id\",\n \"INSTANCE_NAME\" : \"$group-$ip-$i\",\n \"WS_SERVER\" : \"$ws_server\",\n \"WS_SECRET\" : \"$ws_secret\",\n }\n }" + + endline="" + if ((i<$N-1)); then + # if [ "$i" -ne "$N" ]; then + endline="," + fi + echo -e "$single_template$endline" >> $conf + done + echo "]" >> $conf +} + +# (re)starts the eth-net-intelligence-api network monitor +function netstatrun { + cd $GETH_DIR/../eth-net-intelligence-api + pm2 kill + pm2 start $dir/*.netstat.json +} + +# kills the eth-net-intelligence-api network monitor +function netstatkill { + cd $GETH_DIR/../eth-net-intelligence-api + pm2 kill +} + +# copies the swarm control script to the remote node(s) +function remote-update-scripts { + scriptdir=$GETH_DIR/swarm/cmd/swarm/ + remotes=$1 + for remote in `cat $remotes|grep -v '^#'`; do echo "updating scripts on $remote..."; ssh $remote mkdir -p bin && scp -r $scriptdir/* $remote:bin/; done +} + +# copies the geth executable to the remote nodes +function remote-update-bin { + remotes=$1 + # remote-update-scripts $remotes + for remote in `cat $remotes|grep -v '^#'`; do echo "updating binary on $remote..."; scp -r $GETH_DIR/geth $remote:bin/; done +} + +# runs a command on remote node or nodes from a file +function remote-run { + remotes=$1 + shift + if `echo "$remotes" | grep -qil @`; then + ssh $remotes '. $HOME/bin/env.sh;' "$*" + else + for remote in `cat $remotes|grep -v '^#'`; do echo "running on $remote..."; remote-run $remote "$*"; done + fi +} + +# updates the code from the given branch +function update { + branch=$1 + echo "cd $GETH_DIR && git remote update && git reset --hard $branch" + (cd $GETH_DIR && git remote update && git reset --hard $branch) +} + + +case $cmd in + "info" ) + info $*;; + "enode" ) + enode $*;; + "status" ) + status $*;; + "peers" ) + peers $*;; + "addpeers" ) + addpeers $*;; + "clean" ) + clean $*;; + "needs" ) + needs $*;; + "up" ) + up $*;; + "key" ) + key $*;; + "down" ) + down $*;; + "download" ) + download $*;; + "init" ) + init $*;; + "exec" ) + execute $*;; + "hive" ) + hive $*;; + "start" ) + start $*;; + "stop" ) + stop $* ;; + "restart" ) + restart $*;; + "reset" ) + reset $*;; + "cluster" ) + cluster $*;; + "attach" ) + attach $*;; + "execute" ) + execute $*;; + "exec" ) + execute $*;; + "cleanbzz" ) + cleanbzz $*;; + "cleanlog" ) + cleanlog $*;; + "log" ) + log $*;; + "viewlog" ) + viewlog $*;; + "less" ) + viewlog $*;; + "connect" ) + connect $*;; + "monitor" ) + monitor $*;; + "remote-update-scripts" ) + remote-update-scripts $*;; + "remote-update-bin" ) + remote-update-bin $*;; + "update-src" ) + update-src $*;; + "remote-run" ) + remote-run $*;; + "netstatconf" ) + netstatconf $*;; + "netstatkill" ) + netstatkill $*;; + "netstatrun" ) + netstatrun $*;; + "options" ) + options $*;; + "rawoptions" ) + rawoptions $*;; + "setup" ) + setup $* ;; + "create-account" ) + create-account $*;; + +esac diff --git a/swarm/cmd/swarm/swarm.sh b/swarm/cmd/swarm/swarm.sh deleted file mode 100644 index dde74d6b71..0000000000 --- a/swarm/cmd/swarm/swarm.sh +++ /dev/null @@ -1,432 +0,0 @@ -# !/bin/bash -# bash cluster [[params]...] -# https://github.com/ethereum/go-ethereum/wiki/Setting-up-monitoring-on-local-cluster - -# sets up a local ethereum network cluster of nodes -# - is the number of nodes in cluster -# - is the root directory for the cluster, the nodes are set up -# with datadir `//00`, `/ /01`, ... -# - new accounts are created for each node -# - they launch on port 30300, 30301, ... -# - they star rpc on port 8100, 8101, ... -# - by collecting the nodes nodeUrl, they get connected to each other -# - if enode has no IP, `` is substituted -# - if `` is not 0, they will not connect to a default client, -# resulting in a private isolated network -# - the nodes log into `//00..log`, `//01..log`, ... -# - The nodes launch in mining mode -# - the cluster can be killed with `killall geth` (FIXME: should record PIDs) -# and restarted from the same state -# - if you want to interact with the nodes, use rpc -# - you can supply additional params on the command line which will be passed -# to each node, for instance `-mine` - -if [ "$GETH" = "" ]; then - echo "env var GETH not set " - exit 1 -fi - -srcdir=`dirname $0` - -root=$1 -shift -network_id=$1 -shift -cmd=$1 -shift -# ip_addr=`curl ipecho.net/plain 2>/dev/null;echo ` - -# echo "external IP: $ip_addr" -swarmoptions='--dev --maxpeers=40 --shh=false --nodiscover' -tmpdir=/tmp - -function attach { - id=$1 - shift - echo "attaching console to instance $id" - cmd="$GETH $* attach ipc:$root/$network_id/data/$id/geth.ipc" - echo $cmd - eval $cmd -} - -function log { - id=$1 - shift - echo "streaming logs for instance $id" - cmd="tail -f $root/$network_id/log/$id.log" - echo $cmd - eval $cmd -} - -function cleanlog { - id=$1 - shift - if [ $id = "all" ]; then - echo "remove logs for all instances" - rm -rf "$root/$network_id/log/" - else - echo "remove logs for instance $id" - rm -rf $root/$network_id/log/$id* - fi -} - -function cleanbzz { - id=$1 - shift - if [ $id = "all" ]; then - echo "remove bzz data for all instances" - rm -rf $root/$network_id/data/*/bzz - else - echo "remove bzz data for instance $id" - rm -rf "$root/$network_id/data/$id" - fi -} - -function less { - id=$1 - shift - echo "viewing logs for instance $id" - cmd="/usr/bin/less $root/$network_id/log/$id.log" - echo $cmd - eval $cmd -} - -function start { - id=$1 - shift - # echo -n "starting instance $id - " - cmd="bash $srcdir/gethup.sh $root/$network_id/ $id '$ip_addr' --networkid=$network_id $swarmoptions $*" - # echo "pid="`cat $root/$network_id/pids/$id.pid` - # echo $cmd - eval $cmd -} - -function stop { - id=$1 - shift - if [ $id = "all" ]; then - procs=`cat $root/$network_id/pids/*.pid 2>/dev/null |perl -pe 's/^\s+//;s/\s+\\$//;s/\s+/\n/g'` - # echo "stopping processes $procs" - for p in $procs; do - shutdown $p - done - rm -rf $root/$network_id/pids/* - else - pid=$root/$network_id/pids/$id.pid - if [ -f $pid ]; then - echo "stopping instance $id, pid="`cat $pid` - shutdown `cat $pid` - rm $pid - fi - fi - # ps auxwww|grep geth|grep bzz|grep -v grep -} - -function shutdown { - echo -n "stopping $1..." - kill -2 $1 - while true ;do - ps auxwww|grep geth|grep -v grep|awk '{print $2}'|grep -ql $1 || break - sleep 1 - done - echo "stopped" -} - -function restart { - id=$1 - shift - stop $id - start $id $* -} - -function init { - stop all - killall geth - reset all - cluster $* - enode all - connect all -} - -function reset { - id=$1 - shift - if [ $id = "all" ]; then - rm -rf $root/$network_id - else - rm -rf$root/$network_id/*/$id* - fi - -} - -function enode { - dir=$root/$network_id - id=$1 - shift - if [ $id = "all" ]; then - N=`ls -1 $dir/enodes/|wc -l` - enodes=$dir/enodes.all - rm -f $enodes - # build a static nodes(-like) list of all enodes of the local cluster - echo "[" >> $enodes - for ((i=0;i> $enodes - echo "," >> $enodes - fi - done - echo "\"\"]" >> $enodes - cmd=$dir/connect.js - for ((i=0;i> $cmd - cat "$enode" >> $cmd - echo ");" >> $cmd - fi - done - else - enode=$dir/enodes/$id.enode - attach $id --exec "'console.log(admin.nodeInfo.enode)'" |head -2 |tail -1| perl -pe 's/^/"/;s/$/"/'|perl -pe 's/\s*$//' > $enode - # cat $enode - fi - -} - -function connect { - dir=$root/$network_id - id=$1 - shift - if [ $id = "all" ]; then - for ((i=0;i tail -f $dir/log/$id.log" - start $id $vmodule $* - done -} - - -function needs { - id=$1 - keyfile=$2 - target=$3 - dir=`dirname $3` - dest=$tmpdir/down - mkdir -p $dest - file=$dest/`basename $target` - rm -f $file - echo -n "waiting for root hash in '$keyfile'..." - while true; do - if [ -f $keyfile ] && [ ! -z $keyfile ]; then - break - fi - sleep 1 - echo -n "." - done - key=`cat $keyfile|tr -d \"` - echo " => $key" - download $id $key $dest && cmp --silent $file $target && echo "PASS" || echo "FAIL" - # && ls -l $keyfile $file $target -} - - -function up { #port, file - echo "Upload file '$2' to node $1... " 1>&2 - file=`basename $2` - attach $1 "--exec 'bzz.upload(\"$2\", \"$file\")'"|tail -n1> /tmp/key - # key=`bash swarm/cmd/bzzup.sh $2 86$1` - cat /tmp/key -} - -function download { - echo "download '$2' from node $1 to '$3'" - # echo attach $1 "--exec 'bzz.download(\"$2\", \"$3\")'" - attach $1 "--exec 'bzz.download(\"$2\", \"$3\")'" > /dev/null -} - - -function down { - echo -n "Download hash '$2' from node $1... " - # echo "wget -O- http://localhost:86$1/$2 > /dev/null 2>&1 && echo 'got it' || echo 'not found'" - # wget -O- http://localhost:86$1/$2 > /dev/null 2>&1 && echo "got it" || echo "not found" - while true; do - attach $1 "--exec 'bzz.get(\"$2\")'" 2> /dev/null |grep -qil "status" && break - sleep 1 - echo -n "." - if ((i++>10)); then - echo "not found" - return - fi - done - echo "found OK" -} - -function clean { #index - echo "Clean up for $1" - rm -rf $root/$network_id/data/$1/{bzz/*/chunks,bzz/*/requests/,bzz/*/bzz-peers.json,chaindata,nodes} -} - -function info { - echo "swarm node information" - echo "ROOTDIR: $root" - echo "DATADIR: $root/$network_id/data/$1" - echo "LOGFILE: $root/$network_id/log/$1.log" - echo "HTTPAPI: http://localhost:322$1" - echo "ETHPORT: 303$1" - echo "RPCPORT: 302$1" - echo "ACCOUNT:" 0x`ls -1 $root/$network_id/data/$1/bzz` - echo "CHEQUEB:" `cat $root/$network_id/data/$1/bzz/*/config.json|grep Contract|awk -F\" '{print $4}'` - echo "ROOTDIR: $root" - echo "DATADIR: $root/$network_id/data/$1" - echo "LOGFILE: $root/$network_id/log/$1.log" -} - - -function status { - attach 00 -exec "'console.log(eth.getBalance(eth.accounts[0])); console.log(eth.getBalance(bzz.info().Swap.Contract)); console.log(chequebook.balance)'" -} - -function netstatconf { - begin=$1 - N=$2 - name_prefix=$3 - ws_server=$4 - ws_secret=$5 - conf="$root/$network_id/$name_prefix.netstat.json" - - echo "writing netstat conf for cluster $name_prefix to $conf" - - echo -e "[" > $conf - - for ((i=$begin;i<$start+$N;++i)); do - id=`printf "%02d" $i` - single_template=" {\n \"name\" : \"$name_prefix-$i\",\n \"cwd\" : \".\",\n \"script\" : \"app.js\",\n \"log_date_format\" : \"YYYY-MM-DD HH:mm Z\",\n \"merge_logs\" : false,\n \"watch\" : false,\n \"exec_interpreter\" : \"node\",\n \"exec_mode\" : \"fork_mode\",\n \"env\":\n {\n \"NODE_ENV\" : \"production\",\n \"RPC_HOST\" : \"localhost\",\n \"RPC_PORT\" : \"302$id\",\n \"INSTANCE_NAME\" : \"$name_prefix-$i\",\n \"WS_SERVER\" : \"$ws_server\",\n \"WS_SECRET\" : \"$ws_secret\",\n }\n }" - - endline="" - if (($i<$N-1)); then - # if [ "$i" -ne "$N" ]; then - endline="," - fi - echo -e "$single_template$endline" >> $conf - done - - echo "]" >> $conf -} - -function remote-update-scripts { - scriptdir=$1 - remotes=$2 - cd $GETH_DIR - for remote in `cat $remotes|grep -v '^#'`; do echo "updating scripts on $remote..."; ssh $remote mkdir -p bin && scp -r $scriptdir/* $remote:bin/; done -} - -function remote-update-bin { - remotes=$1 - remote-update-scripts $GETH_DIR/swarm/cmd/swarm/ $remotes - for remote in `cat $remotes|grep -v '^#'`; do echo "updating binary on $remote..."; scp -r $GETH_DIR/geth $remote:bin/; done -} - -function remote-run { - remotes=$1 - shift - for remote in `cat $remotes|grep -v '^#'`; do echo "running on $remote..."; ssh $remote ". ~/bin/env.sh; $*"; done -} - -function update-src { - branch=$1 - echo "cd $GETH_DIR && git remote update && git reset --hard $branch" - (cd $GETH_DIR && git remote update && git reset --hard $branch) -} - -function netstatrun { - cd ~/eth-net-intelligence-api - pm2 kill - pm2 start $root/$network_id/*.netstat.json -} - - -case $cmd in - "info" ) - info $*;; - "enode" ) - enode $*;; - "connect" ) - connect $*;; - "status" ) - status $*;; - "clean" ) - clean $*;; - "needs" ) - needs $*;; - "up" ) - up $*;; - "down" ) - down $*;; - "download" ) - download $*;; - "init" ) - init $*;; - "start" ) - start $*;; - "stop" ) - stop $* ;; - "restart" ) - restart $*;; - "reset" ) - reset $*;; - "cluster" ) - cluster $*;; - "attach" ) - attach $*;; - "cleanbzz" ) - cleanbzz $*;; - "cleanlog" ) - cleanlog $*;; - "log" ) - log $*;; - "less" ) - less $*;; - "remote-update-scripts" ) - remote-update-scripts $*;; - "remote-update-bin" ) - remote-update-bin $*;; - "update-src" ) - update-src $*;; - "remote-run" ) - remote-run $*;; - "netstatconf" ) - netstatconf $*;; - "netstatrun" ) - netstatrun $*;; - -esac diff --git a/swarm/cmd/swarm/test.sh b/swarm/cmd/swarm/test.sh deleted file mode 100644 index 5ded17493a..0000000000 --- a/swarm/cmd/swarm/test.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -TEST_DIR=`dirname $0` -TEST_NAME=`basename $0 .sh` -TEST_TYPE=`basename $TEST_DIR` - - -export SWARM_BIN=$TEST_DIR/../../cmd/swarm -export GETH=$SWARM_BIN/../../../geth -export NETWORKID=322$TEST_NAME -export TMPDIR=~/BZZ/test/$TEST_TYPE -export DATA_ROOT=$TMPDIR/$NETWORKID -# alias swarm='bash $SWARM_BIN/swarm.sh $DATA_ROOT $NETWORKID' -EXTRA_ARGS=$* - -rm -rf $DATA_ROOT - -wait=1 - -function swarm { - # echo bash $SWARM_BIN/swarm.sh $TMPDIR $NETWORKID $* $EXTRA_ARGS - bash $SWARM_BIN/swarm.sh $TMPDIR $NETWORKID $* $EXTRA_ARGS -} - - -function randomfile { - dd if=/dev/urandom of=/dev/stdout bs=1024 count=$1 2>/dev/null -} \ No newline at end of file diff --git a/swarm/network/hive.go b/swarm/network/hive.go index bb01365a09..904a687ad4 100644 --- a/swarm/network/hive.go +++ b/swarm/network/hive.go @@ -30,6 +30,7 @@ type Hive struct { addr kademlia.Address kad *kademlia.Kademlia path string + quit chan bool toggle chan bool more chan bool @@ -106,6 +107,7 @@ func (self *Hive) Addr() kademlia.Address { func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPeer func(string) error) (err error) { self.toggle = make(chan bool) self.more = make(chan bool) + self.quit = make(chan bool) self.id = id self.listenAddr = listenAddr err = self.kad.Load(self.path, nil) @@ -145,11 +147,14 @@ func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPee } else { glog.V(logger.Warn).Infof("[BZZ] KΛÐΞMLIΛ hive: no peer") } - self.toggle <- true glog.V(logger.Detail).Infof("[BZZ] KΛÐΞMLIΛ hive: buzz kept alive") } else { glog.V(logger.Info).Infof("[BZZ] KΛÐΞMLIΛ hive: no need for more bees") - self.toggle <- false + } + select { + case self.toggle <- need: + case <-self.quit: + return } glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: queen's address: %v, population: %d (%d)", self.addr, self.kad.Count(), self.kad.DBCount()) } @@ -174,11 +179,7 @@ func (self *Hive) keepAlive() { default: } } - case need, alive := <-self.toggle: - if !alive { - self.more <- false - return - } + case need := <-self.toggle: if alarm == nil && need { alarm = time.NewTicker(time.Duration(self.callInterval)).C } @@ -186,13 +187,15 @@ func (self *Hive) keepAlive() { alarm = nil } + case <-self.quit: + return } } } func (self *Hive) Stop() error { // closing toggle channel quits the updateloop - close(self.toggle) + close(self.quit) return self.kad.Save(self.path, saveSync) } diff --git a/swarm/network/kademlia/kaddb.go b/swarm/network/kademlia/kaddb.go index 959a70beab..33cae221cb 100644 --- a/swarm/network/kademlia/kaddb.go +++ b/swarm/network/kademlia/kaddb.go @@ -25,8 +25,7 @@ type NodeRecord struct { Seen time.Time // last connected at time Meta *json.RawMessage // arbitrary metadata saved for a peer - node Node - connected bool + node Node } func (self *NodeRecord) setSeen() { @@ -45,7 +44,7 @@ type KadDb struct { Nodes [][]*NodeRecord index map[Address]*NodeRecord cursors []int - lock sync.Mutex + lock sync.RWMutex purgeInterval time.Duration initialRetryInterval time.Duration connRetryExp int @@ -161,10 +160,10 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe var interval time.Duration var found bool - var count int var purge []bool var delta time.Duration var cursor int + var count int var after time.Time // iterate over columns maximum bucketsize times @@ -181,7 +180,6 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe proxLimit = po need = true } - cursor = self.cursors[po] purge = make([]bool, len(dbrow)) // there is a missing slot - finding a node to connect to @@ -192,7 +190,7 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe node = dbrow[cursor] // skip already connected nodes - if node.connected { + if node.node != nil { glog.V(logger.Debug).Infof("[KΛÐ]: kaddb record %v (PO%03d:%d/%d) already connected", node.Addr, po, cursor, len(dbrow)) continue ROW } @@ -223,7 +221,6 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe glog.V(logger.Debug).Infof("[KΛÐ]: kaddb record %v (PO%03d:%d) selected as candidate connection %v. seen at %v (%v ago), selectable since %v, retry after %v (in %v)", node.Addr, po, cursor, rounds, node.Seen, delta, node.After, after, interval) node.After = after found = true - break ROW } // ROW self.cursors[po] = cursor self.delete(po, purge) diff --git a/swarm/network/kademlia/kademlia.go b/swarm/network/kademlia/kademlia.go index 6b7e8066ac..92f24b512b 100644 --- a/swarm/network/kademlia/kademlia.go +++ b/swarm/network/kademlia/kademlia.go @@ -16,12 +16,13 @@ const ( proxBinSize = 4 maxProx = 8 connRetryExp = 2 + maxPeers = 100 ) var ( purgeInterval = 42 * time.Hour - initialRetryInterval = 42 * 100 * time.Millisecond - maxIdleInterval = 42 * 10 * time.Second + initialRetryInterval = 42 * time.Millisecond + maxIdleInterval = 42 * 100 * time.Millisecond ) type KadParams struct { @@ -31,6 +32,7 @@ type KadParams struct { BucketSize int PurgeInterval time.Duration InitialRetryInterval time.Duration + MaxIdleInterval time.Duration ConnRetryExp int } @@ -41,6 +43,7 @@ func NewKadParams() *KadParams { BucketSize: bucketSize, PurgeInterval: purgeInterval, InitialRetryInterval: initialRetryInterval, + MaxIdleInterval: maxIdleInterval, ConnRetryExp: connRetryExp, } } @@ -52,7 +55,7 @@ type Kademlia struct { proxLimit int // state, the PO of the first row of the most proximate bin proxSize int // state, the number of peers in the most proximate bin count int // number of active peers (w live connection) - buckets []*bucket // the actual bins + buckets [][]Node // the actual bins db *KadDb // kaddb, node record database lock sync.RWMutex // mutex to access buckets } @@ -68,14 +71,9 @@ type Node interface { // add is the base address of the table // params is KadParams configuration func New(addr Address, params *KadParams) *Kademlia { - buckets := make([]*bucket, params.MaxProx+1) - for i, _ := range buckets { - buckets[i] = &bucket{size: params.BucketSize} // will initialise bucket{int(0),[]Node(nil),sync.Mutex} - } - glog.V(logger.Info).Infof("[KΛÐ] base address %v", addr) + buckets := make([][]Node, params.MaxProx+1) + glog.V(logger.Info).Infof("[KΛÐ] base address %v", addr.String()[:6]) - // ! temporary hack fixme: - params.ProxBinSize = 4 return &Kademlia{ addr: addr, KadParams: params, @@ -109,9 +107,6 @@ func (self *Kademlia) On(node Node, cb func(*NodeRecord, Node) error) (err error index := self.proximityBin(node.Addr()) record := self.db.findOrCreate(index, node.Addr(), node.Url()) - // callback on add node - // setting the node on the record, set it checked (for connectivity) - record.node = node if cb != nil { err = cb(record, node) @@ -119,34 +114,46 @@ func (self *Kademlia) On(node Node, cb func(*NodeRecord, Node) error) (err error if err != nil { return fmt.Errorf("unable to add node %v, callback error: %v", node.Addr(), err) } - glog.V(logger.Info).Infof("[KΛÐ]: add node record %v with node %v", record, node) + glog.V(logger.Debug).Infof("[KΛÐ]: add node record %v with node %v", record, node) } - record.connected = true // insert in kademlia table of active nodes bucket := self.buckets[index] // if bucket is full insertion replaces the worst node // TODO: give priority to peers with active traffic - replaced, err := bucket.insert(node) - if err != nil { - glog.V(logger.Debug).Infof("[KΛÐ]: node %v not needed: %v", node, err) - return err - // no prox adjustment needed - // do not change count - } - if replaced != nil { - glog.V(logger.Debug).Infof("[KΛÐ]: node %v replaced by %v ", replaced, node) + if len(bucket) >= self.BucketSize { // >= allows us to add peers beyond the bucketsize limitation + // always rotate peers + idle := self.MaxIdleInterval + var pos int + var replaced Node + for i, p := range bucket { + idleInt := time.Since(p.LastActive()) + if idleInt > idle { + idle = idleInt + pos = i + replaced = p + } + } + if replaced == nil { + glog.V(logger.Debug).Infof("[KΛÐ]: all peers wanted, PO%03d bucket full", index) + return fmt.Errorf("bucket full") + } + glog.V(logger.Debug).Infof("[KΛÐ]: node %v replaced by %v (idle for %v > %v)", replaced, node, idle, self.MaxIdleInterval) replaced.Drop() + self.buckets[index] = append(bucket[:pos], bucket[(pos+1):]...) + // there is no change in bucket cardinalities so no prox limit adjustment is needed return nil + } else { + self.buckets[index] = append(bucket, node) + glog.V(logger.Debug).Infof("[KΛÐ]: add node %v to table", node) + self.count++ + self.setProxLimit(index, true) } - // new node added - glog.V(logger.Info).Infof("[KΛÐ]: add node %v to table", node) - self.count++ - self.setProxLimit(index, false) + record.node = node return nil } -// is the entrypoint called when a node is taken offline +// Off is the called when a node is taken offline (from the protocol main loop exit) func (self *Kademlia) Off(node Node, cb func(*NodeRecord, Node)) (err error) { self.lock.Lock() defer self.lock.Unlock() @@ -154,68 +161,70 @@ func (self *Kademlia) Off(node Node, cb func(*NodeRecord, Node)) (err error) { var found bool index := self.proximityBin(node.Addr()) bucket := self.buckets[index] - for i := 0; i < len(bucket.nodes); i++ { - if node.Addr() == bucket.nodes[i].Addr() { + for i := 0; i < len(bucket); i++ { + if node.Addr() == bucket[i].Addr() { found = true - bucket.nodes = append(bucket.nodes[:i], bucket.nodes[(i+1):]...) + self.buckets[index] = append(bucket[:i], bucket[(i+1):]...) + break } } if !found { - // gracefully return without error if peer already offline + // gracefully return without error if peer already unregistered + glog.V(logger.Warn).Infof("[KΛÐ]: remove node %v not in table, population now is %v", node, self.count) return nil } - glog.V(logger.Info).Infof("[KΛÐ]: remove node %v from table", node) self.count-- - if len(bucket.nodes) < bucket.size { - err = fmt.Errorf("insufficient nodes (%v) in bucket %v", len(bucket.nodes), index) - } + glog.V(logger.Debug).Infof("[KΛÐ]: remove node %v from table, population now is %v", node, self.count) + self.setProxLimit(index, false) - self.setProxLimit(index, true) - - r := self.db.index[node.Addr()] + record := self.db.index[node.Addr()] // callback on remove if cb != nil { - cb(r, r.node) + cb(record, record.node) } - r.node = nil - r.connected = false + record.node = nil return } // proxLimit is dynamically adjusted so that // 1) there is no empty buckets in bin < proxLimit and -// 2) the sum of all items sare the maximpossible but lower than ProxBinSize +// 2) the sum of all items are the minimum possible but higher than ProxBinSize // adjust Prox (proxLimit and proxSize after an insertion/removal of nodes) // caller holds the lock -func (self *Kademlia) setProxLimit(r int, off bool) { - // glog.V(logger.Info).Infof("[KΛÐ]: adjust proxbin for (bin: %v, off: %v)", r, off) - if r < self.proxLimit && len(self.buckets[r].nodes) > 0 { +func (self *Kademlia) setProxLimit(r int, on bool) { + // if the change is outside the core (PO lower) + // and the change does not leave a bucket empty then + // no adjustment needed + if r < self.proxLimit && len(self.buckets[r]) > 0 { return } - glog.V(logger.Detail).Infof("[KΛÐ]: set proxbin (size: %v, limit: %v, bin: %v, off: %v)", self.proxSize, self.proxLimit, r, off) - if off { - self.proxSize-- - for (self.proxSize < self.ProxBinSize || r < self.proxLimit) && - self.proxLimit > 0 { - // - self.proxLimit-- - self.proxSize += len(self.buckets[self.proxLimit].nodes) - glog.V(logger.Detail).Infof("[KΛÐ]: proxbin expansion (size: %v, limit: %v, bin: %v, off: %v)", self.proxSize, self.proxLimit, r, off) + // if on=a node was added, then r must be within prox limit so increment cardinality + if on { + self.proxSize++ + curr := len(self.buckets[self.proxLimit]) + // if now core is big enough without the furthest bucket, then contract + // this can never result in more than one bucket change + if self.proxSize >= self.ProxBinSize+curr && curr > 0 { + self.proxSize -= curr + self.proxLimit++ + glog.V(logger.Detail).Infof("[KΛÐ]: proxbin contraction (size: %v, limit: %v, bin: %v)", self.proxSize, self.proxLimit, r) } - // glog.V(logger.Detail).Infof("%v", self) return } - self.proxSize++ - for self.proxLimit < self.MaxProx && - len(self.buckets[self.proxLimit].nodes) > 0 && - self.proxSize-len(self.buckets[self.proxLimit].nodes) >= self.ProxBinSize { + // otherwise + if r >= self.proxLimit { + self.proxSize-- + } + // expand core by lowering prox limit until hit zero or cover the empty bucket or reached target cardinality + for (self.proxSize < self.ProxBinSize || r < self.proxLimit) && + self.proxLimit > 0 { // - self.proxSize -= len(self.buckets[self.proxLimit].nodes) - self.proxLimit++ - glog.V(logger.Detail).Infof("[KΛÐ]: proxbin contraction (size: %v, limit: %v, bin: %v, off: %v)", self.proxSize, self.proxLimit, r, off) + self.proxLimit-- + self.proxSize += len(self.buckets[self.proxLimit]) + glog.V(logger.Detail).Infof("[KΛÐ]: proxbin expansion (size: %v, limit: %v, bin: %v)", self.proxSize, self.proxLimit, r) } } @@ -227,60 +236,55 @@ proxLimit and MaxProx. func (self *Kademlia) FindClosest(target Address, max int) []Node { defer self.lock.RUnlock() self.lock.RLock() + r := nodesByDistance{ target: target, } - index := self.proximityBin(target) - start := index - var down bool - if index >= self.proxLimit { + po := self.proximityBin(target) + index := po + step := 1 + + // set proxbin iteration full + if index > self.proxLimit { index = self.proxLimit - start = self.MaxProx - down = true } - var n int + + // if max is set to 0, just want a full bucket, dynamic number + min := max + // set limit to max limit := max if max == 0 { - limit = 1000 + min = 1 + limit = maxPeers } - for { - bucket := self.buckets[start].nodes - for i := 0; i < len(bucket); i++ { - r.push(bucket[i], limit) + var n int + for index >= 0 { + // add entire bucket + for _, p := range self.buckets[index] { + r.push(p, limit) n++ } - if max == 0 && start <= index && (n > 0 || start == 0) || max > 0 && down && start <= index && (n >= limit || n == self.count || start == 0) { + // terminate if index reached the bottom or enough peers > min + if n >= min && (step < 0 || max == 0) { break } - if down { - start-- - } else { - if start == self.MaxProx { - if index == 0 { - break - } - start = index - 1 - down = true - } else { - start++ - } + // reach top most non-empty PO bucket, turn around + if index == self.MaxProx { + index = po + step = -1 } + index += step } - glog.V(logger.Detail).Infof("[KΛÐ]: serve %d (=<%d) nodes for target lookup %v (PO%d)", n, self.MaxProx, target, index) + glog.V(logger.Detail).Infof("[KΛÐ]: serve %d (<=%d) nodes for target lookup %v (PO%03d)", n, max, target, po) return r.nodes } -func (self *Kademlia) binsize(p int) int { - b := self.buckets[p] - defer b.lock.RUnlock() - b.lock.RLock() - return len(b.nodes) -} - func (self *Kademlia) Suggest() (*NodeRecord, bool, int) { - return self.db.findBest(self.BucketSize, self.binsize) + defer self.lock.RUnlock() + self.lock.RLock() + return self.db.findBest(self.BucketSize, func(i int) int { return len(self.buckets[i]) }) } // adds node records to kaddb (persisted node record db) @@ -288,13 +292,6 @@ func (self *Kademlia) Add(nrs []*NodeRecord) { self.db.add(nrs, self.proximityBin) } -// in situ mutable bucket -type bucket struct { - size int - nodes []Node - lock sync.RWMutex -} - // nodesByDistance is a list of nodes, ordered by distance to target. type nodesByDistance struct { nodes []Node @@ -331,31 +328,6 @@ func (h *nodesByDistance) push(node Node, max int) { } } -// insert adds a peer to a bucket either by appending to existing items if -// bucket length does not exceed bucketSize, or by replacing the worst -// Node in the bucket -func (self *bucket) insert(node Node) (replaced Node, err error) { - self.lock.Lock() - defer self.lock.Unlock() - if len(self.nodes) >= self.size { // >= allows us to add peers beyond the bucketsize limitation - // dev p2p kicks out nodes idle for > 30 s, so here we never replace nodes if - // bucket is full - // update, it seems we need to replace nodes - // return nil, fmt.Errorf("bucket full") - replaced := self.nodes[0] - self.nodes = append(self.nodes[1:], node) - return replaced, nil - } - self.nodes = append(self.nodes, node) - return -} - -func (self *bucket) length(node Node) int { - self.lock.Lock() - defer self.lock.Unlock() - return len(self.nodes) -} - /* Taking the proximity order relative to a fix point x classifies the points in the space (n byte long byte sequences) into bins. Items in each are at @@ -400,43 +372,46 @@ func (self *Kademlia) Load(path string, cb func(*NodeRecord, Node) error) (err e } // kademlia table + kaddb table displayed with ascii -// callerholds the lock func (self *Kademlia) String() string { + defer self.lock.RUnlock() + self.lock.RLock() + defer self.db.lock.RUnlock() + self.db.lock.RLock() var rows []string rows = append(rows, "=========================================================================") - rows = append(rows, fmt.Sprintf("KΛÐΞMLIΛ hive: queen's address: %v, population: %d (%d)", self.addr, self.count, self.DBCount())) - rows = append(rows, fmt.Sprintf("%v : MaxProx: %d, ProxBinSize: %d, BucketSize: %d, proxLimit: %d, proxSize: %d", time.Now(), self.MaxProx, self.ProxBinSize, self.BucketSize, self.proxLimit, self.proxSize)) + rows = append(rows, fmt.Sprintf("%v KΛÐΞMLIΛ hive: queen's address: %v", time.Now().UTC().Format(time.UnixDate), self.addr.String()[:6])) + rows = append(rows, fmt.Sprintf("population: %d (%d), proxLimit: %d, proxSize: %d", self.count, len(self.db.index), self.proxLimit, self.proxSize)) + rows = append(rows, fmt.Sprintf("MaxProx: %d, ProxBinSize: %d, BucketSize: %d", self.MaxProx, self.ProxBinSize, self.BucketSize)) - for i, b := range self.buckets { + for i, bucket := range self.buckets { if i == self.proxLimit { - rows = append(rows, fmt.Sprintf("===================== PROX LIMIT: %d =================================", i)) + rows = append(rows, fmt.Sprintf("============ PROX LIMIT: %d ==========================================", i)) } - row := []string{fmt.Sprintf("%03d", i), fmt.Sprintf("%2d", len(b.nodes))} + row := []string{fmt.Sprintf("%03d", i), fmt.Sprintf("%2d", len(bucket))} var k int c := self.db.cursors[i] - for ; k < len(b.nodes); k++ { - p := b.nodes[(c+k)%len(b.nodes)] - row = append(row, fmt.Sprintf("%s", p.Addr().String()[:8])) - if k == 3 { + for ; k < len(bucket); k++ { + p := bucket[(c+k)%len(bucket)] + row = append(row, p.Addr().String()[:6]) + if k == 4 { break } } - for ; k < 3; k++ { - row = append(row, " ") + for ; k < 4; k++ { + row = append(row, " ") } row = append(row, fmt.Sprintf("| %2d %2d", len(self.db.Nodes[i]), self.db.cursors[i])) for j, p := range self.db.Nodes[i] { - row = append(row, fmt.Sprintf("%08x", p.Addr[:4])) - if j == 2 { + row = append(row, p.Addr.String()[:6]) + if j == 3 { break } } rows = append(rows, strings.Join(row, " ")) if i == self.MaxProx { - break } } rows = append(rows, "=========================================================================") diff --git a/swarm/network/kademlia/kademlia_test.go b/swarm/network/kademlia/kademlia_test.go index de906cabe6..135963a090 100644 --- a/swarm/network/kademlia/kademlia_test.go +++ b/swarm/network/kademlia/kademlia_test.go @@ -78,7 +78,7 @@ func TestBootstrap(t *testing.T) { n := 0 for n < 100 { err = kad.On(node, nil) - if err != nil && err.Error() != "bucket full" { + if err != nil { t.Fatalf("backend not accepting node: %v", err) } @@ -150,7 +150,7 @@ func TestFindClosest(t *testing.T) { // check that the result nodes have minimum distance to target. farthestResult := nodes[len(nodes)-1].Addr() for i, b := range kad.buckets { - for j, n := range b.nodes { + for j, n := range b { if contains(nodes, n.Addr()) { continue // don't run the check below for nodes in result } @@ -235,12 +235,12 @@ func TestSaveLoad(t *testing.T) { nodes := kad.FindClosest(self, 100) path := "/tmp/bzz.peers" err = kad.Save(path, nil) - if err != nil { + if err != nil && err.Error() != "bucket full" { t.Fatalf("unepected error saving kaddb: %v", err) } kad = New(self, params) err = kad.Load(path, nil) - if err != nil { + if err != nil && err.Error() != "bucket full" { t.Fatalf("unepected error loading kaddb: %v", err) } for _, b := range kad.db.Nodes { @@ -260,30 +260,30 @@ func TestSaveLoad(t *testing.T) { } func (self *Kademlia) proxCheck(t *testing.T) bool { - var sum, i int - var b *bucket - for i, b = range self.buckets { - l := len(b.nodes) + var sum int + for i, b := range self.buckets { + l := len(b) // if we are in the high prox multibucket if i >= self.proxLimit { sum += l } else if l == 0 { - t.Errorf("bucket %d empty, yet proxLimit is %d\n%v", len(b.nodes), self.proxLimit, self) + t.Errorf("bucket %d empty, yet proxLimit is %d\n%v", len(b), self.proxLimit, self) return false } } // check if merged high prox bucket does not exceed size if sum > 0 { - // if sum > self.ProxBinSize { - // t.Errorf("bucket %d is empty, yet proxSize is %d\n%v", i, self.proxSize, self) - // return false - // } if sum != self.proxSize { t.Errorf("proxSize incorrect, expected %v, got %v", sum, self.proxSize) return false } - if self.proxLimit > 0 && sum+len(self.buckets[self.proxLimit-1].nodes) < self.ProxBinSize { - t.Errorf("proxBinSize incorrect, expected %v got %v", sum, self.proxSize) + last := len(self.buckets[self.proxLimit]) + if last > 0 && sum >= self.ProxBinSize+last { + t.Errorf("proxLimit %v incorrect, redundant non-empty bucket %d added to proxBin with %v (target %v)", self.proxLimit, last, sum-last, self.ProxBinSize) + return false + } + if self.proxLimit > 0 && sum < self.ProxBinSize { + t.Errorf("proxLimit %v incorrect. proxSize %v is less than target %v, yet there is more peers", self.proxLimit, sum, self.ProxBinSize) return false } } diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index e4e247d458..7ba7755e34 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -181,7 +181,8 @@ func run(requestDb *storage.LDBDatabase, depo StorageHandler, backend bind.Backe // the main forever loop that handles incoming requests for { if self.hive.blockRead { - time.Sleep(1 * time.Second) + glog.V(logger.Warn).Infof("[BZZ] Cannot read network") + time.Sleep(100 * time.Millisecond) continue } err = self.handle() @@ -225,7 +226,7 @@ func (self *bzz) handle() error { if err := msg.Decode(&req); err != nil { return self.protoError(ErrDecode, "<- %v: %v", msg, err) } - glog.V(logger.Debug).Infof("[BZZ] incoming store request: %s", req.String()) + glog.V(logger.Detail).Infof("[BZZ] incoming store request: %s", req.String()) // swap accounting is done within forwarding self.storage.HandleStoreRequestMsg(&req, &peer{bzz: self}) diff --git a/swarm/network/syncer.go b/swarm/network/syncer.go index 3bbb1ed7d3..1d94ee4331 100644 --- a/swarm/network/syncer.go +++ b/swarm/network/syncer.go @@ -464,11 +464,11 @@ LOOP: // send the unsynced keys stateCopy := *state err := self.unsyncedKeys(unsynced, &stateCopy) - self.state = state - glog.V(logger.Debug).Infof("[BZZ] syncer[%v]: --> %v keys sent: (total: %v (%v), history: %v), sent sync state: %v", self.key.Log(), len(unsynced), keyCounts, keyCount, historyCnt, stateCopy) if err != nil { glog.V(logger.Warn).Infof("[BZZ] syncer[%v]: unable to send unsynced keys: %v", err) } + self.state = state + glog.V(logger.Debug).Infof("[BZZ] syncer[%v]: --> %v keys sent: (total: %v (%v), history: %v), sent sync state: %v", self.key.Log(), len(unsynced), keyCounts, keyCount, historyCnt, stateCopy) unsynced = nil keys = nil } @@ -585,7 +585,7 @@ func (self *syncer) syncDeliveries() { total++ msg, err = self.newStoreRequestMsgData(req) if err != nil { - glog.V(logger.Warn).Infof("[BZZ] syncer[%v]: failed to deliver %v: %v", self.key.Log(), req, err) + glog.V(logger.Warn).Infof("[BZZ] syncer[%v]: failed to create store request for %v: %v", self.key.Log(), req, err) } else { err = self.store(msg) if err != nil { diff --git a/swarm/test/connections/00.sh b/swarm/test/connections/00.sh index 7f4c4445e9..b080d51cfe 100644 --- a/swarm/test/connections/00.sh +++ b/swarm/test/connections/00.sh @@ -1,33 +1,25 @@ #!/bin/bash -dir=`dirname $0` -source $dir/../../cmd/swarm/test.sh - swarm init 4 echo "expect each node to have 3 peers" -cmd="'net.peerCount'" +cmd="net.peerCount" sleep 5 -swarm attach 00 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" -swarm attach 01 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" -swarm attach 02 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" -swarm attach 03 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" +swarm execute 00 "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" +swarm execute 01 "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" +swarm execute 02 "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" +swarm execute 03 "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" swarm stop all -echo "after static nodes is deleted, connections are recovered from kaddb in bzz-peers.json" -# echo rm -rf $DATA_ROOT/enodes\* -# echo rm -rf $DATA_ROOT/data/\*/static-nodes.json -rm -rf $DATA_ROOT/enodes* -rm -rf $DATA_ROOT/data/*/static-nodes.json +echo "connections are recovered from kaddb in bzz-peers.json" swarm cluster 4 echo "expect each node to have 3 peers" -cmd="'net.peerCount'" -sleep 10 -swarm attach 00 --exec "$cmd" |tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" -swarm attach 01 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" -swarm attach 02 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" -swarm attach 03 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" +sleep 5 +swarm execute 00 "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" +swarm execute 01 "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" +swarm execute 02 "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" +swarm execute 03 "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL" swarm stop all diff --git a/swarm/test/swap/00.sh b/swarm/test/swap/00.sh index 0cb0b05a4d..81006a8d6f 100644 --- a/swarm/test/swap/00.sh +++ b/swarm/test/swap/00.sh @@ -4,7 +4,7 @@ echo " two nodes that do not sync and do not have any funds" echo " cannot retrieve content from each other" dir=`dirname $0` -source $dir/../../cmd/swarm/test.sh +source $dir/../test.sh FILE_00=/tmp/1K.0 randomfile 1 > $FILE_00 diff --git a/swarm/test/swap/01.sh b/swarm/test/swap/01.sh index 362f0da73a..d9d50fa2cc 100644 --- a/swarm/test/swap/01.sh +++ b/swarm/test/swap/01.sh @@ -3,19 +3,20 @@ echo " two nodes that do not sync but have enough funds" echo " can retrieve content from each other" dir=`dirname $0` -source $dir/../../cmd/swarm/test.sh +source $dir/..s/test.sh file=/tmp/test.file mininginterval=120 key=/tmp/key -logargs="--verbosity=0 --vmodule='swarm/*=6'" -# logargs='--verbosity=6' +# logargs="--verbosity=0 --vmodule='swarm/*=6'" +logargs='--verbosity=6' +# swarm init 2 --mine --bzznosync --bzznoswap=false $logargsc +# echo "Mining some ether..." +# sleep $mininginterval -swarm init 2 --mine --bzznosync $logargs +swarm cluster 2 --mine --bzznosync --bzznoswap=false $logargsc -echo "Mining some ether..." -sleep $mininginterval randomfile 10 > $file swarm up 00 $file|tail -n1 > $key diff --git a/swarm/test/syncing/00.sh b/swarm/test/syncing/00.sh index a793fea916..3785ba1e2f 100644 --- a/swarm/test/syncing/00.sh +++ b/swarm/test/syncing/00.sh @@ -4,7 +4,7 @@ echo " two nodes that sync (no swap and do not have any funds)" echo " can be in sync content with each other" dir=`dirname $0` -source $dir/../../cmd/swarm/test.sh +source $dir/../test.sh mkdir -p /tmp/swarm-test-files FILE_00=/tmp/swarm-test-files/00 @@ -28,7 +28,6 @@ swarm needs 00 $key $FILE_00 swarm needs 01 $key $FILE_00 swarm stop 01 -# exit 1; swarm up 00 $FILE_01|tail -n1 > $key swarm needs 00 $key $FILE_01 @@ -46,7 +45,8 @@ swarm needs 00 $key $FILE_03 swarm stop 00 swarm up 01 $FILE_04|tail -n1 > $key swarm needs 01 $key $FILE_04 -swarm start 00 #--bzznoswap +swarm start 00 +sleep $wait swarm needs 00 $key $FILE_04 swarm stop all diff --git a/swarm/test/syncing/01.sh b/swarm/test/syncing/01.sh index 4e152dcd9f..bd090d391a 100644 --- a/swarm/test/syncing/01.sh +++ b/swarm/test/syncing/01.sh @@ -4,7 +4,7 @@ echo " two nodes that do not have any funds" echo " can still sync content with each other" dir=`dirname $0` -source $dir/../../cmd/swarm/test.sh +source $dir/../test.sh key=/tmp/key long=/tmp/10M diff --git a/swarm/test/syncing/02.sh b/swarm/test/syncing/02.sh index 48760d6e97..2b0d23443c 100644 --- a/swarm/test/syncing/02.sh +++ b/swarm/test/syncing/02.sh @@ -5,26 +5,26 @@ echo " two nodes that sync (no swap and do not have any funds)" echo " can sync content with each other even with intermittent network connection" dir=`dirname $0` -source $dir/../../cmd/swarm/test.sh +source $dir/../test.sh long=/tmp/10M key=/tmp/key -randomfile 10000 > $long +randomfile 100000 > $long ls -l $long -swarm init 2 -sleep $wait +swarm init 2 --vmodule='swarm/*=5' swarm up 00 $long |tail -n1 > $key & -sleep $wait -swarm attach 01 -exec "'bzz.blockNetworkRead(true)'" -sleep $wait -swarm attach 01 -exec "'bzz.blockNetworkRead(false)'" -sleep $wait -swarm attach 01 -exec "'bzz.blockNetworkRead(true)'" -sleep $wait +sleep 1 +swarm execute 01 'bzz.blockNetworkRead(true)' +sleep 3 +swarm execute 01 'bzz.blockNetworkRead(false)' +# sleep $wait +# swarm attach 01 -exec "'bzz.blockNetworkRead(true)'" +# sleep $wait swarm stop 01 -swarm start 01 -swarm needs 01 $key $long - +# swarm start 01 +# sleep $wait +# swarm needs 01 $key $long +# sleep 3 swarm stop all \ No newline at end of file diff --git a/swarm/test/test.sh b/swarm/test/test.sh new file mode 100644 index 0000000000..01703452a5 --- /dev/null +++ b/swarm/test/test.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +TEST_DIR=`dirname $0` +TEST_NAME=`basename $0 .sh` +TEST_TYPE=`basename $TEST_DIR` +export IP_ADDR="[::]" + + +export SWARM_NETWORK_ID=322$TEST_NAME +export SWARM_DIR=~/bzz/test/$TEST_TYPE + +rm -rf $SWARM_DIR/$SWARM_NETWORK_ID + +wait=1 + + + +function randomfile { + dd if=/dev/urandom of=/dev/stdout bs=1024 count=$1 2>/dev/null +} \ No newline at end of file From 9b0065e05a367b7ff707253de713ce37a9d922cc Mon Sep 17 00:00:00 2001 From: zelig Date: Tue, 5 Jul 2016 04:15:54 +0200 Subject: [PATCH 21/24] swarm/cmd: improve README --- swarm/cmd/README.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/swarm/cmd/README.md b/swarm/cmd/README.md index a28ea84a7b..6be77fba37 100644 --- a/swarm/cmd/README.md +++ b/swarm/cmd/README.md @@ -125,6 +125,8 @@ wget -O- gateway-url/bzz:/swarm-url # Further examples ```shell +# start with updaing +swarm update chambers # display CLI options given to geth used to launch swarm instance 02 swarm options 02 @@ -154,6 +156,15 @@ swarm remote-run nodes.lst 'swarm enode all' > enodes.lst for node in `cat nodes.lst|grep -v '^#'`; do scp enodes.lst $node:; done swarm remote-run nodes.lst 'swarm addpeers enodes.lst' +# if you run a local network and your nodes do not listen to external IPs +swarm remote-run pivot.lst 'swarm restart all' + +# to add just one or a few guardians and let the network bootstrap +# swarm remote-run 'swarm enode all' +swarm addpeers pivot.lst +# or directly +swarm addpeers all <(IP_ADDR='[::]' swarm enode 01|tr -d '"') + # stop all running instances on the node swarm stop all @@ -180,7 +191,7 @@ swarm netstatun swarm remote-run nodes.lst 'swarm netstatconf cicada-sworm; swarm netstatrun' -swarm remote-run-all nodes.lst 'swarm netstatconf cicada-sworm; swarm netstatrun' +swarm remote nodes.lst 'swarm netstatconf cicada-sworm; swarm netstatrun' ``` @@ -214,5 +225,5 @@ npm install -g pm2 ## configure and run netstats client for each node ```shell -swarm remote-run-all nodes.lst 'swarm netstatconf cicada-sworm; swarm netstatrun' +swarm remote-run nodes.lst 'swarm netstatconf cicada-sworm; swarm netstatrun' ``` From 3a7f86d61f9825fa7bd634104d56d78fd80a67b4 Mon Sep 17 00:00:00 2001 From: zelig Date: Tue, 5 Jul 2016 04:19:02 +0200 Subject: [PATCH 22/24] swarm/api: fs uploader fixes --- swarm/api/filesystem.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/swarm/api/filesystem.go b/swarm/api/filesystem.go index cc8fd2c818..ac09602bc6 100644 --- a/swarm/api/filesystem.go +++ b/swarm/api/filesystem.go @@ -86,23 +86,26 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) { errors := make([]error, cnt) done := make(chan bool, maxParallelFiles) dcnt := 0 - wg := &sync.WaitGroup{} + awg := &sync.WaitGroup{} for i, entry := range list { if i >= dcnt+maxParallelFiles { <-done dcnt++ } + awg.Add(1) go func(i int, entry *manifestTrieEntry, done chan bool) { f, err := os.Open(entry.Path) if err == nil { stat, _ := f.Stat() var hash storage.Key + wg := &sync.WaitGroup{} hash, err = self.api.dpa.Store(f, stat.Size(), wg) if hash != nil { list[i].Hash = hash.String() } wg.Wait() + awg.Done() if err == nil { first512 := make([]byte, 512) fread, _ := f.ReadAt(first512, 0) @@ -150,7 +153,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) { if err2 == nil { hs = trie.hash.String() } - wg.Wait() + awg.Wait() return hs, err2 } From 65537422ef1edaebe8af8195dfb1ccfe8efa9b36 Mon Sep 17 00:00:00 2001 From: zelig Date: Fri, 8 Jul 2016 02:14:34 +0200 Subject: [PATCH 23/24] swarm/cmd: improvements * add latency loggingo to upload and download * add checkdownload and checkaccess subcommands * add mem/cpu/disk-info commands * include randomfile cmd from test * TODO: simplify tests using new checks --- swarm/cmd/swarm/swarm | 103 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 99 insertions(+), 4 deletions(-) diff --git a/swarm/cmd/swarm/swarm b/swarm/cmd/swarm/swarm index 6f78060a6f..d9f5b099db 100755 --- a/swarm/cmd/swarm/swarm +++ b/swarm/cmd/swarm/swarm @@ -13,7 +13,8 @@ if [ "$SWARM_NETWORK_ID" = "" ]; then export SWARM_NETWORK_ID=322; fi if [ "$SWARM_DIR" = "" ]; then export SWARM_DIR=$HOME/bzz; fi if [ "$IP_ADDR" = "" ]; then - export IP_ADDR=`curl ipecho.net/plain 2>/dev/null;echo ` + # export IP_ADDR=`curl ipecho.net/plain 2>/dev/null;echo f` + export IP_ADDR= fi root=$SWARM_DIR @@ -25,6 +26,10 @@ dir="$root/$network_id" tmpdir=/tmp +function randomfile { + dd if=/dev/urandom of=/dev/stdout bs=1024 count=$1 2>/dev/null +} + # swarm attach 00 brings up a console attached to a running instance function attach { id=$1 @@ -428,7 +433,7 @@ function needs { function up { #port, file echo "Upload file '$2' to node $1... " 1>&2 file=`basename $2` - execute $1 "bzz.upload(\"$2\", \"$file\")"|tail -n1> /tmp/key + /usr/bin/time -f "latency: %e" swarm execute $1 "bzz.upload(\"$2\", \"$file\")"|tail -n1> /tmp/key cat /tmp/key } @@ -565,7 +570,8 @@ function remote-run { remotes=$1 shift if `echo "$remotes" | grep -qil @`; then - ssh $remotes '. $HOME/bin/env.sh;' "$*" + ip=`echo "$remotes"|cut -d@ -f2` + ssh $remotes "export IP_ADDR=$ip;" '. $HOME/bin/env.sh;' "$*" else for remote in `cat $remotes|grep -v '^#'`; do echo "running on $remote..."; remote-run $remote "$*"; done fi @@ -578,6 +584,84 @@ function update { (cd $GETH_DIR && git remote update && git reset --hard $branch) } +function checksum { + tar -cf - $1 | md5sum|awk '{print $1}' +} + +function checkaccess { + nodes=$1 + target=`basename $2` + chsum=`md5sum $2|cut -f1 -d' '` + master=`head -1 $nodes` + echo "uploading target on $master (md5sum $chsum, size: `du -b -d0 $2|cut -f1`)" + scp $2 $master:$target + hash=`swarm remote-run $master "swarm up 00 $target $file"|tr -d '"'` + remote-run $nodes swarm checkdownload all $hash $chsum +} + +function checkdownload { + id=$1 + if [ "$id" = "all" ]; then + shift + N=`ls -1 -d $dir/data/* |wc -l` + for ((i=0;i /dev/null + echo + if [ -f $target ]; then + cmp --silent $target $tmpdir/$file/* && echo PASS || echo FAIL + elif [ -r $target ]; then + diff -r $target $tmpdir/$file/ >/dev/null && echo PASS || echo FAIL + else + exp=`md5sum $tmpdir/$file/*|cut -f1 -d' '` + if [ "$exp" = "$target" ]; then + echo -n PASS + else + echo FAIL "$exp = $target" + fi + fi + echo " latency: " `cat $tmpdir/$file.log` + fi +} + +function meminfo { + pid="$dir/pids/$1.pid" + if [ -f "$pid" ]; then + # cd /proc/`cat "$pid"` && cat status + ps aux|awk -v PID=`cat $pid` '$2 == PID {print $5 "Kb (" $4 "%)" }' + fi +} + +function cpuinfo { + pid="$dir/pids/$1.pid" + if [ -f "$pid" ]; then + # cd /proc/`cat "$pid"` && cat status + ps aux|awk -v PID=`cat $pid` '$2 == PID {print $3 "%" }' + fi +} + +function diskusage { + if [ "$1" == "" ]; then + echo "DISK USAGE:" `df -m |grep '/$'|awk '{print $(NF-2) "Mb (" $(NF-1) ")"} '` + else + du -m -d0 $*|cut -f1 + fi +} + +function diskinfo { + echo "DISK USAGE $1:" + echo "blockchain:" `diskusage $dir/data/$1/chaindata` + echo "chunkstore:" `diskusage $dir/data/$1/bzz/*/chunks` + echo "overall: /" `diskusage` +} case $cmd in "info" ) @@ -656,9 +740,20 @@ case $cmd in options $*;; "rawoptions" ) rawoptions $*;; + "randomfile" ) + randomfile $*;; + "diskinfo" ) + diskinfo $*;; + "meminfo" ) + meminfo $*;; + "cpuinfo" ) + cpuinfo $*;; "setup" ) setup $* ;; "create-account" ) create-account $*;; - + "checkaccess" ) + checkaccess $* ;; + "checkdownload" ) + checkdownload $* ;; esac From ad6f7d5d093a277e5e86bb3388696463053e5cbc Mon Sep 17 00:00:00 2001 From: zelig Date: Fri, 8 Jul 2016 05:13:21 +0200 Subject: [PATCH 24/24] swarm/api, swarm/network/kademlia: change KAD defaults bucketsize 4, minproxbinsize 2 --- swarm/api/config_test.go | 4 ++-- swarm/network/kademlia/kademlia.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/swarm/api/config_test.go b/swarm/api/config_test.go index 96f112f4bd..ad0f6aab5a 100644 --- a/swarm/api/config_test.go +++ b/swarm/api/config_test.go @@ -23,8 +23,8 @@ var ( "CallInterval": 3000000000, "KadDbPath": "` + filepath.Join("TMPDIR", "0d2f62485607cf38d9d795d93682a517661e513e", "bzz-peers.json") + `", "MaxProx": 8, - "ProxBinSize": 4, - "BucketSize": 3, + "ProxBinSize": 2, + "BucketSize": 4, "PurgeInterval": 151200000000000, "InitialRetryInterval": 42000000, "MaxIdleInterval": 4200000000, diff --git a/swarm/network/kademlia/kademlia.go b/swarm/network/kademlia/kademlia.go index 92f24b512b..08cc39aa31 100644 --- a/swarm/network/kademlia/kademlia.go +++ b/swarm/network/kademlia/kademlia.go @@ -12,8 +12,8 @@ import ( ) const ( - bucketSize = 3 - proxBinSize = 4 + bucketSize = 4 + proxBinSize = 2 maxProx = 8 connRetryExp = 2 maxPeers = 100