Merge ad6f7d5d09 into e2853948da

swarm/api/api.go

@@ -43,12 +43,12 @@ func NewApi(dpa *storage.DPA, dns Resolver) (self *Api) {
 }
 
 // DPA reader API
-func (self *Api) Retrieve(key storage.Key) storage.SectionReader {
+func (self *Api) Retrieve(key storage.Key) storage.LazySectionReader {
 	return self.dpa.Retrieve(key)
 }
 
-func (self *Api) Store(data storage.SectionReader, wg *sync.WaitGroup) (key storage.Key, err error) {
+func (self *Api) Store(data io.Reader, size int64, wg *sync.WaitGroup) (key storage.Key, err error) {
-	return self.dpa.Store(data, wg)
+	return self.dpa.Store(data, size, wg)
 }
 
 type ErrResolve error
@@ -105,15 +105,15 @@ func (self *Api) parseAndResolve(uri string, nameresolver bool) (contentHash sto
 
 // Put provides singleton manifest creation on top of dpa store
 func (self *Api) Put(content, contentType string) (string, error) {
-	sr := io.NewSectionReader(strings.NewReader(content), 0, int64(len(content)))
+	r := strings.NewReader(content)
 	wg := &sync.WaitGroup{}
-	key, err := self.dpa.Store(sr, wg)
+	key, err := self.dpa.Store(r, int64(len(content)), wg)
 	if err != nil {
 		return "", err
 	}
 	manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType)
-	sr = io.NewSectionReader(strings.NewReader(manifest), 0, int64(len(manifest)))
+	r = strings.NewReader(manifest)
-	key, err = self.dpa.Store(sr, wg)
+	key, err = self.dpa.Store(r, int64(len(manifest)), wg)
 	if err != nil {
 		return "", err
 	}
@@ -124,11 +124,11 @@ func (self *Api) Put(content, contentType string) (string, error) {
 // Get uses iterative manifest retrieval and prefix matching
 // to resolve path to content using dpa retrieve
 // it returns a section reader, mimeType, status and an error
-func (self *Api) Get(uri string, nameresolver bool) (reader storage.SectionReader, mimeType string, status int, err error) {
+func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionReader, mimeType string, status int, err error) {
 
 	key, _, path, err := self.parseAndResolve(uri, nameresolver)
-
+	quitC := make(chan bool)
-	trie, err := loadManifest(self.dpa, key)
+	trie, err := loadManifest(self.dpa, key, quitC)
 	if err != nil {
 		glog.V(logger.Warn).Infof("[BZZ] Swarm: loadManifestTrie error: %v", err)
 		return
@@ -151,7 +151,8 @@ func (self *Api) Get(uri string, nameresolver bool) (reader storage.SectionReade
 
 func (self *Api) Modify(uri, contentHash, contentType string, nameresolver bool) (newRootHash string, err error) {
 	root, _, path, err := self.parseAndResolve(uri, nameresolver)
-	trie, err := loadManifest(self.dpa, root)
+	quitC := make(chan bool)
+	trie, err := loadManifest(self.dpa, root, quitC)
 	if err != nil {
 		return
 	}
@@ -162,9 +163,9 @@ func (self *Api) Modify(uri, contentHash, contentType string, nameresolver bool)
 			Hash:        contentHash,
 			ContentType: contentType,
 		}
-		trie.addEntry(entry)
+		trie.addEntry(entry, quitC)
 	} else {
-		trie.deleteEntry(path)
+		trie.deleteEntry(path, quitC)
 	}
 
 	err = trie.recalcAndStore()

swarm/api/api_test.go

@@ -1,11 +1,13 @@
 package api
 
 import (
-	// "bytes"
+	"io"
 	"io/ioutil"
 	"os"
 	"testing"
 
+	"github.com/ethereum/go-ethereum/logger"
+	"github.com/ethereum/go-ethereum/logger/glog"
 	"github.com/ethereum/go-ethereum/swarm/storage"
 )
 
@@ -27,7 +29,7 @@ func testApi(t *testing.T, f func(*Api)) {
 }
 
 type testResponse struct {
-	reader storage.SectionReader
+	reader storage.LazySectionReader
 	*Response
 }
 
@@ -51,13 +53,14 @@ func checkResponse(t *testing.T, resp *testResponse, exp *Response) {
 		resp.Content = string(content)
 	}
 	if resp.Content != exp.Content {
-		// if !bytes.Equal(resp.Content, exp.Content) {
+		// if !bytes.Equal(resp.Content, exp.Content)
 		t.Errorf("incorrect content. expected '%s...', got '%s...'", string(exp.Content), string(resp.Content))
 	}
 }
 
 // func expResponse(content []byte, mimeType string, status int) *Response {
 func expResponse(content string, mimeType string, status int) *Response {
+	glog.V(logger.Detail).Infof("expected content (%v): %v ", len(content), content)
 	return &Response{mimeType, status, int64(len(content)), content}
 }
 
@@ -67,7 +70,19 @@ func testGet(t *testing.T, api *Api, bzzhash string) *testResponse {
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
-	return &testResponse{reader, &Response{mimeType, status, reader.Size(), ""}}
+	quitC := make(chan bool)
+	size, err := reader.Size(quitC)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	glog.V(logger.Detail).Infof("reader size: %v ", size)
+	s := make([]byte, size)
+	_, err = reader.Read(s)
+	if err != io.EOF {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	reader.Seek(0, 0)
+	return &testResponse{reader, &Response{mimeType, status, size, string(s)}}
 	// return &testResponse{reader, &Response{mimeType, status, reader.Size(), nil}}
 }
 

swarm/api/config_test.go

@@ -19,16 +19,15 @@ var (
     "CacheCapacity": 5000,
     "Radius": 0,
     "Branches": 128,
-    "Hash": "SHA256",
+    "Hash": "SHA3",
-    "JoinTimeout": 120,
+    "CallInterval": 3000000000,
-    "SplitTimeout": 120,
-    "CallInterval": 10000000000,
     "KadDbPath": "` + filepath.Join("TMPDIR", "0d2f62485607cf38d9d795d93682a517661e513e", "bzz-peers.json") + `",
     "MaxProx": 8,
-    "ProxBinSize": 4,
+    "ProxBinSize": 2,
-    "BucketSize": 3,
+    "BucketSize": 4,
     "PurgeInterval": 151200000000000,
-    "InitialRetryInterval": 4200000000,
+    "InitialRetryInterval": 42000000,
+    "MaxIdleInterval": 4200000000,
     "ConnRetryExp": 2,
     "Swap": {
         "BuyAt": 20000000000,

swarm/api/filesystem.go

@@ -86,27 +86,29 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
 	errors := make([]error, cnt)
 	done := make(chan bool, maxParallelFiles)
 	dcnt := 0
+	awg := &sync.WaitGroup{}
 
 	for i, entry := range list {
 		if i >= dcnt+maxParallelFiles {
 			<-done
 			dcnt++
 		}
+		awg.Add(1)
 		go func(i int, entry *manifestTrieEntry, done chan bool) {
 			f, err := os.Open(entry.Path)
 			if err == nil {
 				stat, _ := f.Stat()
-				sr := io.NewSectionReader(f, 0, stat.Size())
-				wg := &sync.WaitGroup{}
 				var hash storage.Key
-				hash, err = self.api.dpa.Store(sr, wg)
+				wg := &sync.WaitGroup{}
+				hash, err = self.api.dpa.Store(f, stat.Size(), wg)
 				if hash != nil {
 					list[i].Hash = hash.String()
 				}
 				wg.Wait()
+				awg.Done()
 				if err == nil {
 					first512 := make([]byte, 512)
-					fread, _ := sr.ReadAt(first512, 0)
+					fread, _ := f.ReadAt(first512, 0)
 					if fread > 0 {
 						mimeType := http.DetectContentType(first512[:fread])
 						if filepath.Ext(entry.Path) == ".css" {
@@ -129,6 +131,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
 	trie := &manifestTrie{
 		dpa: self.api.dpa,
 	}
+	quitC := make(chan bool)
 	for i, entry := range list {
 		if errors[i] != nil {
 			return "", errors[i]
@@ -140,9 +143,9 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
 				Hash:        entry.Hash,
 				ContentType: entry.ContentType,
 			}
-			trie.addEntry(ientry)
+			trie.addEntry(ientry, quitC)
 		}
-		trie.addEntry(entry)
+		trie.addEntry(entry, quitC)
 	}
 
 	err2 := trie.recalcAndStore()
@@ -150,6 +153,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
 	if err2 == nil {
 		hs = trie.hash.String()
 	}
+	awg.Wait()
 	return hs, err2
 }
 
@@ -170,11 +174,13 @@ func (self *FileSystem) Download(bzzpath, localpath string) error {
 	if err != nil {
 		return err
 	}
-	// if len(path) > 0 {
-	// 	path += "/"
-	// }
 
-	trie, err := loadManifest(self.api.dpa, key)
+	if len(path) > 0 {
+		path += "/"
+	}
+
+	quitC := make(chan bool)
+	trie, err := loadManifest(self.api.dpa, key, quitC)
 	if err != nil {
 		glog.V(logger.Warn).Infof("[BZZ] fs.Download: loadManifestTrie error: %v", err)
 		return err
@@ -186,46 +192,47 @@ func (self *FileSystem) Download(bzzpath, localpath string) error {
 	}
 
 	var list []*downloadListEntry
-	var mde, mderr error
+	var mde error
 
 	prevPath := lpath
-	err = trie.listWithPrefix(path, func(entry *manifestTrieEntry, suffix string) { // TODO: paralellize
+	err = trie.listWithPrefix(path, quitC, func(entry *manifestTrieEntry, suffix string) {
 		glog.V(logger.Detail).Infof("[BZZ] fs.Download: %#v", entry)
 
-		key := common.Hex2Bytes(entry.Hash)
+		key = common.Hex2Bytes(entry.Hash)
 		path := lpath + "/" + suffix
 		dir := filepath.Dir(path)
 		if dir != prevPath {
 			mde = os.MkdirAll(dir, os.ModePerm)
-			if mde != nil {
-				mderr = mde
-			}
 			prevPath = dir
 		}
 		if (mde == nil) && (path != dir+"/") {
 			list = append(list, &downloadListEntry{key: key, path: path})
 		}
 	})
-	if err == nil {
+	if err != nil {
-		err = mderr
+		return err
 	}
 
-	cnt := len(list)
+	wg := sync.WaitGroup{}
-	errors := make([]error, cnt)
+	errC := make(chan error)
 	done := make(chan bool, maxParallelFiles)
-	dcnt := 0
-
 	for i, entry := range list {
-		if i >= dcnt+maxParallelFiles {
+		select {
-			<-done
+		case done <- true:
-			dcnt++
+			wg.Add(1)
+		case <-quitC:
+			return fmt.Errorf("aborted")
 		}
-		go func(i int, entry *downloadListEntry, done chan bool) {
+		go func(i int, entry *downloadListEntry) {
+			defer wg.Done()
 			f, err := os.Create(entry.path) // TODO: path separators
 			if err == nil {
+
 				reader := self.api.dpa.Retrieve(entry.key)
 				writer := bufio.NewWriter(f)
-				_, err = io.CopyN(writer, reader, reader.Size()) // TODO: handle errors
+				size, err := reader.Size(quitC)
+				if err == nil {
+					_, err = io.CopyN(writer, reader, size) // TODO: handle errors
 					err2 := writer.Flush()
 					if err == nil {
 						err = err2
@@ -235,23 +242,26 @@ func (self *FileSystem) Download(bzzpath, localpath string) error {
 						err = err2
 					}
 				}
-
-			errors[i] = err
-			done <- true
-		}(i, entry, done)
 			}
-	for dcnt < cnt {
-		<-done
-		dcnt++
-	}
-
 			if err != nil {
+				select {
+				case errC <- err:
+				case <-quitC:
+				}
+				return
+			}
+			<-done
+		}(i, entry)
+	}
+	go func() {
+		wg.Wait()
+		close(errC)
+	}()
+	select {
+	case err = <-errC:
 		return err
+	case <-quitC:
+		return fmt.Errorf("aborted")
 	}
-	for i, _ := range list {
+
-		if errors[i] != nil {
-			return errors[i]
-		}
-	}
-	return err
 }

swarm/api/filesystem_test.go

@@ -1,10 +1,12 @@
 package api
 
 import (
+	"bytes"
 	"io/ioutil"
 	"os"
 	"path"
 	"runtime"
+	"sync"
 	"testing"
 )
 
@@ -26,9 +28,9 @@ func testFileSystem(t *testing.T, f func(*FileSystem)) {
 }
 
 func readPath(t *testing.T, parts ...string) string {
-	// func readPath(t *testing.T, parts ...string) []byte {
 	file := path.Join(parts...)
 	content, err := ioutil.ReadFile(file)
+
 	if err != nil {
 		t.Fatalf("unexpected error reading '%v': %v", file, err)
 	}
@@ -36,14 +38,12 @@ func readPath(t *testing.T, parts ...string) string {
 }
 
 func TestApiDirUpload0(t *testing.T) {
-	// t.Skip("FIXME")
 	testFileSystem(t, func(fs *FileSystem) {
 		api := fs.api
 		bzzhash, err := fs.Upload(path.Join(testDir, "test0"), "")
 		if err != nil {
 			t.Fatalf("unexpected error: %v", err)
 		}
-
 		content := readPath(t, testDir, "test0", "index.html")
 		resp := testGet(t, api, bzzhash+"/index.html")
 		exp := expResponse(content, "text/html; charset=utf-8", 0)
@@ -54,17 +54,12 @@ func TestApiDirUpload0(t *testing.T) {
 		exp = expResponse(content, "text/css", 0)
 		checkResponse(t, resp, exp)
 
-		content = readPath(t, testDir, "test0", "img", "logo.png")
-		resp = testGet(t, api, bzzhash+"/img/logo.png")
-		exp = expResponse(content, "image/png", 0)
-
 		_, _, _, err = api.Get(bzzhash, true)
 		if err == nil {
 			t.Fatalf("expected error: %v", err)
 		}
 
 		downloadDir := path.Join(testDownloadDir, "test0")
-		os.RemoveAll(downloadDir)
 		defer os.RemoveAll(downloadDir)
 		err = fs.Download(bzzhash, downloadDir)
 		if err != nil {
@@ -77,12 +72,10 @@ func TestApiDirUpload0(t *testing.T) {
 		if bzzhash != newbzzhash {
 			t.Fatalf("download %v reuploaded has incorrect hash, expected %v, got %v", downloadDir, bzzhash, newbzzhash)
 		}
-
 	})
 }
 
 func TestApiDirUploadModify(t *testing.T) {
-	// t.Skip("FIXME")
 	testFileSystem(t, func(fs *FileSystem) {
 		api := fs.api
 		bzzhash, err := fs.Upload(path.Join(testDir, "test0"), "")
@@ -96,12 +89,24 @@ func TestApiDirUploadModify(t *testing.T) {
 			t.Errorf("unexpected error: %v", err)
 			return
 		}
-		bzzhash, err = api.Modify(bzzhash+"/index2.html", "9ea1f60ebd80786d6005f6b256376bdb494a82496cd86fe8c307cdfb23c99e71", "text/html; charset=utf-8", true)
+		index, err := ioutil.ReadFile(path.Join(testDir, "test0", "index.html"))
 		if err != nil {
 			t.Errorf("unexpected error: %v", err)
 			return
 		}
-		bzzhash, err = api.Modify(bzzhash+"/img/logo.png", "9ea1f60ebd80786d6005f6b256376bdb494a82496cd86fe8c307cdfb23c99e71", "text/html; charset=utf-8", true)
+		wg := &sync.WaitGroup{}
+		hash, err := api.Store(bytes.NewReader(index), int64(len(index)), wg)
+		wg.Wait()
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+			return
+		}
+		bzzhash, err = api.Modify(bzzhash+"/index2.html", hash.Hex(), "text/html; charset=utf-8", true)
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+			return
+		}
+		bzzhash, err = api.Modify(bzzhash+"/img/logo.png", hash.Hex(), "text/html; charset=utf-8", true)
 		if err != nil {
 			t.Errorf("unexpected error: %v", err)
 			return

swarm/api/http/server.go

@@ -86,8 +86,10 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 			return
 		}
 	}
+	if len(proto) > 4 {
 		raw = proto[1:5] == "bzzr"
 		nameresolver = proto[1:5] != "bzzi"
+	}
 
 	glog.V(logger.Debug).Infof(
 		"[BZZ] Swarm: %s request over protocol %s '%s' received.",
@@ -96,10 +98,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 
 	switch {
 	case r.Method == "POST" || r.Method == "PUT":
-		key, err := a.Store(io.NewSectionReader(&sequentialReader{
+		key, err := a.Store(r.Body, r.ContentLength, nil)
-			reader: r.Body,
-			ahead:  make(map[int64]chan bool),
-		}, 0, r.ContentLength), nil)
 		if err == nil {
 			glog.V(logger.Debug).Infof("[BZZ] Swarm: Content for %v stored", key.Log())
 		} else {
@@ -165,7 +164,9 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 
 			// retrieving content
 			reader := a.Retrieve(key)
-			glog.V(logger.Debug).Infof("[BZZ] Swarm: Reading %d bytes.", reader.Size())
+			quitC := make(chan bool)
+			size, err := reader.Size(quitC)
+			glog.V(logger.Debug).Infof("[BZZ] Swarm: Reading %d bytes.", size)
 
 			// setting mime type
 			qv := requestURL.Query()
@@ -176,7 +177,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 
 			w.Header().Set("Content-Type", mimeType)
 			http.ServeContent(w, r, uri, forever(), reader)
-			glog.V(logger.Debug).Infof("[BZZ] Swarm: Serve raw content '%s' (%d bytes) as '%s'", uri, reader.Size(), mimeType)
+			glog.V(logger.Debug).Infof("[BZZ] Swarm: Serve raw content '%s' (%d bytes) as '%s'", uri, size, mimeType)
 
 			// retrieve path via manifest
 		} else {
@@ -203,7 +204,9 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 			} else {
 				status = 200
 			}
-			glog.V(logger.Debug).Infof("[BZZ] Swarm: Served '%s' (%d bytes) as '%s' (status code: %v)", uri, reader.Size(), mimeType, status)
+			quitC := make(chan bool)
+			size, err := reader.Size(quitC)
+			glog.V(logger.Debug).Infof("[BZZ] Swarm: Served '%s' (%d bytes) as '%s' (status code: %v)", uri, size, mimeType, status)
 
 			http.ServeContent(w, r, path, forever(), reader)
 

swarm/api/manifest.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io"
 	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -35,24 +34,24 @@ type manifestTrieEntry struct {
 	subtrie     *manifestTrie
 }
 
-func loadManifest(dpa *storage.DPA, hash storage.Key) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
+func loadManifest(dpa *storage.DPA, hash storage.Key, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
 
 	glog.V(logger.Detail).Infof("[BZZ] manifest lookup key: '%v'.", hash.Log())
 	// retrieve manifest via DPA
 	manifestReader := dpa.Retrieve(hash)
-	return readManifest(manifestReader, hash, dpa)
+	return readManifest(manifestReader, hash, dpa, quitC)
 }
 
-func readManifest(manifestReader storage.SectionReader, hash storage.Key, dpa *storage.DPA) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
+func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dpa *storage.DPA, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
 
 	// TODO check size for oversized manifests
-	manifestData := make([]byte, manifestReader.Size())
+	size, err := manifestReader.Size(quitC)
-	var size int
+	manifestData := make([]byte, size)
-	size, err = manifestReader.Read(manifestData)
+	read, err := manifestReader.Read(manifestData)
-	if int64(size) < manifestReader.Size() {
+	if int64(read) < size {
 		glog.V(logger.Detail).Infof("[BZZ] Manifest %v not found.", hash.Log())
 		if err == nil {
-			err = fmt.Errorf("Manifest retrieval cut short: read %v, expect %v", size, manifestReader.Size())
+			err = fmt.Errorf("Manifest retrieval cut short: read %v, expect %v", read, size)
 		}
 		return
 	}
@@ -72,12 +71,12 @@ func readManifest(manifestReader storage.SectionReader, hash storage.Key, dpa *s
 		dpa: dpa,
 	}
 	for _, entry := range man.Entries {
-		trie.addEntry(entry)
+		trie.addEntry(entry, quitC)
 	}
 	return
 }
 
-func (self *manifestTrie) addEntry(entry *manifestTrieEntry) {
+func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
 	self.hash = nil // trie modified, hash needs to be re-calculated on demand
 
 	if len(entry.Path) == 0 {
@@ -98,11 +97,11 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry) {
 	}
 
 	if (oldentry.ContentType == manifestType) && (cpl == len(oldentry.Path)) {
-		if self.loadSubTrie(oldentry) != nil {
+		if self.loadSubTrie(oldentry, quitC) != nil {
 			return
 		}
 		entry.Path = entry.Path[cpl:]
-		oldentry.subtrie.addEntry(entry)
+		oldentry.subtrie.addEntry(entry, quitC)
 		oldentry.Hash = ""
 		return
 	}
@@ -114,8 +113,8 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry) {
 	}
 	entry.Path = entry.Path[cpl:]
 	oldentry.Path = oldentry.Path[cpl:]
-	subtrie.addEntry(entry)
+	subtrie.addEntry(entry, quitC)
-	subtrie.addEntry(oldentry)
+	subtrie.addEntry(oldentry, quitC)
 
 	self.entries[b] = &manifestTrieEntry{
 		Path:        commonPrefix,
@@ -135,7 +134,7 @@ func (self *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) {
 	return
 }
 
-func (self *manifestTrie) deleteEntry(path string) {
+func (self *manifestTrie) deleteEntry(path string, quitC chan bool) {
 	self.hash = nil // trie modified, hash needs to be re-calculated on demand
 
 	if len(path) == 0 {
@@ -155,10 +154,10 @@ func (self *manifestTrie) deleteEntry(path string) {
 
 	epl := len(entry.Path)
 	if (entry.ContentType == manifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) {
-		if self.loadSubTrie(entry) != nil {
+		if self.loadSubTrie(entry, quitC) != nil {
 			return
 		}
-		entry.subtrie.deleteEntry(path[epl:])
+		entry.subtrie.deleteEntry(path[epl:], quitC)
 		entry.Hash = ""
 		// remove subtree if it has less than 2 elements
 		cnt, lastentry := entry.subtrie.getCountLast()
@@ -198,24 +197,24 @@ func (self *manifestTrie) recalcAndStore() error {
 		return err
 	}
 
-	sr := io.NewSectionReader(bytes.NewReader(manifest), 0, int64(len(manifest)))
+	sr := bytes.NewReader(manifest)
 	wg := &sync.WaitGroup{}
-	key, err2 := self.dpa.Store(sr, wg)
+	key, err2 := self.dpa.Store(sr, int64(len(manifest)), wg)
 	wg.Wait()
 	self.hash = key
 	return err2
 }
 
-func (self *manifestTrie) loadSubTrie(entry *manifestTrieEntry) (err error) {
+func (self *manifestTrie) loadSubTrie(entry *manifestTrieEntry, quitC chan bool) (err error) {
 	if entry.subtrie == nil {
 		hash := common.Hex2Bytes(entry.Hash)
-		entry.subtrie, err = loadManifest(self.dpa, hash)
+		entry.subtrie, err = loadManifest(self.dpa, hash, quitC)
 		entry.Hash = "" // might not match, should be recalculated
 	}
 	return
 }
 
-func (self *manifestTrie) listWithPrefixInt(prefix, rp string, cb func(entry *manifestTrieEntry, suffix string)) (err error) {
+func (self *manifestTrie) listWithPrefixInt(prefix, rp string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) error {
 	plen := len(prefix)
 	var start, stop int
 	if plen == 0 {
@@ -227,6 +226,11 @@ func (self *manifestTrie) listWithPrefixInt(prefix, rp string, cb func(entry *ma
 	}
 
 	for i := start; i <= stop; i++ {
+		select {
+		case <-quitC:
+			return fmt.Errorf("aborted")
+		default:
+		}
 		entry := self.entries[i]
 		if entry != nil {
 			epl := len(entry.Path)
@@ -236,11 +240,13 @@ func (self *manifestTrie) listWithPrefixInt(prefix, rp string, cb func(entry *ma
 					l = epl
 				}
 				if prefix[:l] == entry.Path[:l] {
-					sterr := self.loadSubTrie(entry)
+					err := self.loadSubTrie(entry, quitC)
-					if sterr == nil {
+					if err != nil {
-						entry.subtrie.listWithPrefixInt(prefix[l:], rp+entry.Path[l:], cb)
+						return err
-					} else {
+					}
-						err = sterr
+					err = entry.subtrie.listWithPrefixInt(prefix[l:], rp+entry.Path[l:], quitC, cb)
+					if err != nil {
+						return err
 					}
 				}
 			} else {
@@ -250,14 +256,14 @@ func (self *manifestTrie) listWithPrefixInt(prefix, rp string, cb func(entry *ma
 			}
 		}
 	}
-	return
+	return nil
 }
 
-func (self *manifestTrie) listWithPrefix(prefix string, cb func(entry *manifestTrieEntry, suffix string)) (err error) {
+func (self *manifestTrie) listWithPrefix(prefix string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) (err error) {
-	return self.listWithPrefixInt(prefix, "", cb)
+	return self.listWithPrefixInt(prefix, "", quitC, cb)
 }
 
-func (self *manifestTrie) findPrefixOf(path string) (entry *manifestTrieEntry, pos int) {
+func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *manifestTrieEntry, pos int) {
 
 	glog.V(logger.Detail).Infof("[BZZ] findPrefixOf(%s)", path)
 
@@ -275,10 +281,10 @@ func (self *manifestTrie) findPrefixOf(path string) (entry *manifestTrieEntry, p
 	if (len(path) >= epl) && (path[:epl] == entry.Path) {
 		glog.V(logger.Detail).Infof("[BZZ] entry.ContentType = %v", entry.ContentType)
 		if entry.ContentType == manifestType {
-			if self.loadSubTrie(entry) != nil {
+			if self.loadSubTrie(entry, quitC) != nil {
 				return nil, 0
 			}
-			entry, pos = entry.subtrie.findPrefixOf(path[epl:])
+			entry, pos = entry.subtrie.findPrefixOf(path[epl:], quitC)
 			if entry != nil {
 				pos += epl
 			}
@@ -308,6 +314,7 @@ func RegularSlashes(path string) (res string) {
 func (self *manifestTrie) getEntry(spath string) (entry *manifestTrieEntry, fullpath string) {
 	path := RegularSlashes(spath)
 	var pos int
-	entry, pos = self.findPrefixOf(path)
+	quitC := make(chan bool)
+	entry, pos = self.findPrefixOf(path, quitC)
 	return entry, path[:pos]
 }

swarm/api/manifest_test.go

@@ -10,18 +10,19 @@ import (
 	"github.com/ethereum/go-ethereum/swarm/storage"
 )
 
-func manifest(paths ...string) (manifestReader storage.SectionReader) {
+func manifest(paths ...string) (manifestReader storage.LazySectionReader) {
 	var entries []string
 	for _, path := range paths {
 		entry := fmt.Sprintf(`{"path":"%s"}`, path)
 		entries = append(entries, entry)
 	}
 	manifest := fmt.Sprintf(`{"entries":[%s]}`, strings.Join(entries, ","))
-	return io.NewSectionReader(strings.NewReader(manifest), 0, int64(len(manifest)))
+	return &storage.LazyTestSectionReader{io.NewSectionReader(strings.NewReader(manifest), 0, int64(len(manifest)))}
 }
 
 func testGetEntry(t *testing.T, path, match string, paths ...string) *manifestTrie {
-	trie, err := readManifest(manifest(paths...), nil, nil)
+	quitC := make(chan bool)
+	trie, err := readManifest(manifest(paths...), nil, nil, quitC)
 	if err != nil {
 		t.Errorf("unexpected error making manifest: %v", err)
 	}

swarm/api/storage.go

@@ -34,7 +34,11 @@ func (self *Storage) Get(bzzpath string) (*Response, error) {
 	if err != nil {
 		return nil, err
 	}
-	expsize := reader.Size()
+	quitC := make(chan bool)
+	expsize, err := reader.Size(quitC)
+	if err != nil {
+		return nil, err
+	}
 	body := make([]byte, expsize)
 	size, err := reader.Read(body)
 	if int64(size) == expsize {
@@ -43,6 +47,8 @@ func (self *Storage) Get(bzzpath string) (*Response, error) {
 	return &Response{mimeType, status, expsize, string(body[:size])}, err
 }
 
+// Modify(rootHash, path, contentHash, contentType) takes th e manifest trie rooted in rootHash,
+// and merge on  to it. creating an entry w conentType (mime)
 func (self *Storage) Modify(rootHash, path, contentHash, contentType string) (newRootHash string, err error) {
 	return self.api.Modify(rootHash+"/"+path, contentHash, contentType, true)
 }

swarm/cmd/README.md

@@ -0,0 +1,229 @@
+
+# install and setup swarm
+
+swarm is developed on a branch of the ethereum/go-ethereum repo
+at this stage of the project there is no packages or binary distro, you need to a dev environment and compile from source.
+[This document spells out a complete server setup on ubuntu linux](https://gist.github.com/zelig/74eb365752ceaacf15e860fb80eacb3e) including git/ssh/screen config, golang and compilation, node/npm and network monitoring (might contain a few bits that are tangential to swarm).
+
+Assuming you got your setup working, you will use the `swarm` command line tool to control your swarm of instances.
+This command line tool is at the moment geared towards developers and testing.
+It is likely that it will be replaced by two different tools, one for devel/testing and one for end users
+
+
+The command can be used to update the code
+
+```shell
+swarm update upstream/swarm
+```
+
+Then compile with
+
+```shell
+godep go build -v ./cmd/geth
+```
+
+Make sure you have `GOPATH` variable set and also that the `swarm` executable is in your PATH.
+These environment variables are relevant and set to the following defaults.
+Make sure you are happy with them, otherwise change them, in which case best to put these lines in your `~/.profile`.
+
+```
+export GETH_DIR=$GOPATH/src/github.com/ethereum/go-ethereum
+export GETH=$GETH_DIR/geth
+export SWARM_DIR=~/bzz
+export SWARM_NETWORK_ID=322
+```
+
+* `GETH_DIR` points to your git working copy (given `GOPATH` its standardly under `$GOPATH/src/github.com/ethereum/go-ethereum`)
+* `GETH` points to the `geth` executable compiled from the swarm branch. If you have systemwide install or use multiple geths you may need to change this, otherwise it is assumed you compile to the working copy of the repo.
+* `SWARM_DIR` is the root directory for all swarm related stuff: logs, configs, as well as geth datadirs, make sure this dir is on a device with sufficient disk space
+* `SWARM_NETWORK_ID`: this is by default the network id of the swarm testnet. If you run your own swarm, you need to change it, choose a number that is not likely chosen by others to avoid others joining you.
+
+# Deploying and remote control
+
+the swarm command supports remote update and remote control of your instances.
+In our setting we assume you want to run a cluster of potentially remote swarm nodes each running a local cluster of instances
+The only assumption is that you have (passwordless) ssh access set up to your swarm servers.
+Assume `nodes.lst` is a list of nodes in the format of `username@ip` one per line. blank lines and lines commented out with `#` are ignored.
+
+
+This copies the  scripts found in  `swarm/cmd/swarm` on all remote  nodes listed in `nodes.lst`
+
+```
+swarm remote-update-scripts nodes.lst
+```
+
+If you just want to deploy a locally compiled binaries to all your remote nodes, this will fail if the remote instances are running, so make sure you stop them beforehand
+
+```shell
+swarm remote-run nodes.lst swarm stop all
+swarm remote-update-bin nodes.lst
+```
+
+
+
+Once  you deployed the executables to the nodes, you can control them all with one command. For instance the following line initialises a cluster of two test swarm instances on each remote node.
+Watch out, this will wipe your storage and  all swarm related data
+
+```shell
+swarm remote-run nodes.lst swarm init 2
+```
+
+
+To (re) start a particular instance on a specific remote node with alternative options (for instance mining and different logging verbosity), you can just:
+
+```shell
+swarm remote-run cicada@3.3.0.1 'swarm restart 01 --mine --verbosity=0 --vmodule=swarm/*=5'
+```
+
+# Logging
+
+To check logs
+
+```shell
+swarm log 00 # taillog flow
+swarm remote-run cicada@3.3.0.1 swarm log 00
+```
+
+You can view the log with a pager for an instance with
+
+```
+swarm viewlog 00
+```
+
+Logs are preserved and viewable with the above commands even when nodes are offline
+Each new run logs to a different file
+
+To purge logs
+
+```
+swarm cleanlog 01
+```
+
+To remove all logs on all nodes:
+
+```
+swarm remote-run nodes.lst swarm cleanlog all
+```
+
+# upload and dowload
+
+upload and download via a running local instance
+
+```shell
+swarm up 00 /path/to/file/or/directory
+swarm down 01 hash /path/to/destination
+```
+
+upload via remote swarm proxy or public gateway
+
+```shell
+swarm remote-up gateway-url /path/to/file/or/directory
+wget -O- gateway-url/bzz:/swarm-url
+```
+
+
+# Further examples
+
+```shell
+# start with updaing
+swarm update chambers
+
+# display CLI options given to geth used to launch swarm instance 02
+swarm options 02
+
+# restart swarm instance 00 with alternatiev options
+swarm restart 00 --mine --bzznosync --verbosity=0 --vmodule=swarm/*=6
+
+# attach console to a running swarm instance
+swarm attach 00
+
+# execute a command; e.g., start mining on a running instance
+swarm execute 00 'miner.stop(1)'
+
+# display static info about a instance (even if its offline)
+swarm info 00
+
+# displays the enode url of a running instance
+swarm enode 01
+
+# add peers to a running swarm instance
+swarm addpeers 00 "enode://1033c1cada...@3.3.0.1:30301"
+
+# to compile a list of enodes from all instances on all remote nodes:
+swarm remote-run nodes.lst 'swarm enode all' > enodes.lst
+
+# to add all peers to all instances on  each node
+for node in `cat nodes.lst|grep -v '^#'`; do scp enodes.lst $node:; done
+swarm remote-run nodes.lst 'swarm addpeers enodes.lst'
+
+# if you run a local network and  your nodes do not listen to external IPs
+swarm  remote-run pivot.lst 'swarm restart all'
+
+# to add just one or a few guardians and let the network bootstrap
+# swarm remote-run 'swarm enode all'
+swarm addpeers pivot.lst
+# or directly
+swarm addpeers all <(IP_ADDR='[::]' swarm enode 01|tr -d '"')
+
+# stop all running instances on the node
+swarm stop all
+
+# stop all running instances on all remote nodes
+swarm remote-run nodes.lst swarm stop all
+
+# display peer connection table of running instance 00
+swarm hive 00
+
+# display peer connection table for a running instance and continually refresh every 4 seconds
+swarm monitor 00 4
+
+# display peer connection table for all running instance on a remote node and continually refresh every 10 seconds
+swarm monitor cicada@3.3.0.1 all 10
+
+
+# configure eth-net-intelligence-api network monitoring client API for a node (the name argument appears as a prefix for all instances in  your cluster)
+swarm netstatconf cicada-sworm
+
+# restart the net monitor client API
+swarm netstatun
+
+# configure eth-net-intelligence-api network monitoring client API and (re)start the monitor tool on all remote nodes
+swarm remote-run nodes.lst 'swarm netstatconf cicada-sworm; swarm netstatrun'
+
+
+swarm remote nodes.lst 'swarm netstatconf cicada-sworm; swarm netstatrun'
+```
+
+
+see also:
+
+* https://github.com/ethereum/go-ethereum/tree/swarm/swarm/test
+* https://github.com/ethereum/go-ethereum/tree/swarm/swarm/cmd
+
+# ethereum netstats client setup
+
+## install
+
+nodejs and npm are prerequisites
+
+```shell
+# MAC
+brew install node npm
+# ubuntu
+sudo apt-get install npm nodejs
+```
+
+clone the git repo and install:
+
+```
+git clone git@github.com:cubedro/eth-net-intelligence-api.git
+cd eth-net-intelligence-api
+npm install
+npm install -g pm2
+```
+
+## configure and run netstats client for each node
+
+```shell
+swarm remote-run nodes.lst 'swarm netstatconf cicada-sworm; swarm netstatrun'
+```

swarm/cmd/bzzhash/bzzhash.go

@@ -3,7 +3,6 @@ package main
 
 import (
 	"fmt"
-	"io"
 	"os"
 	"runtime"
 
@@ -24,15 +23,11 @@ func main() {
 	}
 
 	stat, _ := f.Stat()
-	sr := io.NewSectionReader(f, 0, stat.Size())
 	chunker := storage.NewTreeChunker(storage.NewChunkerParams())
-	hash := make([]byte, chunker.KeySize())
+	key, err := chunker.Split(f, stat.Size(), nil, nil, nil)
-	errC := chunker.Split(hash, sr, nil, nil)
-	err, ok := <-errC
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "%v\n", err)
-	}
+	} else {
-	if !ok {
+		fmt.Printf("%v\n", key)
-		fmt.Printf("%064x\n", hash)
 	}
 }

swarm/cmd/swarm/env.sh

@@ -0,0 +1,6 @@
+export PATH=$HOME/bin:$PATH
+export GETH_DIR=$HOME/bin
+export SWARM_DIR=
+
+export NVM_DIR=$HOME/.nvm
+[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"  # This loads nvm

swarm/cmd/swarm/gethup.sh

@@ -1,145 +0,0 @@
-#!/bin/bash
-# Usage:
-# bash /path/to/eth-utils/gethup.sh <datadir> <instance_name> <ip_addr>
-
-root=$1  # base directory to use for datadir and logs
-shift
-id=$1  # double digit instance id like 00 01 02
-shift
-ip_addr=$1 # ip address to substitute
-shift
-
-# logs are output to a date-tagged file for each run , while a link is
-# created to the latest, so that monitoring be easier with the same filename
-# TODO: use this if GETH not set
-# GETH=geth
-# echo "ls -l $GETH"
-# ls -l $GETH
-
-# geth CLI params       e.g., (dd=04, run=09)
-datetag=`date "+%c%y%m%d-%H%M%S"|cut -d ' ' -f 5`
-datadir=$root/data/$id        # /tmp/eth/04
-log=$root/log/$id.$datetag.log     # /tmp/eth/04.09.log
-linklog=$root/log/$id.log     # /tmp/eth/04.09.log
-password=$id            # 04
-port=303$id              # 34504
-bzzport=322$id              # 32204
-rpcport=302$id            # 3204
-
-mkdir -p $root/data
-mkdir -p $root/enodes
-mkdir -p $root/pids
-mkdir -p $root/log
-# if we do not have an account, create one
-# will not prompt for password, we use the double digit instance id as passwd
-# NEVER EVER USE THESE ACCOUNTS FOR INTERACTING WITH A LIVE CHAIN
-keystoredir="$datadir/keystore/"
-# echo "KeyStore dir: $keystoredir"
-if [ ! -d "$keystoredir" ]; then
-  # echo "create an account with password $id [DO NOT EVER USE THIS ON LIVE]"
-  # mkdir -p $datadir/keystore
-  $GETH --datadir $datadir --password <(echo -n $id) account new >/dev/null 2>&1
-      # create account with password 00, 01, ...
-  # note that the account key will be stored also separately outside
-  # datadir
-  # this way you can safely clear the data directory and still keep your key
-  # under `<rootdir>/keystore/dd
-  # LS=`ls $datadir/keystore`
-  # echo $LS
-  while [ ! -d "$keystoredir" ]; do
-    echo "."
-    ((i++))
-    if ((i>10)); then break; fi
-    sleep 1
-  done
-  # echo "copying keys $datadir/keystore $root/keystore/$id"
-  mkdir -p $root/keystore/$id
-  cp -R "$datadir/keystore/" $root/keystore/$id
-fi
-
-# # mkdir -p $datadir/keystore
-# if [ ! -d "$datadir/keystore" ]; then
-#   echo "copying keys $root/keystore/$id $datadir/keystore"
-#   cp -R $root/keystore/$id/keystore/ $datadir/keystore/
-# fi
-
-
-# query node's enode url
-if [ $ip_addr="" ]; then
-  pattern='\d+\.\d+\.\d+\.\d+'
-  ip_addr="[::]"
-else
-  pattern='\[\:\:\]'
-fi
-
-geth="$GETH --datadir $datadir --port $port"
-
-# echo -n "enode for instance $id...  "
-if [ ! "$GETH" = "" ] && [ ! -f $root/enodes/$id.enode ]; then
-  cmd="$geth js <(echo 'console.log(admin.nodeInfo.enode); exit();') "
-  # echo $cmd '2>/dev/null |grep enode | perl -pe "s/'$pattern'/'$ip_addr'/g" | perl -pe "s/^/\"/; s/\s*$/\"/;" > '$root/enodes/$id.enode
-  eval $cmd 2>/dev/null |grep enode | perl -pe "s/$pattern/$ip_addr/g" | perl -pe "s/^/\"/; s/\s*\$/\"/;" > $root/enodes/$id.enode
-fi
-# cat  $root/enodes/$id.enode
-echo
-
-# copy cluster enodes list to node's static node list
-# echo "copy cluster enodes list to node's static node list"
-if [  -f $root/enodes.all ]; then
-  cp $root/enodes.all $datadir/static-nodes.json
-fi
-
-if [ ! -f $root/pids/$id.pid ]; then
-  # bring up node `dd` (double digit)
-  # - using <rootdir>/<dd>
-  # - listening on port 303dd, (like 30300, 30301, ...)
-  # - with the account unlocked
-  # - launching json-rpc server on port 81dd (like 8100, 8101, 8102, ...)
-  # echo "BZZKEY=$geth account list|head -n1|perl -ne '/([a-f0-9]{40})/ && print \$1'"
-  BZZKEY=`$geth account list|head -n1|perl -ne '/([a-f0-9]{40})/ && print \$1'`
-  echo -n "starting instance $id ($BZZKEY @ $datadir )..."
-  # echo "$geth \
-  #   --identity=$id \
-  #   --bzzaccount=$BZZKEY --bzzport=$bzzport \
-  #   --unlock=$BZZKEY \
-  #   --password=<(echo -n $id) \
-  #   --rpc --rpcport=$rpcport --rpccorsdomain='*' $* \
-  #   2>&1 | tee "$stablelog" > "$log" &  # comment out if you pipe it to a tty etc.
-  # " >&2
-
-
-  $GETH --datadir=$datadir \
-    --identity=$id \
-    --bzzaccount=$BZZKEY --bzzport=$bzzport \
-    --port=$port \
-    --unlock=$BZZKEY \
-    --password=<(echo -n $id) \
-    --rpc --rpcport=$rpcport --rpccorsdomain='*' $* \
-     > "$log" 2>&1 &  # comment out if you pipe it to a tty etc.
-
-  ln -sf "$log" "$linklog"
-
-     # wait until ready
-  # pid=`ps auxwww|grep geth|grep "ty=$id"|grep -v grep|awk '{print $2}'`
-  # echo "pid: $pid"
-  # ps auxwww|grep geth|grep "ty=$id"|grep -v grep
-  # echo $pid > $root/pids/$id.pid
-  #echo $! > $root/pids/$id.pid
-  while true; do
-    $GETH --exec="net" attach ipc:$datadir/geth.ipc > /dev/null 2>&1 && break
-    sleep 1
-    echo -n "."
-    if ((i++>10)); then
-      echo "instance $id failed to start"
-      exit 1
-    fi
-  done
-  echo -n "started - "
-  pid=`ps auxwww|grep geth|grep "ty=$id"|grep -v grep|awk '{print $2}'`
-  echo "pid: $pid"
-  # ps auxwww|grep geth|grep "ty=$id"|grep -v grep
-  echo $pid > $root/pids/$id.pid
-fi
-
-# to bring up logs, uncomment
-# tail -f $log

swarm/cmd/swarm/swarm

@@ -0,0 +1,759 @@
+#!/bin/bash
+if [ "$GETH_DIR" = "" ]; then
+  if [ "$GOPATH" = "" ]; then echo "either GETH_DIR or GOPATH environment variable must be set"; exit 1; fi
+  export GETH_DIR=$GOPATH/src/github.com/ethereum/go-ethereum
+fi
+
+if [ "$GETH" = "" ]; then
+  export GETH=$GETH_DIR/geth
+fi
+
+if [ "$SWARM_NETWORK_ID" = "" ]; then export SWARM_NETWORK_ID=322; fi
+
+if [ "$SWARM_DIR" = "" ]; then export SWARM_DIR=$HOME/bzz; fi
+
+if [ "$IP_ADDR" = "" ]; then
+  # export IP_ADDR=`curl ipecho.net/plain 2>/dev/null;echo f`
+  export IP_ADDR=
+fi
+
+root=$SWARM_DIR
+network_id=$SWARM_NETWORK_ID
+cmd=$1
+shift
+
+dir="$root/$network_id"
+
+tmpdir=/tmp
+
+function randomfile {
+  dd if=/dev/urandom of=/dev/stdout bs=1024 count=$1 2>/dev/null
+}
+
+# swarm attach 00 brings up a console attached to a running instance
+function attach {
+  id=$1
+  shift
+  echo "attaching console to instance $id"
+  cmd="$GETH --datadir=$dir/data/$id $* attach ipc:$dir/data/$id/geth.ipc"
+  # echo $cmd
+  eval $cmd
+ }
+
+# swarm attach 00 brings up a console attached to a running instance
+function execute {
+  id=$1
+  shift
+  # attach $id --exec "'$*' "
+  cmd="$GETH --datadir=$dir/data/$id --exec '$*' attach ipc:$dir/data/$id/geth.ipc"
+  # echo $cmd
+  eval $cmd
+}
+
+# swarm hive 00 displays the kademlia table of the given running instance
+function hive {
+  if [ "$1" = "all" ]; then
+    N=`ls -1 -d $dir/data/* |wc -l`
+    for ((i=0;i<N;++i)); do
+      instance=`printf "%02d" $i`
+      hive $instance
+    done
+  else
+    # echo "kademlia table of instance $id"
+    execute $1 'console.log(bzz.hive)'|grep -v undefined
+  fi
+}
+
+# swarm log 00 shows the running tail logs of an instance
+function log {
+  id=$1
+  shift
+  echo "streaming logs for instance $id"
+  cmd="tail -f $dir/log/$id.log"
+  echo $cmd
+  eval $cmd
+}
+
+# swarm cleanlog 00 removes the old log files for a given instance (all for every instance)
+function cleanlog {
+  id=$1
+  shift
+  if [ $id = "all" ]; then
+    echo "remove logs for all instances"
+    rm -rf "$dir/log/"
+  else
+    echo "remove logs for instance $id"
+    rm -rf $dir/log/$id*
+  fi
+}
+
+# display kademlia tables of istances
+function monitor {
+  if [ "$3" = "" ]; then
+    id=$1
+    period=$2
+    while true; do
+      hive $id
+      sleep $period
+    done
+  else
+    node=$1
+    id=$2
+    period=$3
+    while true; do
+      remote-run $node swarm hive $id
+      sleep $period
+    done
+  fi
+}
+
+
+# swarm cleanbzz 00 removes the bzz subdirectory for a given instance (all for every instance)
+function cleanbzz {
+  id=$1
+  shift
+  if [ $id = "all" ]; then
+    echo "remove bzz data for all instances"
+    rm -rf $dir/data/*/bzz
+  else
+    echo "remove bzz data for instance $id"
+    rm -rf "$dir/data/$id"
+  fi
+}
+
+# swarm less/viewlogd 00 displays the last (current) log for the given instance (in a pager)
+function viewlog {
+  id=$1
+  shift
+  echo "viewing logs for instance $id"
+  cmd="/usr/bin/less $dir/log/$id.log"
+  echo $cmd
+  eval $cmd
+}
+
+# display the swawrm base account for an instance
+function key {
+  id=$1
+  shift
+  mkdir -p $dir/data/$id/
+  $GETH --datadir=$dir/data/$id account list|head -n1|perl -ne '/([a-f0-9]{40})/ && print $1'
+}
+
+# swarm options 00 displays the geth command line options used to start the swarm
+function rawoptions {
+  id=$1
+  shift
+
+  globaloptions="
+  --dev
+  --maxpeers=40
+  --shh=false
+  --nodiscover
+  --networkid=$network_id
+  --bzznoswap
+  --verbosity=0
+  --vmodule=swarm/*=5"
+
+  datadir=$dir/data/$id
+  password=$id
+  port=303$id
+  bzzport=322$id
+  rpcport=302$id
+  key=`swarm key $id`
+
+  instanceoptions="
+  --datadir=$datadir
+  --identity=$id
+  --bzzaccount=$key
+  --unlock=$key
+  --bzzport=$bzzport
+  --port=$port
+  --rpc
+  --rpcport=$rpcport
+  --rpccorsdomain='*'"
+
+  echo "$globaloptions"
+  echo "$instanceoptions"
+  echo "$*"
+}
+
+function options {
+  echo "The command line options passed to geth are the following:"
+  echo "use 'geth help' to see further options"
+  rawoptions $*
+}
+
+function start {
+  id=$1
+  shift
+
+  if [ -f $dir/pids/$id.pid ]; then
+    echo "instance $id already running"
+    return
+  fi
+
+  datetag=`date "+%Y-%m-%d-%H:%M:%S"`
+  log=$dir/log/$id.$datetag.log
+  linklog=$dir/log/$id.log
+  opts=`swarm rawoptions $id $*|tr '\r' ' '`
+  # echo; echo "$GETH $opts > $log 2>&1 &"
+
+  $GETH $opts  --password=<(echo -n $id) > "$log" 2>&1 &  # comment out if you pipe it to a tty etc.
+  ln -sf "$log" "$linklog"
+
+   # wait until ready
+
+  ((j=0))
+  while true; do
+      execute $id "net" > /dev/null 2>&1 && break
+    sleep 1
+    echo -n "."
+    if ((j++>10)); then
+      echo "instance $id failed to start"
+      exit 1
+    fi
+  done
+  echo -n "started - "
+  pid=`ps auxwww|grep geth|grep "ty=$id"|grep -v grep|awk '{print $2}'`
+  echo "pid: $pid"
+  echo $pid > $dir/pids/$id.pid
+}
+
+
+# setup 00 creates the direcories for instance
+function setup {
+  id=$1
+  shift
+  mkdir -p $dir/data/$id
+  mkdir -p $dir/enodes
+  mkdir -p $dir/pids
+  mkdir -p $dir/log
+}
+
+# creates the swarm base account for an  instance
+function create-account {
+  id=$1
+  datadir=$dir/data/$id
+  # if we do not have an account, create one
+  # will not prompt for password, we use the double digit instance id as passwd
+  # NEVER EVER USE THESE ACCOUNTS FOR INTERACTING WITH A LIVE CHAIN
+  keystoredir="$datadir/keystore/"
+  # echo "KeyStore dir: $keystoredir"
+  if [ ! -d "$keystoredir" ]; then
+    # echo "create an account with password $id [DO NOT EVER USE THIS ON LIVE]"
+    # mkdir -p $datadir/keystore
+    $GETH --datadir=$datadir --password=<(echo -n $id) account new >/dev/null 2>&1
+    # create account with password 00, 01, ...
+    # note that the account key will be stored also separately outside
+    # datadir
+    # this way you can safely clear the data directory and still keep your key
+    # under <rootdir>/keystore/dd
+    # LS=$(ls $datadir/keystore)
+    # echo $LS
+    while [ ! -d "$keystoredir" ]; do
+      echo "."
+      ((i++))
+      if ((i>10)); then break; fi
+      sleep 1
+    done
+    # echo "copying keys $datadir/keystore $root/keystore/$id"
+    mkdir -p $dir/keystore/$id
+    cp -R "$keystoredir" $dir/keystore/$id
+  fi
+}
+
+# shuts down a running instance, cleans the pid
+function stop {
+  id=$1
+  shift
+  if [ $id = "all" ]; then
+    procs=`cat $dir/pids/*.pid 2>/dev/null |perl -pe 's/^\s+//;s/\s+\\$//;s/\s+/\n/g'`
+    # echo "stopping processes $procs"
+    for p in $procs; do
+      shutdown $p
+    done
+    rm -rf $dir/pids/*
+  else
+    pid=$dir/pids/$id.pid
+    if [ -f $pid ]; then
+      echo "stopping instance $id, pid="`cat $pid`
+      shutdown `cat $pid`
+      rm $pid
+    fi
+  fi
+}
+
+# shutdown kills the node with interrupt 2 - if it resits falls back to -9 after 10s
+function shutdown {
+  echo -n "stopping $1..."
+  kill -2 $1
+  while true; do
+    ps auxwww|grep geth|grep -v grep|awk '{print $2}'|grep -ql $1 || break
+    if ((i++>5)); then
+      echo "not stopping. killing it"
+      kill -QUIT $1
+      break
+    fi
+    echo '.'
+    sleep 1
+  done
+  echo "stopped"
+}
+
+# swarm restart 00 calls stop and start
+function restart {
+  id=$1
+  shift
+  if [ $id = "all" ]; then
+    stop all
+    N=`ls -d1 $dir/data/*|wc -l`
+    cluster $N $*
+  else
+    stop $id
+    start $id $*
+  fi
+}
+
+# swarm init X sets up and starts a new client instance
+##########################################################
+#
+# IT WIPES THE DATABASE
+#
+##########################################################
+function init {
+  killall geth
+  reset all
+  cluster $*
+  enode all
+  connect all
+}
+
+# reset wipes the datadirs of the instance
+function reset {
+  id=$1
+  shift
+  if [ $id = "all" ]; then
+    rm -rf $dir
+  else
+    rm -rf$dir/*/$id*
+  fi
+}
+
+# enode displays the instance's enode address
+# swarm enode all writes all instances' enodes in a file
+function enode {
+  id=$1
+  shift
+
+  if [ $id = "all" ]; then
+    json=$dir/static-nodes.json
+    enodes=$dir/enodes.lst
+    cmd=$dir/connect.js
+    rm -f $enodes $json $cmd
+    # build a static nodes(-like) list of all enodes of the local cluster
+    echo "[" >> $json
+    N=`ls -1 -d $dir/data/* |wc -l`
+    for ((i=0;i<N;++i)); do
+      id=`printf "%02d" $i`
+      f=$dir/enodes/$id.enode
+      enode $id > $f
+      echo -n "admin.addPeer(" >> $cmd
+      cat "$f"  |  perl -pe 's/\s*$//' >> $cmd
+      echo ");" >> $cmd
+      cat $f |perl -pe 's/"//g'>> $enodes
+      cat $f >> $json
+      echo "," >> $json
+    done
+    echo "\"\"]" >> $json
+    cat $enodes
+  else
+    # echo "local IP: $ip_addr "
+    execute $id 'admin.nodeInfo.enode' |perl -pe "s/\[\:\:\]/$IP_ADDR/ "
+  fi
+}
+
+# connect sources the local or remote set of peers and connects the node to the peers
+function connect {
+  id=$1
+  shift
+  if [ $id = "all" ]; then
+    N=`ls -1 -d $dir/data/* |wc -l`
+    for ((i=0;i<N;++i)); do
+      id=`printf "%02d" $i`
+      connect $id
+    done
+  else
+    echo -n 'peer count: '
+    attach $id --preload $dir/connect.js --exec net.peerCount
+  fi
+}
+
+# swarm cluster N launches N nodes; 00 01 02 ...
+function cluster {
+  N=$1
+  shift
+  echo "launching cluster of $N instances"
+  for ((i=0;i<N;++i)); do
+    id=`printf "%02d" $i`
+    setup $id $*
+    create-account $id
+    echo "launching node $i/$N.."
+    start $id $*
+    # info $id
+    enode $id
+  done
+}
+
+# swarm needs instance keyfile destination
+# tests if the content for the key is available for the intance
+#
+function needs {
+  id=$1
+  keyfile=$2
+
+  target=$3
+  dest=$tmpdir/down
+  mkdir -p $dest
+  file=$dest/`basename $target`
+  rm -f $file
+  echo -n "waiting for root hash in '$keyfile'..."
+  while true; do
+   if [ ! -z $keyfile ]; then
+    break
+   fi
+   sleep 1
+   echo -n "."
+  done
+  key=`cat $keyfile|tr -d \"`
+  echo " => $key"
+  download $id $key $dest && cmp --silent $file $target && echo "PASS" || echo "FAIL"
+}
+
+# swarm up 00 file uploads file via instances CLI
+function up { #port, file
+  echo "Upload file '$2' to node $1... " 1>&2
+  file=`basename $2`
+  /usr/bin/time -f "latency: %e" swarm execute $1 "bzz.upload(\"$2\", \"$file\")"|tail -n1> /tmp/key
+  cat /tmp/key
+}
+
+# swarm download 00 file download file via instances CLI
+function download {
+  echo "download '$2' from node $1 to '$3'"
+  execute $1 "bzz.download(\"$2\", \"$3\")" >/dev/null
+}
+
+# swarm down issues bzz.get to download the content 10 attempts
+function down {
+  echo -n "Download hash '$2' from node $1... "
+  while true; do
+    execute $1 "bzz.get(\"$2\")" 2> /dev/null |grep -qil "status" && break
+    sleep 1
+    echo -n "."
+    if ((i++>10)); then
+      echo "not found"
+      return
+    fi
+  done
+  echo "found OK"
+}
+
+# static info about an instance (available even if node is off)
+function info {
+  echo "swarm node information"
+  echo "ROOTDIR: $root"
+  echo "DATADIR: $dir/data/$1"
+  echo "LOGFILE: $dir/log/$1.log"
+  echo "HTTPAPI: http://localhost:322$1"
+  echo "ETHPORT: 303$1"
+  echo "RPCPORT: 302$1"
+  echo "ACCOUNT:" 0x`ls -1 $dir/data/$1/bzz`
+  echo "CHEQUEB:" `cat $dir/data/$1/bzz/*/config.json|grep Contract|awk -F\" '{print $4}'`
+  echo "ROOTDIR: $root"
+  echo "DATADIR: $dir/data/$1"
+  echo "LOGFILE: $dir/log/$1.log"
+}
+
+# live into about an instance
+function status {
+  echo -n "account balance:     "
+  execute $1 'eth.getBalance(eth.accounts[0])'
+  echo -n "swap contract balance:       "
+  execute $1 "eth.getBalance(bzz.info.Swap.Contract)"
+  echo -n "chequebook balance:  "
+  execute $1 "chequebook.balance"
+  echo -n "peer count:          "
+  execute $1 'net.peerCount'
+  echo -n "latest block number: "
+  execute $1 "eth.blockNumber"
+}
+
+# display peers for an instance
+function peers {
+  execute $1 'admin.peers'
+}
+
+# add peers into an instance (connection not guaranteed)
+function addpeers {
+  id=$1
+  peers=$2
+  if [ $id = "all" ]; then
+    N=`ls -1 -d $dir/data/* |wc -l`
+    for ((i=0;i<N;++i)); do
+      id=`printf "%02d" $i`
+      addpeers $id $peers
+    done
+  else
+    echo "addpeer to instance $id"
+    for peer in `cat $peers|grep -v '^#'`; do
+      execute  $id "admin.addPeer(\"$peer\")"
+    done
+  fi
+}
+
+# configures the eth-net-intelligence-api network monitor
+function netstatconf {
+  group=$1
+  N=`ls -1 -d $dir/data/* |wc -l`
+  ip=`curl ipecho.net/plain 2>/dev/null;echo `
+  ws_server="ws://146.185.130.117:3000"
+  ws_secret=BZZ322
+  conf="$dir/$group-$ip.netstat.json"
+
+  echo "writing netstat conf for cluster $group-$ip ($N instances) -> $conf"
+
+  echo -e "[" > $conf
+
+  for ((i=0;i<$N;++i)); do
+    id=`printf "%02d" $i`
+    single_template="  {\n    \"name\"        : \"$group-$ip-$i\",\n    \"cwd\"         : \".\",\n    \"script\"      : \"app.js\",\n    \"log_date_format\"   : \"YYYY-MM-DD HH:mm Z\",\n    \"merge_logs\"    : false,\n    \"watch\"       : false,\n    \"exec_interpreter\"  : \"node\",\n    \"exec_mode\"     : \"fork_mode\",\n    \"env\":\n    {\n      \"NODE_ENV\"    : \"production\",\n      \"RPC_HOST\"    : \"localhost\",\n      \"RPC_PORT\"    : \"302$id\",\n      \"INSTANCE_NAME\"   : \"$group-$ip-$i\",\n      \"WS_SERVER\"     : \"$ws_server\",\n      \"WS_SECRET\"     : \"$ws_secret\",\n    }\n  }"
+
+    endline=""
+    if ((i<$N-1)); then
+    # if [ "$i" -ne "$N" ]; then
+        endline=","
+    fi
+    echo -e "$single_template$endline" >> $conf
+  done
+  echo "]" >> $conf
+}
+
+# (re)starts the eth-net-intelligence-api network monitor
+function netstatrun {
+  cd $GETH_DIR/../eth-net-intelligence-api
+  pm2 kill
+  pm2 start $dir/*.netstat.json
+}
+
+# kills the eth-net-intelligence-api network monitor
+function netstatkill {
+  cd $GETH_DIR/../eth-net-intelligence-api
+  pm2 kill
+}
+
+# copies the swarm control script to the remote node(s)
+function remote-update-scripts {
+  scriptdir=$GETH_DIR/swarm/cmd/swarm/
+  remotes=$1
+  for remote in `cat $remotes|grep -v '^#'`; do echo "updating scripts on $remote..."; ssh $remote mkdir -p bin && scp -r $scriptdir/* $remote:bin/; done
+}
+
+# copies the geth executable to the  remote nodes
+function remote-update-bin {
+  remotes=$1
+  # remote-update-scripts $remotes
+  for remote in `cat $remotes|grep -v '^#'`; do  echo "updating binary on $remote..."; scp -r $GETH_DIR/geth $remote:bin/; done
+}
+
+# runs a command on remote node or nodes from a file
+function remote-run {
+  remotes=$1
+  shift
+  if `echo "$remotes" | grep -qil @`; then
+    ip=`echo "$remotes"|cut -d@ -f2`
+    ssh $remotes "export IP_ADDR=$ip;" '. $HOME/bin/env.sh;' "$*"
+  else
+    for remote in `cat $remotes|grep -v '^#'`; do echo "running on $remote..."; remote-run $remote "$*"; done
+  fi
+}
+
+# updates the code from the given branch
+function update {
+  branch=$1
+  echo "cd $GETH_DIR &&  git remote update && git reset --hard $branch"
+  (cd $GETH_DIR &&  git remote update && git reset --hard $branch)
+}
+
+function checksum {
+  tar -cf - $1 | md5sum|awk '{print $1}'
+}
+
+function checkaccess {
+  nodes=$1
+  target=`basename $2`
+  chsum=`md5sum $2|cut -f1 -d' '`
+  master=`head  -1 $nodes`
+  echo "uploading target on $master (md5sum $chsum, size: `du -b -d0 $2|cut -f1`)"
+  scp $2 $master:$target
+  hash=`swarm remote-run $master "swarm up 00 $target $file"|tr -d '"'`
+  remote-run $nodes swarm checkdownload all $hash $chsum
+}
+
+function checkdownload {
+  id=$1
+  if [ "$id" = "all"  ]; then
+    shift
+    N=`ls -1 -d $dir/data/* |wc -l`
+    for ((i=0;i<N;++i)); do
+      id=`printf "%02d" $i`
+      checkdownload $id $*
+    done
+  else
+    hash=$2
+    target=$3
+    datetag=`date "+%Y-%m-%d-%H:%M:%S"`
+    file="swarm-$datetag"
+    rm -rf $tmpdir/$file
+    /usr/bin/time -o $tmpdir/$file.log -f "%e" swarm download $id $hash $tmpdir/$file > /dev/null
+    echo
+    if [ -f $target ]; then
+      cmp --silent  $target $tmpdir/$file/* && echo PASS || echo FAIL
+    elif [ -r $target ]; then
+      diff -r $target $tmpdir/$file/ >/dev/null && echo PASS || echo FAIL
+    else
+      exp=`md5sum $tmpdir/$file/*|cut -f1 -d' '`
+      if [ "$exp" = "$target" ]; then
+        echo -n  PASS
+      else
+        echo FAIL "$exp = $target"
+      fi
+    fi
+    echo " latency: " `cat $tmpdir/$file.log`
+  fi
+}
+
+function meminfo {
+  pid="$dir/pids/$1.pid"
+  if [ -f "$pid" ]; then
+    # cd  /proc/`cat "$pid"` && cat status
+    ps aux|awk -v PID=`cat $pid` '$2 == PID {print $5 "Kb (" $4 "%)" }'
+  fi
+}
+
+function cpuinfo {
+  pid="$dir/pids/$1.pid"
+  if [ -f "$pid" ]; then
+    # cd  /proc/`cat "$pid"` && cat status
+   ps aux|awk -v PID=`cat $pid`  '$2  == PID {print $3 "%" }'
+ fi
+}
+
+function diskusage {
+  if [ "$1" == "" ]; then
+    echo "DISK USAGE:" `df -m |grep '/$'|awk '{print $(NF-2) "Mb (" $(NF-1) ")"} '`
+  else
+    du -m -d0 $*|cut -f1
+  fi
+}
+
+function diskinfo {
+  echo "DISK USAGE $1:"
+  echo "blockchain:" `diskusage $dir/data/$1/chaindata`
+  echo "chunkstore:" `diskusage $dir/data/$1/bzz/*/chunks`
+  echo "overall:  /"  `diskusage`
+}
+
+case $cmd in
+  "info" )
+    info $*;;
+  "enode" )
+    enode $*;;
+  "status" )
+    status $*;;
+  "peers" )
+    peers $*;;
+  "addpeers" )
+    addpeers $*;;
+  "clean" )
+    clean $*;;
+  "needs" )
+    needs $*;;
+  "up" )
+    up $*;;
+  "key" )
+    key $*;;
+  "down" )
+    down $*;;
+  "download" )
+    download $*;;
+  "init" )
+    init $*;;
+  "exec" )
+    execute $*;;
+  "hive" )
+    hive $*;;
+  "start" )
+    start $*;;
+  "stop" )
+    stop $* ;;
+  "restart" )
+    restart $*;;
+  "reset" )
+    reset $*;;
+  "cluster" )
+    cluster $*;;
+  "attach" )
+    attach $*;;
+  "execute" )
+    execute $*;;
+  "exec" )
+    execute $*;;
+  "cleanbzz" )
+    cleanbzz $*;;
+  "cleanlog" )
+    cleanlog $*;;
+  "log" )
+    log $*;;
+  "viewlog" )
+    viewlog $*;;
+  "less" )
+    viewlog $*;;
+  "connect" )
+    connect $*;;
+  "monitor" )
+    monitor $*;;
+  "remote-update-scripts" )
+    remote-update-scripts $*;;
+  "remote-update-bin" )
+    remote-update-bin $*;;
+  "update-src" )
+    update-src $*;;
+  "remote-run" )
+    remote-run $*;;
+  "netstatconf" )
+    netstatconf  $*;;
+  "netstatkill" )
+    netstatkill  $*;;
+  "netstatrun" )
+    netstatrun  $*;;
+  "options" )
+    options $*;;
+  "rawoptions" )
+    rawoptions $*;;
+  "randomfile" )
+    randomfile $*;;
+  "diskinfo" )
+    diskinfo $*;;
+  "meminfo" )
+    meminfo $*;;
+  "cpuinfo" )
+    cpuinfo $*;;
+  "setup" )
+    setup $* ;;
+  "create-account" )
+    create-account $*;;
+  "checkaccess" )
+    checkaccess $* ;;
+  "checkdownload" )
+    checkdownload $* ;;
+esac

swarm/cmd/swarm/swarm.sh

@@ -1,360 +0,0 @@
-# !/bin/bash
-# bash cluster <root> <network_id> <number_of_nodes>  <runid> <local_IP> [[params]...]
-# https://github.com/ethereum/go-ethereum/wiki/Setting-up-monitoring-on-local-cluster
-
-# sets up a local ethereum network cluster of nodes
-# - <number_of_nodes> is the number of nodes in cluster
-# - <root> is the root directory for the cluster, the nodes are set up
-#   with datadir `<root>/<network_id>/00`, `<root>/ <network_id>/01`, ...
-# - new accounts are created for each node
-# - they launch on port 30300, 30301, ...
-# - they star rpc on port 8100, 8101, ...
-# - by collecting the nodes nodeUrl, they get connected to each other
-# - if enode has no IP, `<local_IP>` is substituted
-# - if `<network_id>` is not 0, they will not connect to a default client,
-#   resulting in a private isolated network
-# - the nodes log into `<root>/<network_id>/00.<runid>.log`, `<root>/<network_id>/01.<runid>.log`, ...
-# - The nodes launch in mining mode
-# - the cluster can be killed with `killall geth` (FIXME: should record PIDs)
-#   and restarted from the same state
-# - if you want to interact with the nodes, use rpc
-# - you can supply additional params on the command line which will be passed
-#   to each node, for instance `-mine`
-
-if [ "$GETH" = "" ]; then
-  echo "env var GETH not set "
-  exit 1
-fi
-
-srcdir=`dirname $0`
-
-root=$1
-shift
-network_id=$1
-shift
-cmd=$1
-shift
-# ip_addr=`curl ipecho.net/plain 2>/dev/null;echo `
-
-# echo "external IP: $ip_addr"
-swarmoptions='--dev --maxpeers=20 --shh=false --nodiscover'
-tmpdir=/tmp
-
-function attach {
-  id=$1
-  shift
-  echo "attaching console to instance $id"
-  cmd="$GETH  $* attach ipc:$root/$network_id/data/$id/geth.ipc"
-  # echo $cmd
-  eval $cmd
-}
-
-function log {
-  id=$1
-  shift
-  echo "streaming logs for instance $id"
-  cmd="tail -f $root/$network_id/log/$id.log"
-  echo $cmd
-  eval $cmd
-}
-
-function less {
-  id=$1
-  shift
-  echo "viewing logs for instance $id"
-  cmd="/usr/bin/less $root/$network_id/log/$id.log"
-  echo $cmd
-  eval $cmd
-}
-
-function start {
-  id=$1
-  shift
-  # echo -n "starting instance $id - "
-  cmd="bash $srcdir/gethup.sh $root/$network_id/ $id '$ip_addr' --networkid=$network_id $swarmoptions $*"
-  # echo "pid="`cat $root/$network_id/pids/$id.pid`
-  # echo $cmd
-  eval $cmd
-}
-
-function stop {
-  id=$1
-  shift
-  if [ $id = "all" ]; then
-    procs=`cat $root/$network_id/pids/*.pid 2>/dev/null |perl -pe 's/^\s+//;s/\s+\\$//;s/\s+/\n/g'`
-    # echo "stopping processes $procs"
-    for p in $procs; do
-      shutdown $p
-    done
-    rm -rf $root/$network_id/pids/*
-  else
-    pid=$root/$network_id/pids/$id.pid
-    if [ -f $pid ]; then
-      echo "stopping instance $id, pid="`cat $pid`
-      shutdown `cat $pid`
-      rm $pid
-    fi
-  fi
-  # ps auxwww|grep geth|grep bzz|grep -v grep
-}
-
-function shutdown {
-  echo -n "stopping $1..."
-  kill -2 $1
-  while true   ;do
-    ps auxwww|grep geth|grep -v grep|awk '{print $2}'|grep -ql $1 || break
-    sleep 1
-  done
-  echo "stopped"
-}
-
-function restart {
-  id=$1
-  shift
-  stop $id
-  start $id $*
-}
-
-function init {
-  stop all
-  killall geth
-  reset all
-  cluster $*
-  enode all
-  connect all
-}
-
-function reset {
-  id=$1
-  shift
-  if [ $id = "all" ]; then
-    rm -rf $root/$network_id
-  else
-    rm -rf$root/$network_id/*/$id*
-  fi
-
-}
-
-function enode {
-  dir=$root/$network_id
-  id=$1
-  shift
-  if [ $id = "all" ]; then
-    N=`ls -1 $dir/enodes/|wc -l`
-    enodes=$dir/enodes.all
-    rm -f $enodes
-    # build a static nodes(-like) list of all enodes of the local cluster
-    echo "[" >> $enodes
-    for ((i=0;i<N;++i)); do
-      id=`printf "%02d" $i`
-      enode=$dir/enodes/$id.enode
-      enode $id
-      if [ -f "$enode" ] && [ ! -z "$enode" ]; then
-        cat "$enode" >> $enodes
-        echo "," >> $enodes
-      fi
-    done
-    echo "\"\"]" >> $enodes
-    cmd=$dir/connect.js
-    for ((i=0;i<N;++i)); do
-      id=`printf "%02d" $i`
-      enode=$dir/enodes/$id.enode
-      if [ -f "$enode" ] && [ ! -z "$enode" ]; then
-        echo -n "admin.addPeer(" >> $cmd
-        cat "$enode" >> $cmd
-        echo ");" >> $cmd
-      fi
-    done
-  else
-    enode=$dir/enodes/$id.enode
-    attach $id --exec "'console.log(admin.nodeInfo.enode)'" |head -2 |tail -1| perl -pe 's/^/"/;s/$/"/'|perl -pe 's/\s*$//' > $enode
-    # cat $enode
-  fi
-
-}
-
-function connect {
-  dir=$root/$network_id
-  id=$1
-  shift
-  if [ $id = "all" ]; then
-    for ((i=0;i<N;++i)); do
-      id=`printf "%02d" $i`
-      connect $id
-    done
-  else
-    cmd="$GETH --preload $dir/connect.js --exec '\"admin.peers\"' attach ipc:$root/$network_id/data/$id/geth.ipc $dir/connect.js"
-    # echo $cmd
-    eval $cmd
-    cat $dir/connect.js
-  fi
-}
-
-function cluster {
-  N=$1
-  shift
-  echo "launching cluster of $N instances"
-  # cmd="bash $srcdir/gethcluster.sh $root $network_id $N '' $swarmoptions $*"
-  # echo $cmd
-  # eval $cmd
-  dir=$root/$network_id
-  mkdir -p $dir/data
-  mkdir -p $dir/enodes
-  mkdir -p $dir/pids
-  mkdir -p $dir/log
-
-  for ((i=0;i<N;++i)); do
-    id=`printf "%02d" $i`
-    mkdir -p $dir/data/$id
-    echo "launching node $i/$N ---> tail -f $dir/log/$id.log"
-    start $id $vmodule $*
-  done
-}
-
-
-function needs {
-  id=$1
-  keyfile=$2
-  target=$3
-  dir=`dirname $3`
-  dest=$tmpdir/down
-  mkdir -p $dest
-  file=$dest/`basename $target`
-  rm -f $file
-  echo -n "waiting for root hash in '$keyfile'..."
-  while true; do
-   if [ -f $keyfile ] && [ ! -z $keyfile ]; then
-    break
-   fi
-   sleep 1
-   echo -n "."
-  done
-  key=`cat $keyfile|tr -d \"`
-  echo " => $key"
-  download $id $key $dest && cmp --silent $file $target && echo "PASS" || echo "FAIL"
-  # && ls -l $keyfile $file $target
-}
-
-
-function up { #port, file
-  echo "Upload file '$2' to node $1... " 1>&2
-  file=`basename $2`
-  attach $1 "--exec 'bzz.upload(\"$2\", \"$file\")'"|tail -n1> /tmp/key
-  # key=`bash swarm/cmd/bzzup.sh $2 86$1`
-  cat /tmp/key
-}
-
-function download {
-  echo "download '$2' from node $1 to '$3'"
-  # echo attach $1 "--exec 'bzz.download(\"$2\", \"$3\")'"
-  attach $1 "--exec 'bzz.download(\"$2\", \"$3\")'" > /dev/null
-}
-
-
-function down {
-  echo -n "Download hash '$2' from node $1... "
-  # echo "wget -O- http://localhost:86$1/$2 > /dev/null 2>&1 && echo 'got it' || echo 'not found'"
-  # wget -O- http://localhost:86$1/$2 > /dev/null 2>&1 && echo "got it" || echo "not found"
-  while true; do
-    attach $1 "--exec 'bzz.get(\"$2\")'" 2> /dev/null |grep -qil "status" && break
-    sleep 1
-    echo -n "."
-    if ((i++>10)); then
-      echo "not found"
-      return
-    fi
-  done
-  echo "found OK"
-}
-
-function clean { #index
-  echo "Clean up for $1"
-  rm -rf $root/$network_id/data/$1/{bzz/*/chunks,bzz/*/requests/,bzz/*/bzz-peers.json,chaindata,nodes}
-}
-
-function info {
-  echo "swarm node information"
-  echo "ROOTDIR: $root"
-  echo "DATADIR: $root/$network_id/data/$1"
-  echo "LOGFILE: $root/$network_id/log/$1.log"
-  echo "HTTPAPI: http://localhost:322$1"
-  echo "ETHPORT: 303$1"
-  echo "RPCPORT: 302$1"
-  echo "ACCOUNT:" 0x`ls -1 $root/$network_id/data/$1/bzz`
-  echo "CHEQUEB:" `cat $root/$network_id/data/$1/bzz/*/config.json|grep Contract|awk -F\" '{print $4}'`
-  echo "ROOTDIR: $root"
-  echo "DATADIR: $root/$network_id/data/$1"
-  echo "LOGFILE: $root/$network_id/log/$1.log"
-}
-
-
-function status {
-  attach 00 -exec "'console.log(eth.getBalance(eth.accounts[0])); console.log(eth.getBalance(bzz.info().Swap.Contract)); console.log(chequebook.balance)'"
-}
-
-function netstatconf {
-  begin=$1
-  N=$2
-  name_prefix=$3
-  ws_server=$4
-  ws_secret=$5
-  conf="$root/$network_id/$name_prefix.netstat.json"
-
-  echo "writing netstat conf for cluster $name_prefix to $conf"
-
-  echo -e "[" > $conf
-
-  for ((i=$begin;i<$start+$N;++i)); do
-      id=`printf "%02d" $i`
-      single_template="  {\n    \"name\"        : \"$name_prefix-$i\",\n    \"cwd\"         : \".\",\n    \"script\"      : \"app.js\",\n    \"log_date_format\"   : \"YYYY-MM-DD HH:mm Z\",\n    \"merge_logs\"    : false,\n    \"watch\"       : false,\n    \"exec_interpreter\"  : \"node\",\n    \"exec_mode\"     : \"fork_mode\",\n    \"env\":\n    {\n      \"NODE_ENV\"    : \"production\",\n      \"RPC_HOST\"    : \"localhost\",\n      \"RPC_PORT\"    : \"302$id\",\n      \"INSTANCE_NAME\"   : \"$name_prefix-$i\",\n      \"WS_SERVER\"     : \"$ws_server\",\n      \"WS_SECRET\"     : \"$ws_secret\",\n    }\n  }"
-
-      endline=""
-      if (($i<$N-1)); then
-      # if [ "$i" -ne "$N" ]; then
-          endline=","
-      fi
-      echo -e "$single_template$endline" >> $conf
-  done
-
-  echo "]" >> $conf
-}
-
-case $cmd in
-  "info" )
-    info $*;;
-  "enode" )
-    enode $*;;
-  "connect" )
-    connect $*;;
-  "status" )
-    status $*;;
-  "clean" )
-    clean $*;;
-  "needs" )
-    needs $*;;
-  "up" )
-    up $*;;
-  "down" )
-    down $*;;
-  "init" )
-    init $*;;
-  "start" )
-    start $*;;
-  "stop" )
-    stop $* ;;
-  "restart" )
-    restart $*;;
-  "reset" )
-    reset $*;;
-  "cluster" )
-    cluster $*;;
-  "attach" )
-    attach $*;;
-  "log" )
-    log $*;;
-  "less" )
-    less $*;;
-    "netstatconf" )
-    netstatconf  $*;;
-
-esac

swarm/cmd/swarm/test.sh

@@ -1,28 +0,0 @@
-#!/bin/bash
-
-TEST_DIR=`dirname $0`
-TEST_NAME=`basename $0 .sh`
-TEST_TYPE=`basename $TEST_DIR`
-
-
-export SWARM_BIN=$TEST_DIR/../../cmd/swarm
-export GETH=$SWARM_BIN/../../../geth
-export NETWORKID=322$TEST_NAME
-export TMPDIR=~/BZZ/test/$TEST_TYPE
-export DATA_ROOT=$TMPDIR/$NETWORKID
-# alias swarm='bash $SWARM_BIN/swarm.sh $DATA_ROOT $NETWORKID'
-EXTRA_ARGS=$*
-
-rm -rf $DATA_ROOT
-
-wait=1
-
-function swarm {
-  # echo bash $SWARM_BIN/swarm.sh $TMPDIR $NETWORKID  $* $EXTRA_ARGS
-  bash $SWARM_BIN/swarm.sh $TMPDIR $NETWORKID $* $EXTRA_ARGS
-}
-
-
-function randomfile {
-  dd if=/dev/urandom of=/dev/stdout bs=1024 count=$1 2>/dev/null
-}

swarm/network/hive.go

@@ -30,6 +30,7 @@ type Hive struct {
 	addr         kademlia.Address
 	kad          *kademlia.Kademlia
 	path         string
+	quit         chan bool
 	toggle       chan bool
 	more         chan bool
 
@@ -106,6 +107,7 @@ func (self *Hive) Addr() kademlia.Address {
 func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPeer func(string) error) (err error) {
 	self.toggle = make(chan bool)
 	self.more = make(chan bool)
+	self.quit = make(chan bool)
 	self.id = id
 	self.listenAddr = listenAddr
 	err = self.kad.Load(self.path, nil)
@@ -123,15 +125,15 @@ func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPee
 				// to attempt to write to more (remove Peer when shutting down)
 				return
 			}
-			node, need, proxLimit := self.kad.FindBest()
+			node, need, proxLimit := self.kad.Suggest()
 
-			glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: select candidate peer")
 			if node != nil && len(node.Url) > 0 {
-				glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: call for bee %v", node.Url)
+				glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: call known bee %v", node.Url)
 				// enode or any lower level connection address is unnecessary in future
 				// discovery table is used to look it up.
 				connectPeer(node.Url)
-			} else if need {
+			}
+			if need {
 				// a random peer is taken from the table
 				peers := self.kad.FindClosest(kademlia.RandomAddressAt(self.addr, rand.Intn(self.kad.MaxProx)), 1)
 				if len(peers) > 0 {
@@ -140,16 +142,19 @@ func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPee
 					req := &retrieveRequestMsgData{
 						Key: storage.Key(randAddr[:]),
 					}
-					glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: call any bee in area %v messenger bee %v", randAddr, peers[0])
+					glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: call any bee near %v (PO%03d) - messenger bee: %v", randAddr, proxLimit, peers[0])
 					peers[0].(*peer).retrieve(req)
 				} else {
-					glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: no peer")
+					glog.V(logger.Warn).Infof("[BZZ] KΛÐΞMLIΛ hive: no peer")
 				}
-				self.toggle <- true
+				glog.V(logger.Detail).Infof("[BZZ] KΛÐΞMLIΛ hive: buzz kept alive")
-				glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: buzz kept alive")
 			} else {
-				glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: no need for more bees")
+				glog.V(logger.Info).Infof("[BZZ] KΛÐΞMLIΛ hive: no need for more bees")
-				self.toggle <- false
+			}
+			select {
+			case self.toggle <- need:
+			case <-self.quit:
+				return
 			}
 			glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: queen's address: %v, population: %d (%d)", self.addr, self.kad.Count(), self.kad.DBCount())
 		}
@@ -174,11 +179,7 @@ func (self *Hive) keepAlive() {
 				default:
 				}
 			}
-		case need, alive := <-self.toggle:
+		case need := <-self.toggle:
-			if !alive {
-				self.more <- false
-				return
-			}
 			if alarm == nil && need {
 				alarm = time.NewTicker(time.Duration(self.callInterval)).C
 			}
@@ -186,20 +187,31 @@ func (self *Hive) keepAlive() {
 				alarm = nil
 
 			}
+		case <-self.quit:
+			return
 		}
 	}
 }
 
 func (self *Hive) Stop() error {
 	// closing toggle channel quits the updateloop
-	close(self.toggle)
+	close(self.quit)
 	return self.kad.Save(self.path, saveSync)
 }
 
 // called at the end of a successful protocol handshake
-func (self *Hive) addPeer(p *peer) {
+func (self *Hive) addPeer(p *peer) error {
+	defer func() {
+		select {
+		case self.more <- true:
+		default:
+		}
+	}()
 	glog.V(logger.Detail).Infof("[BZZ] KΛÐΞMLIΛ hive: hi new bee %v", p)
-	self.kad.On(p, loadSync)
+	err := self.kad.On(p, loadSync)
+	if err != nil {
+		return err
+	}
 	// self lookup (can be encoded as nil/zero key since peers addr known) + no id ()
 	// the most common way of saying hi in bzz is initiation of gossip
 	// let me know about anyone new from my hood , here is the storageradius
@@ -207,10 +219,8 @@ func (self *Hive) addPeer(p *peer) {
 	// we do not record as request or forward it, just reply with peers
 	p.retrieve(&retrieveRequestMsgData{})
 	glog.V(logger.Detail).Infof("[BZZ] KΛÐΞMLIΛ hive: 'whatsup wheresdaparty' sent to %v", p)
-	select {
+
-	case self.more <- true:
+	return nil
-	default:
-	}
 }
 
 // called after peer disconnected
@@ -342,7 +352,7 @@ func (self *Hive) peers(req *retrieveRequestMsgData) {
 			for _, peer := range self.getPeers(key, int(req.MaxPeers)) {
 				addrs = append(addrs, peer.remoteAddr)
 			}
-			glog.V(logger.Debug).Infof("[BZZ] Hive sending %d peer addresses to %v. req.Id: %v, req.Key: %x", len(addrs), req.from, req.Id, req.Key.Log())
+			glog.V(logger.Debug).Infof("[BZZ] Hive sending %d peer addresses to %v. req.Id: %v, req.Key: %v", len(addrs), req.from, req.Id, req.Key.Log())
 
 			peersData := &peersMsgData{
 				Peers: addrs,

swarm/network/kademlia/kaddb.go

@@ -26,7 +26,6 @@ type NodeRecord struct {
 	Meta  *json.RawMessage // arbitrary metadata saved for a peer
 
 	node Node
-	connected bool
 }
 
 func (self *NodeRecord) setSeen() {
@@ -45,7 +44,7 @@ type KadDb struct {
 	Nodes                [][]*NodeRecord
 	index                map[Address]*NodeRecord
 	cursors              []int
-	lock                 sync.Mutex
+	lock                 sync.RWMutex
 	purgeInterval        time.Duration
 	initialRetryInterval time.Duration
 	connRetryExp         int
@@ -161,10 +160,10 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe
 
 	var interval time.Duration
 	var found bool
-	var count int
+	var purge []bool
-	var purge []int
 	var delta time.Duration
 	var cursor int
+	var count int
 	var after time.Time
 
 	// iterate over columns maximum bucketsize times
@@ -181,7 +180,7 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe
 				proxLimit = po
 				need = true
 			}
-			cursor = self.cursors[po]
+			purge = make([]bool, len(dbrow))
 
 			// there is a missing slot - finding a node to connect to
 			// select a node record from the relavant kaddb row (of identical prox order)
@@ -191,7 +190,7 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe
 				node = dbrow[cursor]
 
 				// skip already connected nodes
-				if node.connected {
+				if node.node != nil {
 					glog.V(logger.Debug).Infof("[KΛÐ]: kaddb record %v (PO%03d:%d/%d) already connected", node.Addr, po, cursor, len(dbrow))
 					continue ROW
 				}
@@ -208,7 +207,7 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe
 				}
 				if delta > self.purgeInterval {
 					// remove node
-					purge = append(purge, cursor)
+					purge[cursor] = true
 					glog.V(logger.Debug).Infof("[KΛÐ]: kaddb record %v (PO%03d:%d) unreachable since %v. Removed", node.Addr, po, cursor, node.Seen)
 					continue ROW
 				}
@@ -222,41 +221,38 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe
 				glog.V(logger.Debug).Infof("[KΛÐ]: kaddb record %v (PO%03d:%d) selected as candidate connection %v. seen at %v (%v ago), selectable since %v, retry after %v (in %v)", node.Addr, po, cursor, rounds, node.Seen, delta, node.After, after, interval)
 				node.After = after
 				found = true
-				break ROW
 			} // ROW
 			self.cursors[po] = cursor
-			self.delete(po, purge...)
+			self.delete(po, purge)
 			if found {
-				return node, true, proxLimit
+				return node, need, proxLimit
 			}
 		} // ROUND
-		if need {
-			return nil, true, proxLimit
-		}
 	} // ROUNDS
 
-	return nil, false, proxLimit
+	return nil, need, proxLimit
 }
 
 // deletes the noderecords of a kaddb row corresponding to the indexes
 // caller must hold the dblock
 // the call is unsafe, no index checks
-func (self *KadDb) delete(row int, indexes ...int) {
+func (self *KadDb) delete(row int, purge []bool) {
-	var prev int
 	var nodes []*NodeRecord
 	dbrow := self.Nodes[row]
-	for _, next := range indexes {
+	for i, del := range purge {
-		// need to adjust dbcursor
+		if i == self.cursors[row] {
-		if next > 0 {
+			//reset cursor
-			if next <= self.cursors[row] {
+			self.cursors[row] = len(nodes)
-				self.cursors[row]--
 		}
-			nodes = append(nodes, dbrow[prev:next]...)
+		// delete the entry to be purged
+		if del {
+			delete(self.index, dbrow[i].Addr)
+			continue
 		}
-		prev = next + 1
+		// otherwise append to new list
-		delete(self.index, dbrow[next].Addr)
+		nodes = append(nodes, dbrow[i])
 	}
-	self.Nodes[row] = append(nodes, dbrow[prev:]...)
+	self.Nodes[row] = nodes
 }
 
 // save persists kaddb on disk (written to file on path in json format.
@@ -306,14 +302,15 @@ func (self *KadDb) load(path string, cb func(*NodeRecord, Node) error) (err erro
 		return
 	}
 	var n int
-	var purge []int
+	var purge []bool
 	for po, b := range self.Nodes {
+		purge = make([]bool, len(b))
 	ROW:
 		for i, node := range b {
 			if cb != nil {
 				err = cb(node, node.node)
 				if err != nil {
-					purge = append(purge, i)
+					purge[i] = true
 					continue ROW
 				}
 			}
@@ -323,7 +320,7 @@ func (self *KadDb) load(path string, cb func(*NodeRecord, Node) error) (err erro
 			}
 			self.index[node.Addr] = node
 		}
-		self.delete(po, purge...)
+		self.delete(po, purge)
 	}
 	glog.V(logger.Info).Infof("[KΛÐ] loaded kaddb with %v nodes from %v", n, path)
 

swarm/network/kademlia/kademlia.go

@@ -12,16 +12,17 @@ import (
 )
 
 const (
-	bucketSize   = 3
+	bucketSize   = 4
-	proxBinSize  = 4
+	proxBinSize  = 2
 	maxProx      = 8
 	connRetryExp = 2
+	maxPeers     = 100
 )
 
 var (
 	purgeInterval        = 42 * time.Hour
-	initialRetryInterval = 42 * 100 * time.Millisecond
+	initialRetryInterval = 42 * time.Millisecond
-	maxIdleInterval      = 42 * 10 * time.Second
+	maxIdleInterval      = 42 * 100 * time.Millisecond
 )
 
 type KadParams struct {
@@ -31,6 +32,7 @@ type KadParams struct {
 	BucketSize           int
 	PurgeInterval        time.Duration
 	InitialRetryInterval time.Duration
+	MaxIdleInterval      time.Duration
 	ConnRetryExp         int
 }
 
@@ -41,6 +43,7 @@ func NewKadParams() *KadParams {
 		BucketSize:           bucketSize,
 		PurgeInterval:        purgeInterval,
 		InitialRetryInterval: initialRetryInterval,
+		MaxIdleInterval:      maxIdleInterval,
 		ConnRetryExp:         connRetryExp,
 	}
 }
@@ -52,7 +55,7 @@ type Kademlia struct {
 	proxLimit  int          // state, the PO of the first row of the most proximate bin
 	proxSize   int          // state, the number of peers in the most proximate bin
 	count      int          // number of active peers (w live connection)
-	buckets    []*bucket    // the actual bins
+	buckets    [][]Node     // the actual bins
 	db         *KadDb       // kaddb, node record database
 	lock       sync.RWMutex // mutex to access buckets
 }
@@ -68,14 +71,9 @@ type Node interface {
 // add is the base address of the table
 // params is KadParams configuration
 func New(addr Address, params *KadParams) *Kademlia {
-	buckets := make([]*bucket, params.MaxProx+1)
+	buckets := make([][]Node, params.MaxProx+1)
-	for i, _ := range buckets {
+	glog.V(logger.Info).Infof("[KΛÐ] base address %v", addr.String()[:6])
-		buckets[i] = &bucket{size: params.BucketSize} // will initialise bucket{int(0),[]Node(nil),sync.Mutex}
-	}
-	glog.V(logger.Info).Infof("[KΛÐ] base address %v", addr)
 
-	// ! temporary hack fixme:
-	params.ProxBinSize = 4
 	return &Kademlia{
 		addr:      addr,
 		KadParams: params,
@@ -109,9 +107,6 @@ func (self *Kademlia) On(node Node, cb func(*NodeRecord, Node) error) (err error
 
 	index := self.proximityBin(node.Addr())
 	record := self.db.findOrCreate(index, node.Addr(), node.Url())
-	// callback on add node
-	// setting the node on the record, set it checked (for connectivity)
-	record.node = node
 
 	if cb != nil {
 		err = cb(record, node)
@@ -119,33 +114,46 @@ func (self *Kademlia) On(node Node, cb func(*NodeRecord, Node) error) (err error
 		if err != nil {
 			return fmt.Errorf("unable to add node %v, callback error: %v", node.Addr(), err)
 		}
-		glog.V(logger.Info).Infof("[KΛÐ]: add node record %v with node %v", record, node)
+		glog.V(logger.Debug).Infof("[KΛÐ]: add node record %v with node %v", record, node)
 	}
-	record.connected = true
 
 	// insert in kademlia table of active nodes
 	bucket := self.buckets[index]
 	// if bucket is full insertion replaces the worst node
 	// TODO: give priority to peers with active traffic
-	replaced, err := bucket.insert(node)
+	if len(bucket) >= self.BucketSize { // >= allows us to add peers beyond the bucketsize limitation
-	if err != nil {
+		// always rotate peers
-		glog.V(logger.Debug).Infof("[KΛÐ]: node %v not needed: %v", node, err)
+		idle := self.MaxIdleInterval
-		return err
+		var pos int
-		// no prox adjustment needed
+		var replaced Node
-		// do not change count
+		for i, p := range bucket {
+			idleInt := time.Since(p.LastActive())
+			if idleInt > idle {
+				idle = idleInt
+				pos = i
+				replaced = p
 			}
-	if replaced != nil {
-		glog.V(logger.Debug).Infof("[KΛÐ]: node %v replaced by %v ", replaced, node)
-		return
 		}
-	// new node added
+		if replaced == nil {
-	glog.V(logger.Info).Infof("[KΛÐ]: add node %v to table", node)
+			glog.V(logger.Debug).Infof("[KΛÐ]: all peers wanted, PO%03d bucket full", index)
+			return fmt.Errorf("bucket full")
+		}
+		glog.V(logger.Debug).Infof("[KΛÐ]: node %v replaced by %v (idle for %v  > %v)", replaced, node, idle, self.MaxIdleInterval)
+		replaced.Drop()
+		self.buckets[index] = append(bucket[:pos], bucket[(pos+1):]...)
+		// there is no change in bucket cardinalities so no prox limit adjustment is needed
+		return nil
+	} else {
+		self.buckets[index] = append(bucket, node)
+		glog.V(logger.Debug).Infof("[KΛÐ]: add node %v to table", node)
 		self.count++
-	self.setProxLimit(index, false)
+		self.setProxLimit(index, true)
-	return
+	}
+	record.node = node
+	return nil
 }
 
-//  is the entrypoint called when a node is taken offline
+// Off is the called when a node is taken offline (from the protocol main loop exit)
 func (self *Kademlia) Off(node Node, cb func(*NodeRecord, Node)) (err error) {
 	self.lock.Lock()
 	defer self.lock.Unlock()
@@ -153,70 +161,71 @@ func (self *Kademlia) Off(node Node, cb func(*NodeRecord, Node)) (err error) {
 	var found bool
 	index := self.proximityBin(node.Addr())
 	bucket := self.buckets[index]
-	for i := 0; i < len(bucket.nodes); i++ {
+	for i := 0; i < len(bucket); i++ {
-		if node.Addr() == bucket.nodes[i].Addr() {
+		if node.Addr() == bucket[i].Addr() {
 			found = true
-			bucket.nodes = append(bucket.nodes[:i], bucket.nodes[(i+1):]...)
+			self.buckets[index] = append(bucket[:i], bucket[(i+1):]...)
+			break
 		}
 	}
 
 	if !found {
-		return
+		// gracefully return without error if peer already unregistered
+		glog.V(logger.Warn).Infof("[KΛÐ]: remove node %v not in table, population now is %v", node, self.count)
+		return nil
 	}
-	glog.V(logger.Info).Infof("[KΛÐ]: remove node %v from table", node)
 
 	self.count--
-	if len(bucket.nodes) < bucket.size {
+	glog.V(logger.Debug).Infof("[KΛÐ]: remove node %v from table, population now is %v", node, self.count)
-		err = fmt.Errorf("insufficient nodes (%v) in bucket %v", len(bucket.nodes), index)
+	self.setProxLimit(index, false)
-	}
 
-	self.setProxLimit(index, true)
+	record := self.db.index[node.Addr()]
-
-	r := self.db.index[node.Addr()]
 	// callback on remove
 	if cb != nil {
-		cb(r, r.node)
+		cb(record, record.node)
 	}
-	r.node = nil
+	record.node = nil
-	r.connected = false
 
 	return
 }
 
 // proxLimit is dynamically adjusted so that
 // 1) there is no empty buckets in bin < proxLimit and
-// 2) the sum of all items sare the maximpossible but lower than ProxBinSize
+// 2) the sum of all items are the minimum possible but higher than ProxBinSize
 // adjust Prox (proxLimit and proxSize after an insertion/removal of nodes)
 // caller holds the lock
-func (self *Kademlia) setProxLimit(r int, off bool) {
+func (self *Kademlia) setProxLimit(r int, on bool) {
-	// glog.V(logger.Info).Infof("[KΛÐ]: adjust proxbin for (bin: %v, off: %v)", r, off)
+	// if the change is outside the core (PO lower)
-	if r < self.proxLimit && len(self.buckets[r].nodes) > 0 {
+	// and the change does not leave a bucket empty then
+	// no adjustment needed
+	if r < self.proxLimit && len(self.buckets[r]) > 0 {
 		return
 	}
-	glog.V(logger.Detail).Infof("[KΛÐ]: set proxbin (size: %v, limit: %v, bin: %v, off: %v)", self.proxSize, self.proxLimit, r, off)
+	// if on=a node was added, then r must be within prox limit so increment cardinality
-	if off {
+	if on {
+		self.proxSize++
+		curr := len(self.buckets[self.proxLimit])
+		// if now core is big enough without the furthest bucket, then contract
+		// this can never result in more than one bucket change
+		if self.proxSize >= self.ProxBinSize+curr && curr > 0 {
+			self.proxSize -= curr
+			self.proxLimit++
+			glog.V(logger.Detail).Infof("[KΛÐ]: proxbin contraction (size: %v, limit: %v, bin: %v)", self.proxSize, self.proxLimit, r)
+		}
+		return
+	}
+	// otherwise
+	if r >= self.proxLimit {
 		self.proxSize--
+	}
+	// expand core by lowering prox limit until hit zero or cover the empty bucket or reached target cardinality
 	for (self.proxSize < self.ProxBinSize || r < self.proxLimit) &&
 		self.proxLimit > 0 {
 		//
 		self.proxLimit--
-			self.proxSize += len(self.buckets[self.proxLimit].nodes)
+		self.proxSize += len(self.buckets[self.proxLimit])
-			glog.V(logger.Detail).Infof("[KΛÐ]: proxbin expansion (size: %v, limit: %v, bin: %v, off: %v)", self.proxSize, self.proxLimit, r, off)
+		glog.V(logger.Detail).Infof("[KΛÐ]: proxbin expansion (size: %v, limit: %v, bin: %v)", self.proxSize, self.proxLimit, r)
 	}
-		// glog.V(logger.Detail).Infof("%v", self)
-		return
-	}
-	self.proxSize++
-	for self.proxLimit < self.MaxProx &&
-		len(self.buckets[self.proxLimit].nodes) > 0 &&
-		self.proxSize-len(self.buckets[self.proxLimit].nodes) >= self.ProxBinSize {
-		//
-		self.proxSize -= len(self.buckets[self.proxLimit].nodes)
-		self.proxLimit++
-		glog.V(logger.Detail).Infof("[KΛÐ]: proxbin contraction (size: %v, limit: %v, bin: %v, off: %v)", self.proxSize, self.proxLimit, r, off)
-	}
-	// glog.V(logger.Detail).Infof("%v", self)
-
 }
 
 /*
@@ -227,60 +236,55 @@ proxLimit and MaxProx.
 func (self *Kademlia) FindClosest(target Address, max int) []Node {
 	defer self.lock.RUnlock()
 	self.lock.RLock()
+
 	r := nodesByDistance{
 		target: target,
 	}
-	index := self.proximityBin(target)
 
-	start := index
+	po := self.proximityBin(target)
-	var down bool
+	index := po
-	if index >= self.proxLimit {
+	step := 1
+
+	// set proxbin iteration full
+	if index > self.proxLimit {
 		index = self.proxLimit
-		start = self.MaxProx
-		down = true
 	}
-	var n int
+
+	// if max is set to 0, just want a full bucket, dynamic number
+	min := max
+	// set limit to max
 	limit := max
 	if max == 0 {
-		limit = 1000
+		min = 1
+		limit = maxPeers
 	}
-	for {
 
-		bucket := self.buckets[start].nodes
+	var n int
-		for i := 0; i < len(bucket); i++ {
+	for index >= 0 {
-			r.push(bucket[i], limit)
+		// add entire bucket
+		for _, p := range self.buckets[index] {
+			r.push(p, limit)
 			n++
 		}
-		if max == 0 && start <= index && (n > 0 || start == 0) || max > 0 && down && start <= index && (n >= limit || n == self.count || start == 0) {
+		// terminate if index reached the bottom or enough peers > min
+		if n >= min && (step < 0 || max == 0) {
 			break
 		}
-		if down {
+		// reach top most non-empty PO bucket, turn around
-			start--
+		if index == self.MaxProx {
-		} else {
+			index = po
-			if start == self.MaxProx {
+			step = -1
-				if index == 0 {
-					break
 		}
-				start = index - 1
+		index += step
-				down = true
-			} else {
-				start++
 	}
-		}
+	glog.V(logger.Detail).Infof("[KΛÐ]: serve %d (<=%d) nodes for target lookup %v (PO%03d)", n, max, target, po)
-	}
-	glog.V(logger.Detail).Infof("[KΛÐ]: serve %d (=<%d) nodes for target lookup %v (PO%d)", n, self.MaxProx, target, index)
 	return r.nodes
 }
 
-func (self *Kademlia) binsize(p int) int {
+func (self *Kademlia) Suggest() (*NodeRecord, bool, int) {
-	b := self.buckets[p]
+	defer self.lock.RUnlock()
-	defer b.lock.RUnlock()
+	self.lock.RLock()
-	b.lock.RLock()
+	return self.db.findBest(self.BucketSize, func(i int) int { return len(self.buckets[i]) })
-	return len(b.nodes)
-}
-
-func (self *Kademlia) FindBest() (*NodeRecord, bool, int) {
-	return self.db.findBest(self.BucketSize, self.binsize)
 }
 
 //  adds node records to kaddb (persisted node record db)
@@ -288,13 +292,6 @@ func (self *Kademlia) Add(nrs []*NodeRecord) {
 	self.db.add(nrs, self.proximityBin)
 }
 
-// in situ mutable bucket
-type bucket struct {
-	size  int
-	nodes []Node
-	lock  sync.RWMutex
-}
-
 // nodesByDistance is a list of nodes, ordered by distance to target.
 type nodesByDistance struct {
 	nodes  []Node
@@ -331,27 +328,6 @@ func (h *nodesByDistance) push(node Node, max int) {
 	}
 }
 
-// insert adds a peer to a bucket either by appending to existing items if
-// bucket length does not exceed bucketSize, or by replacing the worst
-// Node in the bucket
-func (self *bucket) insert(node Node) (replaced Node, err error) {
-	self.lock.Lock()
-	defer self.lock.Unlock()
-	if len(self.nodes) >= self.size { // >= allows us to add peers beyond the bucketsize limitation
-		// dev p2p kicks out nodes idle for > 30 s, so here we never replace nodes if
-		// bucket is full
-		return nil, fmt.Errorf("bucket full")
-	}
-	self.nodes = append(self.nodes, node)
-	return
-}
-
-func (self *bucket) length(node Node) int {
-	self.lock.Lock()
-	defer self.lock.Unlock()
-	return len(self.nodes)
-}
-
 /*
 Taking the proximity order relative to a fix point x classifies the points in
 the space (n byte long byte sequences) into bins. Items in each are at
@@ -396,43 +372,46 @@ func (self *Kademlia) Load(path string, cb func(*NodeRecord, Node) error) (err e
 }
 
 // kademlia table + kaddb table displayed with ascii
-// callerholds the lock
 func (self *Kademlia) String() string {
+	defer self.lock.RUnlock()
+	self.lock.RLock()
+	defer self.db.lock.RUnlock()
+	self.db.lock.RLock()
 
 	var rows []string
 	rows = append(rows, "=========================================================================")
-	rows = append(rows, fmt.Sprintf("KΛÐΞMLIΛ hive: queen's address: %v, population: %d (%d)", self.addr, self.count, self.DBCount()))
+	rows = append(rows, fmt.Sprintf("%v KΛÐΞMLIΛ hive: queen's address: %v", time.Now().UTC().Format(time.UnixDate), self.addr.String()[:6]))
-	rows = append(rows, fmt.Sprintf("%v : MaxProx: %d, ProxBinSize: %d, BucketSize: %d, proxLimit: %d, proxSize: %d", time.Now(), self.MaxProx, self.ProxBinSize, self.BucketSize, self.proxLimit, self.proxSize))
+	rows = append(rows, fmt.Sprintf("population: %d (%d), proxLimit: %d, proxSize: %d", self.count, len(self.db.index), self.proxLimit, self.proxSize))
+	rows = append(rows, fmt.Sprintf("MaxProx: %d, ProxBinSize: %d, BucketSize: %d", self.MaxProx, self.ProxBinSize, self.BucketSize))
 
-	for i, b := range self.buckets {
+	for i, bucket := range self.buckets {
 
 		if i == self.proxLimit {
-			rows = append(rows, fmt.Sprintf("===================== PROX LIMIT: %d =================================", i))
+			rows = append(rows, fmt.Sprintf("============ PROX LIMIT: %d ==========================================", i))
 		}
-		row := []string{fmt.Sprintf("%03d", i), fmt.Sprintf("%2d", len(b.nodes))}
+		row := []string{fmt.Sprintf("%03d", i), fmt.Sprintf("%2d", len(bucket))}
 		var k int
 		c := self.db.cursors[i]
-		for ; k < len(b.nodes); k++ {
+		for ; k < len(bucket); k++ {
-			p := b.nodes[(c+k)%len(b.nodes)]
+			p := bucket[(c+k)%len(bucket)]
-			row = append(row, fmt.Sprintf("%s", p.Addr().String()[:8]))
+			row = append(row, p.Addr().String()[:6])
-			if k == 3 {
+			if k == 4 {
 				break
 			}
 		}
-		for ; k < 3; k++ {
+		for ; k < 4; k++ {
 			row = append(row, "      ")
 		}
 		row = append(row, fmt.Sprintf("| %2d %2d", len(self.db.Nodes[i]), self.db.cursors[i]))
 
 		for j, p := range self.db.Nodes[i] {
-			row = append(row, fmt.Sprintf("%08x", p.Addr[:4]))
+			row = append(row, p.Addr.String()[:6])
-			if j == 2 {
+			if j == 3 {
 				break
 			}
 		}
 		rows = append(rows, strings.Join(row, " "))
 		if i == self.MaxProx {
-			break
 		}
 	}
 	rows = append(rows, "=========================================================================")

swarm/network/kademlia/kademlia_test.go

@@ -78,11 +78,11 @@ func TestBootstrap(t *testing.T) {
 		n := 0
 		for n < 100 {
 			err = kad.On(node, nil)
-			if err != nil && err.Error() != "bucket full" {
+			if err != nil {
 				t.Fatalf("backend not accepting node: %v", err)
 			}
 
-			record, need, _ := kad.FindBest()
+			record, need, _ := kad.Suggest()
 			if !need {
 				break
 			}
@@ -150,7 +150,7 @@ func TestFindClosest(t *testing.T) {
 		// check that the result nodes have minimum distance to target.
 		farthestResult := nodes[len(nodes)-1].Addr()
 		for i, b := range kad.buckets {
-			for j, n := range b.nodes {
+			for j, n := range b {
 				if contains(nodes, n.Addr()) {
 					continue // don't run the check below for nodes in result
 				}
@@ -235,12 +235,12 @@ func TestSaveLoad(t *testing.T) {
 	nodes := kad.FindClosest(self, 100)
 	path := "/tmp/bzz.peers"
 	err = kad.Save(path, nil)
-	if err != nil {
+	if err != nil && err.Error() != "bucket full" {
 		t.Fatalf("unepected error saving kaddb: %v", err)
 	}
 	kad = New(self, params)
 	err = kad.Load(path, nil)
-	if err != nil {
+	if err != nil && err.Error() != "bucket full" {
 		t.Fatalf("unepected error loading kaddb: %v", err)
 	}
 	for _, b := range kad.db.Nodes {
@@ -260,30 +260,30 @@ func TestSaveLoad(t *testing.T) {
 }
 
 func (self *Kademlia) proxCheck(t *testing.T) bool {
-	var sum, i int
+	var sum int
-	var b *bucket
+	for i, b := range self.buckets {
-	for i, b = range self.buckets {
+		l := len(b)
-		l := len(b.nodes)
 		// if we are in the high prox multibucket
 		if i >= self.proxLimit {
 			sum += l
 		} else if l == 0 {
-			t.Errorf("bucket %d empty, yet proxLimit is %d\n%v", len(b.nodes), self.proxLimit, self)
+			t.Errorf("bucket %d empty, yet proxLimit is %d\n%v", len(b), self.proxLimit, self)
 			return false
 		}
 	}
 	// check if merged high prox bucket does not exceed size
 	if sum > 0 {
-		// if sum > self.ProxBinSize {
-		// 	t.Errorf("bucket %d is empty, yet proxSize is %d\n%v", i, self.proxSize, self)
-		// 	return false
-		// }
 		if sum != self.proxSize {
 			t.Errorf("proxSize incorrect, expected %v, got %v", sum, self.proxSize)
 			return false
 		}
-		if self.proxLimit > 0 && sum+len(self.buckets[self.proxLimit-1].nodes) < self.ProxBinSize {
+		last := len(self.buckets[self.proxLimit])
-			t.Errorf("proxBinSize incorrect, expected %v got %v", sum, self.proxSize)
+		if last > 0 && sum >= self.ProxBinSize+last {
+			t.Errorf("proxLimit %v incorrect, redundant non-empty bucket %d added to proxBin with %v (target %v)", self.proxLimit, last, sum-last, self.ProxBinSize)
+			return false
+		}
+		if self.proxLimit > 0 && sum < self.ProxBinSize {
+			t.Errorf("proxLimit %v incorrect. proxSize %v is less than target %v, yet there is more peers", self.proxLimit, sum, self.ProxBinSize)
 			return false
 		}
 	}

swarm/network/protocol.go

@@ -49,6 +49,7 @@ const (
 	ErrExtraStatusMsg
 	ErrSwap
 	ErrSync
+	ErrUnwanted
 )
 
 var errorToString = map[int]string{
@@ -61,6 +62,7 @@ var errorToString = map[int]string{
 	ErrExtraStatusMsg:    "Extra status message",
 	ErrSwap:              "SWAP error",
 	ErrSync:              "Sync error",
+	ErrUnwanted:          "Unwanted peer",
 }
 
 // bzz represents the swarm wire protocol
@@ -179,7 +181,8 @@ func run(requestDb *storage.LDBDatabase, depo StorageHandler, backend bind.Backe
 	// the main forever loop that handles incoming requests
 	for {
 		if self.hive.blockRead {
-			time.Sleep(1 * time.Second)
+			glog.V(logger.Warn).Infof("[BZZ] Cannot read network")
+			time.Sleep(100 * time.Millisecond)
 			continue
 		}
 		err = self.handle()
@@ -223,7 +226,7 @@ func (self *bzz) handle() error {
 		if err := msg.Decode(&req); err != nil {
 			return self.protoError(ErrDecode, "<- %v: %v", msg, err)
 		}
-		glog.V(logger.Debug).Infof("[BZZ] incoming store request: %s", req.String())
+		glog.V(logger.Detail).Infof("[BZZ] incoming store request: %s", req.String())
 		// swap accounting is done within forwarding
 		self.storage.HandleStoreRequestMsg(&req, &peer{bzz: self})
 
@@ -254,7 +257,7 @@ func (self *bzz) handle() error {
 			return self.protoError(ErrDecode, "<- %v: %v", msg, err)
 		}
 		req.from = &peer{bzz: self}
-		glog.V(logger.Debug).Infof("[BZZ] <- peer addresses: %v", req)
+		glog.V(logger.Detail).Infof("[BZZ] <- peer addresses: %v", req)
 		self.hive.HandlePeersMsg(&req, &peer{bzz: self})
 
 	case syncRequestMsg:
@@ -366,7 +369,10 @@ func (self *bzz) handleStatus() (err error) {
 	}
 
 	glog.V(logger.Info).Infof("[BZZ] Peer %08x is [bzz] capable (%d/%d)", self.remoteAddr.Addr[:4], status.Version, status.NetworkId)
-	self.hive.addPeer(&peer{bzz: self})
+	err = self.hive.addPeer(&peer{bzz: self})
+	if err != nil {
+		return self.protoError(ErrUnwanted, "%v", err)
+	}
 
 	// hive sets syncstate so sync should start after node added
 	glog.V(logger.Info).Infof("[BZZ] syncronisation request sent with %v", self.syncState)
@@ -516,7 +522,6 @@ func (self *bzz) send(msg uint64, data interface{}) error {
 	if self.hive.blockWrite {
 		return fmt.Errorf("network write blocked")
 	}
-	// self.messages = append(self.messages, "")
 	glog.V(logger.Detail).Infof("[BZZ] -> %v: %v (%T) to %v", msg, data, data, self)
 	err := p2p.Send(self.rw, msg, data)
 	if err != nil {

swarm/network/syncdb.go

@@ -126,7 +126,7 @@ LOOP:
 			// if syncdb is stopped. In this case we need to save the item to the db
 			more = deliver(req, self.quit)
 			if !more {
-				glog.V(logger.Debug).Infof("[BZZ] syncDb[%v] quit: switching to db. session tally (db/total): %v/%v", self.priority, self.dbTotal, self.total)
+				glog.V(logger.Debug).Infof("[BZZ] syncDb[%v/%v] quit: switching to db. session tally (db/total): %v/%v", self.key.Log(), self.priority, self.dbTotal, self.total)
 				// received quit signal, save request currently waiting delivery
 				// by switching to db mode and closing the buffer
 				buffer = nil
@@ -136,12 +136,12 @@ LOOP:
 				break      // break from select, this item will be written to the db
 			}
 			self.total++
-			glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] deliver (db/total): %v/%v", self.priority, self.dbTotal, self.total)
+			glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] deliver (db/total): %v/%v", self.key.Log(), self.priority, self.dbTotal, self.total)
 			// by the time deliver returns, there were new writes to the buffer
 			// if buffer contention is detected, switch to db mode which drains
 			// the buffer so no process will block on pushing store requests
 			if len(buffer) == cap(buffer) {
-				glog.V(logger.Debug).Infof("[BZZ] syncDb[%v] buffer full %v: switching to db. session tally (db/total): %v/%v", self.priority, cap(buffer), self.dbTotal, self.total)
+				glog.V(logger.Debug).Infof("[BZZ] syncDb[%v/%v] buffer full %v: switching to db. session tally (db/total): %v/%v", self.key.Log(), self.priority, cap(buffer), self.dbTotal, self.total)
 				buffer = nil
 				db = self.buffer
 			}
@@ -154,18 +154,18 @@ LOOP:
 				binary.BigEndian.PutUint64(counterValue, counter)
 				batch.Put(self.counterKey, counterValue) // persist counter in batch
 				self.writeSyncBatch(batch)               // save batch
-				glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] quitting: save current batch to db", self.priority)
+				glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] quitting: save current batch to db", self.key.Log(), self.priority)
 				break LOOP
 			}
 			self.dbTotal++
 			self.total++
-			// otherwise break after selec
+			// otherwise break after select
 		case dbSize = <-self.batch:
 			// explicit request for batch
 			if inBatch == 0 && quit != nil {
 				// there was no writes since the last batch so db depleted
 				// switch to buffer mode
-				glog.V(logger.Debug).Infof("[BZZ] syncDb[%v] empty db: switching to buffer", self.priority)
+				glog.V(logger.Debug).Infof("[BZZ] syncDb[%v/%v] empty db: switching to buffer", self.key.Log(), self.priority)
 				db = nil
 				buffer = self.buffer
 				dbSize <- 0 // indicates to 'caller' that batch has been written
@@ -174,7 +174,7 @@ LOOP:
 			}
 			binary.BigEndian.PutUint64(counterValue, counter)
 			batch.Put(self.counterKey, counterValue)
-			glog.V(logger.Debug).Infof("[BZZ] syncDb[%v] write batch %v/%v - %x - %x", self.priority, inBatch, counter, self.counterKey, counterValue)
+			glog.V(logger.Debug).Infof("[BZZ] syncDb[%v/%v] write batch %v/%v - %x - %x", self.key.Log(), self.priority, inBatch, counter, self.counterKey, counterValue)
 			batch = self.writeSyncBatch(batch)
 			dbSize <- inBatch // indicates to 'caller' that batch has been written
 			inBatch = 0
@@ -186,7 +186,7 @@ LOOP:
 			db = self.buffer
 			buffer = nil
 			quit = nil
-			glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] quitting: save buffer to db", self.priority)
+			glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] quitting: save buffer to db", self.key.Log(), self.priority)
 			close(db)
 			continue LOOP
 		}
@@ -194,15 +194,15 @@ LOOP:
 		// only get here if we put req into db
 		entry, err = self.newSyncDbEntry(req, counter)
 		if err != nil {
-			glog.V(logger.Warn).Infof("[BZZ] syncDb[%v] saving request %v (#%v/%v) failed: %v", self.priority, req, inBatch, inDb, err)
+			glog.V(logger.Warn).Infof("[BZZ] syncDb[%v/%v] saving request %v (#%v/%v) failed: %v", self.key.Log(), self.priority, req, inBatch, inDb, err)
 			continue LOOP
 		}
 		batch.Put(entry.key, entry.val)
-		glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] to batch %v '%v' (#%v/%v/%v)", self.priority, req, entry, inBatch, inDb, counter)
+		glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] to batch %v '%v' (#%v/%v/%v)", self.key.Log(), self.priority, req, entry, inBatch, inDb, counter)
 		// if just switched to db mode and not quitting, then launch dbRead
 		// in a parallel go routine to send deliveries from db
 		if inDb == 0 && quit != nil {
-			glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] start dbRead")
+			glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] start dbRead")
 			go self.dbRead(true, counter, deliver)
 		}
 		inDb++
@@ -221,7 +221,7 @@ LOOP:
 func (self *syncDb) writeSyncBatch(batch *leveldb.Batch) *leveldb.Batch {
 	err := self.db.Write(batch)
 	if err != nil {
-		glog.V(logger.Warn).Infof("[BZZ] syncDb[%v] saving batch to db failed: %v", self.priority, err)
+		glog.V(logger.Warn).Infof("[BZZ] syncDb[%v/%v] saving batch to db failed: %v", self.key.Log(), self.priority, err)
 		return batch
 	}
 	return new(leveldb.Batch)
@@ -295,7 +295,7 @@ func (self *syncDb) dbRead(useBatches bool, counter uint64, fun func(interface{}
 			continue
 		}
 		del = new(leveldb.Batch)
-		glog.V(logger.Detail).Infof("[BZZ] syncDb[%v]: new iterator: %x (batch %v, count %v)", self.priority, key, batches, cnt)
+		glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v]: new iterator: %x (batch %v, count %v)", self.key.Log(), self.priority, key, batches, cnt)
 
 		for n = 0; !useBatches || n < cnt; it.Next() {
 			copy(key, it.Key())
@@ -307,11 +307,11 @@ func (self *syncDb) dbRead(useBatches bool, counter uint64, fun func(interface{}
 			val := make([]byte, 40)
 			copy(val, it.Value())
 			entry = &syncDbEntry{key, val}
-			// glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] - %v, batches: %v, total: %v, session total from db: %v/%v", self.priority, self.key.Log(), batches, total, self.dbTotal, self.total)
+			// glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] - %v, batches: %v, total: %v, session total from db: %v/%v", self.key.Log(), self.priority, self.key.Log(), batches, total, self.dbTotal, self.total)
 			more = fun(entry, self.quit)
 			if !more {
 				// quit received when waiting to deliver entry, the entry will not be deleted
-				glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] batch %v quit after %v/%v items", self.priority, batches, n, cnt)
+				glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] batch %v quit after %v/%v items", self.key.Log(), self.priority, batches, n, cnt)
 				break
 			}
 			// since subsequent batches of the same db session are indexed incrementally

swarm/network/syncer.go

@@ -298,6 +298,7 @@ func (self *syncer) sync() {
 	if state.LastSeenAt < state.SessionAt {
 		state.Last = state.SessionAt
 		glog.V(logger.Debug).Infof("[BZZ] syncer[%v]: start syncronising history since last disconnect at %v up until session start at %v: %v", self.key.Log(), state.LastSeenAt, state.SessionAt, state)
+		// blocks until state syncing is finished
 		self.syncState(state)
 	}
 	glog.V(logger.Info).Infof("[BZZ] syncer[%v]: syncing all history complete", self.key.Log())
@@ -358,7 +359,9 @@ func (self *syncer) syncHistory(state *syncState) chan interface{} {
 		IT:
 			for {
 				key := it.Next()
-				if key != nil {
+				if key == nil {
+					break IT
+				}
 				select {
 				// blocking until history channel is read from
 				case history <- storage.Key(key):
@@ -368,9 +371,6 @@ func (self *syncer) syncHistory(state *syncState) chan interface{} {
 				case <-self.quit:
 					return
 				}
-				} else {
-					break IT
-				}
 			}
 			glog.V(logger.Debug).Infof("[BZZ] syncer[%v]: finished syncing history between %v - %v for chunk addresses %v - %v (at %v) (chunks = %v)", self.key.Log(), state.First, state.Last, state.Start, state.Stop, state.Latest, n)
 		}()
@@ -416,26 +416,32 @@ LOOP:
 		// are checked first - integrity can only be guaranteed if writing
 		// is locked while selecting
 		if priority != High || len(keys) == 0 {
+			// selection is not needed if the High priority queue has items
 			keys = nil
+		PRIORITIES:
 			for priority = High; priority >= 0; priority-- {
+				// the first priority channel that is non-empty will be assigned to keys
 				if len(self.keys[priority]) > 0 {
 					glog.V(logger.Detail).Infof("[BZZ] syncer[%v]: reading request with	 priority %v", self.key.Log(), priority)
 					keys = self.keys[priority]
-					break
+					break PRIORITIES
 				}
+				glog.V(logger.Debug).Infof("[BZZ] syncer[%v/%v]: queue: [%v, %v, %v]", self.key.Log(), priority, len(self.keys[High]), len(self.keys[Medium]), len(self.keys[Low]))
+				// if the input queue is empty on this level, resort to history if there is any
 				if uint(priority) == histPrior && history != nil {
 					glog.V(logger.Detail).Infof("[BZZ] syncer[%v]: reading history for %v", self.key.Log(), self.key)
 					keys = history
-					break
+					break PRIORITIES
 				}
 			}
+		}
+
 		// if peer ready to receive but nothing to send
 		if keys == nil && deliveryRequest == nil {
 			// if no items left and switch to waiting mode
 			glog.V(logger.Detail).Infof("[BZZ] syncer[%v]: buffers consumed. Waiting", self.key.Log())
 			newUnsyncedKeys = self.newUnsyncedKeys
 		}
-		}
 
 		// send msg iff
 		// * peer is ready to receive keys AND (
@@ -447,7 +453,7 @@ LOOP:
 				len(unsynced) > 0 && keys == nil ||
 				len(unsynced) == int(self.SyncBatchSize)) {
 			justSynced = false
-			// listen to requests again
+			// listen to requests
 			deliveryRequest = self.deliveryRequest
 			newUnsyncedKeys = nil // not care about data until next req comes in
 			// set sync to current counter
@@ -458,11 +464,11 @@ LOOP:
 			//  send the unsynced keys
 			stateCopy := *state
 			err := self.unsyncedKeys(unsynced, &stateCopy)
-			self.state = state
-			glog.V(logger.Debug).Infof("[BZZ] syncer[%v]: --> %v keys sent: (total: %v (%v), history: %v), sent sync state: %v", self.key.Log(), len(unsynced), keyCounts, keyCount, historyCnt, stateCopy)
 			if err != nil {
 				glog.V(logger.Warn).Infof("[BZZ] syncer[%v]: unable to send unsynced keys: %v", err)
 			}
+			self.state = state
+			glog.V(logger.Debug).Infof("[BZZ] syncer[%v]: --> %v keys sent: (total: %v (%v), history: %v), sent sync state: %v", self.key.Log(), len(unsynced), keyCounts, keyCount, historyCnt, stateCopy)
 			unsynced = nil
 			keys = nil
 		}
@@ -579,7 +585,7 @@ func (self *syncer) syncDeliveries() {
 		total++
 		msg, err = self.newStoreRequestMsgData(req)
 		if err != nil {
-			glog.V(logger.Warn).Infof("[BZZ] syncer[%v]: failed to deliver %v: %v", self.key.Log(), req, err)
+			glog.V(logger.Warn).Infof("[BZZ] syncer[%v]: failed to create store request for %v: %v", self.key.Log(), req, err)
 		} else {
 			err = self.store(msg)
 			if err != nil {

swarm/storage/chunker.go

@@ -2,10 +2,11 @@ package storage
 
 import (
 	"encoding/binary"
+	"errors"
 	"fmt"
+	"hash"
 	"io"
 	"sync"
-	"time"
 
 	"github.com/ethereum/go-ethereum/logger"
 	"github.com/ethereum/go-ethereum/logger/glog"
@@ -37,11 +38,9 @@ data_{i} := size(subtree_{i}) || key_{j} || key_{j+1} .... || key_{j+n-1}
 */
 
 const (
-	// defaultHash           = "SHA3"   // http://golang.org/pkg/hash/#Hash
+	defaultHash = "SHA3" // http://golang.org/pkg/hash/#Hash
-	defaultHash           = "SHA256" // http://golang.org/pkg/hash/#Hash
+	// defaultHash           = "SHA256" // http://golang.org/pkg/hash/#Hash
 	defaultBranches int64 = 128
-	joinTimeout           = 120 // second
-	splitTimeout          = 120 // second
 	// hashSize     int64 = hasherfunc.New().Size() // hasher knows about its own length in bytes
 	// chunksize    int64 = branches * hashSize     // chunk is defined as this
 )
@@ -57,128 +56,105 @@ The hashing itself does use extra copies and allocation though, since it does ne
 type ChunkerParams struct {
 	Branches int64
 	Hash     string
-	JoinTimeout  time.Duration
-	SplitTimeout time.Duration
 }
 
 func NewChunkerParams() *ChunkerParams {
 	return &ChunkerParams{
 		Branches: defaultBranches,
 		Hash:     defaultHash,
-		JoinTimeout:  joinTimeout,
-		SplitTimeout: splitTimeout,
 	}
 }
 
 type TreeChunker struct {
 	branches int64
 	hashFunc Hasher
-	joinTimeout  time.Duration
-	splitTimeout time.Duration
 	// calculated
 	hashSize    int64 // self.hashFunc.New().Size()
 	chunkSize   int64 // hashSize* branches
+	workerCount int
 }
 
 func NewTreeChunker(params *ChunkerParams) (self *TreeChunker) {
 	self = &TreeChunker{}
 	self.hashFunc = MakeHashFunc(params.Hash)
 	self.branches = params.Branches
-	self.joinTimeout = params.JoinTimeout * time.Second
-	self.splitTimeout = params.SplitTimeout * time.Second
 	self.hashSize = int64(self.hashFunc().Size())
 	self.chunkSize = self.hashSize * self.branches
+	self.workerCount = 1
 	return
 }
 
-func (self *TreeChunker) KeySize() int64 {
+// func (self *TreeChunker) KeySize() int64 {
-	return self.hashSize
+// 	return self.hashSize
-}
+// }
 
 // String() for pretty printing
 func (self *Chunk) String() string {
 	return fmt.Sprintf("Key: %v TreeSize: %v Chunksize: %v", self.Key.Log(), self.Size, len(self.SData))
 }
 
-// The treeChunkers own Hash hashes together
+type hashJob struct {
-// - the size (of the subtree encoded in the Chunk)
+	key      Key
-// - the Chunk, ie. the contents read from the input reader
+	chunk    []byte
-func (self *TreeChunker) Hash(input []byte) []byte {
+	size     int64
-	hasher := self.hashFunc()
+	parentWg *sync.WaitGroup
-	hasher.Write(input)
-	return hasher.Sum(nil)
 }
 
-func (self *TreeChunker) Split(key Key, data SectionReader, chunkC chan *Chunk, swg *sync.WaitGroup) (errC chan error) {
+func (self *TreeChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, swg, wwg *sync.WaitGroup) (Key, error) {
-
-	if swg != nil {
-		swg.Add(1)
-		defer swg.Done()
-	}
 
 	if self.chunkSize <= 0 {
 		panic("chunker must be initialised")
 	}
 
-	if int64(len(key)) != self.hashSize {
+	jobC := make(chan *hashJob, 2*processors)
-		panic(fmt.Sprintf("root key buffer must be allocated byte slice of length %d", self.hashSize))
-	}
-
 	wg := &sync.WaitGroup{}
-	errC = make(chan error)
+	errC := make(chan error)
-	rerrC := make(chan error)
-	timeout := time.After(self.splitTimeout)
 
-	wg.Add(1)
+	// wwg = workers waitgroup keeps track of hashworkers spawned by this split call
-	go func() {
+	if wwg != nil {
+		wwg.Add(1)
+	}
+	go self.hashWorker(jobC, chunkC, errC, swg, wwg)
 
 	depth := 0
 	treeSize := self.chunkSize
-		size := data.Size()
+
 	// takes lowest depth such that chunksize*HashCount^(depth+1) > size
 	// power series, will find the order of magnitude of the data size in base hashCount or numbers of levels of branching in the resulting tree.
-
 	for ; treeSize < size; treeSize *= self.branches {
 		depth++
 	}
 
+	key := make([]byte, self.hashFunc().Size())
 	// glog.V(logger.Detail).Infof("[BZZ] split request received for data (%v bytes, depth: %v)", size, depth)
-
+	// this waitgroup member is released after the root hash is calculated
-		//launch actual recursive function passing the workgroup
+	wg.Add(1)
-		self.split(depth, treeSize/self.branches, key, data, chunkC, rerrC, wg, swg)
+	//launch actual recursive function passing the waitgroups
-	}()
+	go self.split(depth, treeSize/self.branches, key, data, size, jobC, chunkC, errC, wg, swg, wwg)
 
 	// closes internal error channel if all subprocesses in the workgroup finished
 	go func() {
+		// waiting for all threads to finish
 		wg.Wait()
-		close(rerrC)
+		// if storage waitgroup is non-nil, we wait for storage to finish too
-
+		if swg != nil {
-	}()
+			// glog.V(logger.Detail).Infof("Waiting for storage to finish")
-
+			swg.Wait()
-	// waiting for request to end with wg finishing, error, or timeout
-	go func() {
-		select {
-		case err := <-rerrC:
-			if err != nil {
-				errC <- err
-			} // otherwise splitting is complete
-		case <-timeout:
-			errC <- fmt.Errorf("split time out")
 		}
 		close(errC)
 	}()
 
-	return
+	select {
+	case err := <-errC:
+		if err != nil {
+			return nil, err
+		}
+		//
+	}
+	return key, nil
 }
 
-func (self *TreeChunker) split(depth int, treeSize int64, key Key, data SectionReader, chunkC chan *Chunk, errc chan error, parentWg *sync.WaitGroup, swg *sync.WaitGroup) {
+func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reader, size int64, jobC chan *hashJob, chunkC chan *Chunk, errC chan error, parentWg, swg, wwg *sync.WaitGroup) {
-
-	defer parentWg.Done()
-
-	size := data.Size()
-	var newChunk *Chunk
-	var hash Key
-	// glog.V(logger.Detail).Infof("[BZZ] depth: %v, max subtree size: %v, data size: %v", depth, treeSize, size)
 
 	for depth > 0 && size < treeSize {
 		treeSize /= self.branches
@@ -187,17 +163,16 @@ func (self *TreeChunker) split(depth int, treeSize int64, key Key, data SectionR
 
 	if depth == 0 {
 		// leaf nodes -> content chunks
-		chunkData := make([]byte, data.Size()+8)
+		chunkData := make([]byte, size+8)
 		binary.LittleEndian.PutUint64(chunkData[0:8], uint64(size))
-		data.ReadAt(chunkData[8:], 0)
+		data.Read(chunkData[8:])
-		hash = self.Hash(chunkData)
+		select {
-		// glog.V(logger.Detail).Infof("[BZZ] content chunk: max subtree size: %v, data size: %v", treeSize, size)
+		case jobC <- &hashJob{key, chunkData, size, parentWg}:
-		newChunk = &Chunk{
+		case <-errC:
-			Key:   hash,
+		}
-			SData: chunkData,
+		// glog.V(logger.Detail).Infof("[BZZ] read %v", size)
-			Size:  size,
+		return
 	}
-	} else {
 	// intermediate chunk containing child nodes hashes
 	branchCnt := int64((size + treeSize - 1) / treeSize)
 	// glog.V(logger.Detail).Infof("[BZZ] intermediate node: setting branches: %v, depth: %v, max subtree size: %v, data size: %v", branches, depth, treeSize, size)
@@ -216,156 +191,205 @@ func (self *TreeChunker) split(depth int, treeSize int64, key Key, data SectionR
 		} else {
 			secSize = treeSize
 		}
-			// take the section of the data encoded in the subTree
-			subTreeData := NewChunkReader(data, pos, secSize)
 		// the hash of that data
 		subTreeKey := chunk[8+i*self.hashSize : 8+(i+1)*self.hashSize]
 
 		childrenWg.Add(1)
-			go self.split(depth-1, treeSize/self.branches, subTreeKey, subTreeData, chunkC, errc, childrenWg, swg)
+		self.split(depth-1, treeSize/self.branches, subTreeKey, data, secSize, jobC, chunkC, errC, childrenWg, swg, wwg)
 
 		i++
 		pos += treeSize
 	}
 	// wait for all the children to complete calculating their hashes and copying them onto sections of the chunk
+	// parentWg.Add(1)
+	// go func() {
 	childrenWg.Wait()
+	if len(jobC) > self.workerCount && self.workerCount < processors {
+		if wwg != nil {
+			wwg.Add(1)
+		}
+		self.workerCount++
+		go self.hashWorker(jobC, chunkC, errC, swg, wwg)
+	}
+	select {
+	case jobC <- &hashJob{key, chunk, size, parentWg}:
+	case <-errC:
+	}
+}
+
+func (self *TreeChunker) hashWorker(jobC chan *hashJob, chunkC chan *Chunk, errC chan error, swg, wwg *sync.WaitGroup) {
+	hasher := self.hashFunc()
+	if wwg != nil {
+		defer wwg.Done()
+	}
+	for {
+		select {
+
+		case job, ok := <-jobC:
+			if !ok {
+				return
+			}
 			// now we got the hashes in the chunk, then hash the chunks
-		hash = self.Hash(chunk)
+			hasher.Reset()
-		newChunk = &Chunk{
+			self.hashChunk(hasher, job, chunkC, swg)
-			Key:   hash,
+			// glog.V(logger.Detail).Infof("[BZZ] hash chunk (%v)", job.size)
-			SData: chunk,
+		case <-errC:
-			Size:  size,
+			return
+		}
+	}
+}
+
+// The treeChunkers own Hash hashes together
+// - the size (of the subtree encoded in the Chunk)
+// - the Chunk, ie. the contents read from the input reader
+func (self *TreeChunker) hashChunk(hasher hash.Hash, job *hashJob, chunkC chan *Chunk, swg *sync.WaitGroup) {
+	hasher.Write(job.chunk)
+	h := hasher.Sum(nil)
+	newChunk := &Chunk{
+		Key:   h,
+		SData: job.chunk,
+		Size:  job.size,
 		wg:    swg,
 	}
 
+	// report hash of this chunk one level up (keys corresponds to the proper subslice of the parent chunk)
+	copy(job.key, h)
+	// send off new chunk to storage
+	if chunkC != nil {
 		if swg != nil {
 			swg.Add(1)
 		}
 	}
-
+	job.parentWg.Done()
-	// send off new chunk to storage
 	if chunkC != nil {
 		chunkC <- newChunk
 	}
-	// report hash of this chunk one level up (keys corresponds to the proper subslice of the parent chunk)x
-	copy(key, hash)
-
-}
-
-func (self *TreeChunker) Join(key Key, chunkC chan *Chunk) SectionReader {
-
-	return &LazyChunkReader{
-		key:     key,
-		chunkC:  chunkC,
-		quitC:   make(chan bool),
-		errC:    make(chan error),
-		chunker: self,
-	}
 }
 
 // LazyChunkReader implements LazySectionReader
 type LazyChunkReader struct {
 	key       Key         // root key
 	chunkC    chan *Chunk // chunk channel to send retrieve requests on
-	size    int64        // size of the entire subtree
+	chunk     *Chunk      // size of the entire subtree
 	off       int64       // offset
-	quitC   chan bool    // channel to abort retrieval
+	chunkSize int64       // inherit from chunker
-	errC    chan error   // error channel to monitor retrieve errors
+	branches  int64       // inherit from chunker
-	chunker *TreeChunker // needs TreeChunker params TODO: should just take
+	hashSize  int64       // inherit from chunker
-	// the chunkSize, branches etc as params
 }
 
-func (self *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) {
+// implements the Joiner interface
-	self.errC = make(chan error)
+func (self *TreeChunker) Join(key Key, chunkC chan *Chunk) LazySectionReader {
-	chunk := &Chunk{
-		Key: self.key,
-		C:   make(chan bool), // close channel to signal data delivery
-	}
-	self.chunkC <- chunk // submit retrieval request, someone should be listening on the other side (or we will time out globally)
-	glog.V(logger.Detail).Infof("[BZZ] readAt: reading %v into %d bytes at offset %d.", chunk.Key.Log(), len(b), off)
 
-	// waiting for the chunk retrieval
+	return &LazyChunkReader{
+		key:       key,
+		chunkC:    chunkC,
+		chunkSize: self.chunkSize,
+		branches:  self.branches,
+		hashSize:  self.hashSize,
+	}
+}
+
+// Size is meant to be called on the LazySectionReader
+func (self *LazyChunkReader) Size(quitC chan bool) (n int64, err error) {
+	if self.chunk != nil {
+		return self.chunk.Size, nil
+	}
+	chunk := retrieve(self.key, self.chunkC, quitC)
+	if chunk == nil {
 		select {
-	case <-self.quitC:
+		case <-quitC:
-		// this is how we control process leakage (quitC is closed once join is finished (after timeout))
+			return 0, errors.New("aborted")
-		// glog.V(logger.Detail).Infof("[BZZ] quit")
+		default:
-		return
+			return 0, fmt.Errorf("root chunk not found for %v", self.key.Hex())
-	case <-chunk.C: // bells are ringing, data have been delivered
-		// glog.V(logger.Detail).Infof("[BZZ] chunk data received for %v", chunk.Key.Log())
 		}
-	if len(chunk.SData) == 0 {
-		// glog.V(logger.Detail).Infof("[BZZ] No payload in %v", chunk.Key.Log())
-		return 0, notFound
 	}
-	chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8]))
+	self.chunk = chunk
-	self.size = chunk.Size
+	return chunk.Size, nil
-	if b == nil {
+}
+
+// read at can be called numerous times
+// concurrent reads are allowed
+// Size() needs to be called synchronously on the LazyChunkReader first
+func (self *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) {
+	// this is correct, a swarm doc cannot be zero length, so no EOF is expected
+	if len(b) == 0 {
 		// glog.V(logger.Detail).Infof("[BZZ] Size query for %v", chunk.Key.Log())
-		return
+		return 0, nil
 	}
-	want := int64(len(b))
+	quitC := make(chan bool)
-	if off+want > self.size {
+	size, err := self.Size(quitC)
-		want = self.size - off
+	if err != nil {
+		return 0, err
 	}
+	glog.V(logger.Detail).Infof("readAt: len(b): %v, off: %v, size: %v ", len(b), off, size)
+
+	errC := make(chan error)
+	// glog.V(logger.Detail).Infof("[BZZ] readAt: reading %v into %d bytes at offset %d.", self.chunk.Key.Log(), len(b), off)
+
+	// }
+	// glog.V(logger.Detail).Infof("-> want: %v, off: %v size: %v ", want, off, self.size)
 	var treeSize int64
 	var depth int
 	// calculate depth and max treeSize
-	treeSize = self.chunker.chunkSize
+	treeSize = self.chunkSize
-	for ; treeSize < chunk.Size; treeSize *= self.chunker.branches {
+	for ; treeSize < size; treeSize *= self.branches {
 		depth++
 	}
 	wg := sync.WaitGroup{}
 	wg.Add(1)
-	go self.join(b, off, off+want, depth, treeSize/self.chunker.branches, chunk, &wg)
+	go self.join(b, off, off+int64(len(b)), depth, treeSize/self.branches, self.chunk, &wg, errC, quitC)
 	go func() {
 		wg.Wait()
-		close(self.errC)
+		close(errC)
 	}()
-	select {
+
-	case err = <-self.errC:
+	err = <-errC
+	if err != nil {
+		close(quitC)
+
+		return 0, err
+	}
 	// glog.V(logger.Detail).Infof("[BZZ] ReadAt received %v", err)
-		read = len(b)
+	glog.V(logger.Detail).Infof("end: len(b): %v, off: %v, size: %v ", len(b), off, size)
-		if off+int64(read) == self.size {
+	if off+int64(len(b)) >= size {
-			err = io.EOF
+		glog.V(logger.Detail).Infof(" len(b): %v EOF", len(b))
+		return len(b), io.EOF
 	}
 	// glog.V(logger.Detail).Infof("[BZZ] ReadAt returning at %d: %v", read, err)
-	case <-self.quitC:
+	return len(b), nil
-		// glog.V(logger.Detail).Infof("[BZZ] ReadAt aborted at %d: %v", read, err)
-	}
-	return
 }
 
-func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeSize int64, chunk *Chunk, parentWg *sync.WaitGroup) {
+func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeSize int64, chunk *Chunk, parentWg *sync.WaitGroup, errC chan error, quitC chan bool) {
 	defer parentWg.Done()
+	// return NewDPA(&LocalStore{})
+	glog.V(logger.Detail).Infof("inh len(b): %v, off: %v eoff: %v ", len(b), off, eoff)
 
 	// glog.V(logger.Detail).Infof("[BZZ] depth: %v, loff: %v, eoff: %v, chunk.Size: %v, treeSize: %v", depth, off, eoff, chunk.Size, treeSize)
 
-	chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8]))
+	// chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8]))
 
 	// find appropriate block level
 	for chunk.Size < treeSize && depth > 0 {
-		treeSize /= self.chunker.branches
+		treeSize /= self.branches
 		depth--
 	}
 
+	// leaf chunk found
 	if depth == 0 {
-		// glog.V(logger.Detail).Infof("[BZZ] depth: %v, len(b): %v, off: %v, eoff: %v, chunk.Size: %v, treeSize: %v", depth, len(b), off, eoff, chunk.Size, treeSize)
+		glog.V(logger.Detail).Infof("[BZZ] depth: %v, len(b): %v, off: %v, eoff: %v, chunk.Size: %v, treeSize: %v", depth, len(b), off, eoff, chunk.Size, treeSize)
-		if int64(len(b)) != eoff-off {
-			//fmt.Printf("len(b) = %v  off = %v  eoff = %v", len(b), off, eoff)
-			panic("len(b) does not match")
-		}
-
 		copy(b, chunk.SData[8+off:8+eoff])
 		return // simply give back the chunks reader for content chunks
 	}
 
-	// subtree index
+	// subtree
 	start := off / treeSize
 	end := (eoff + treeSize - 1) / treeSize
-	wg := sync.WaitGroup{}
+
+	wg := &sync.WaitGroup{}
+	defer wg.Wait()
+	glog.V(logger.Detail).Infof("[BZZ] start %v,end %v", start, end)
 
 	for i := start; i < end; i++ {
-
 		soff := i * treeSize
 		roff := soff
 		seoff := soff + treeSize
@@ -376,36 +400,91 @@ func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, tr
 		if seoff > eoff {
 			seoff = eoff
 		}
-
+		if depth > 1 {
+			wg.Wait()
+		}
 		wg.Add(1)
 		go func(j int64) {
-			childKey := chunk.SData[8+j*self.chunker.hashSize : 8+(j+1)*self.chunker.hashSize]
+			childKey := chunk.SData[8+j*self.hashSize : 8+(j+1)*self.hashSize]
-			// glog.V(logger.Detail).Infof("[BZZ] subtree index: %v -> %v", j, childKey.Log())
+			// glog.V(logger.Detail).Infof("[BZZ] subtree ind.ex: %v -> %v", j, childKey.Log())
-
+			chunk := retrieve(childKey, self.chunkC, quitC)
-			ch := &Chunk{
+			if chunk == nil {
-				Key: childKey,
-				C:   make(chan bool), // close channel to signal data delivery
-			}
-			// glog.V(logger.Detail).Infof("[BZZ] chunk data sent for %v (key interval in chunk %v-%v)", ch.Key.Log(), j*self.chunker.hashSize, (j+1)*self.chunker.hashSize)
-			self.chunkC <- ch // submit retrieval request, someone should be listening on the other side (or we will time out globally)
-
-			// waiting for the chunk retrieval
 				select {
-			case <-self.quitC:
+				case errC <- fmt.Errorf("chunk %v-%v not found", off, off+treeSize):
-				// this is how we control process leakage (quitC is closed once join is finished (after timeout))
+				case <-quitC:
+				}
 				return
-			case <-ch.C: // bells are ringing, data have been delivered
-				// glog.V(logger.Detail).Infof("[BZZ] chunk data received")
 			}
 			if soff < off {
 				soff = off
 			}
-			if len(ch.SData) == 0 {
+			self.join(b[soff-off:seoff-off], soff-roff, seoff-roff, depth-1, treeSize/self.branches, chunk, wg, errC, quitC)
-				self.errC <- fmt.Errorf("chunk %v-%v not found", off, off+treeSize)
-				return
-			}
-			self.join(b[soff-off:seoff-off], soff-roff, seoff-roff, depth-1, treeSize/self.chunker.branches, ch, &wg)
 		}(i)
 	} //for
-	wg.Wait()
+}
+
+// the helper method submits chunks for a key to a oueue (DPA) and
+// block until they time out or arrive
+// abort if quitC is readable
+func retrieve(key Key, chunkC chan *Chunk, quitC chan bool) *Chunk {
+	chunk := &Chunk{
+		Key: key,
+		C:   make(chan bool), // close channel to signal data delivery
+	}
+	// glog.V(logger.Detail).Infof("[BZZ] chunk data sent for %v (key interval in chunk %v-%v)", ch.Key.Log(), j*self.chunker.hashSize, (j+1)*self.chunker.hashSize)
+	// submit chunk for retrieval
+	select {
+	case chunkC <- chunk: // submit retrieval request, someone should be listening on the other side (or we will time out globally)
+	case <-quitC:
+		return nil
+	}
+	// waiting for the chunk retrieval
+	select { // chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8]))
+
+	case <-quitC:
+		// this is how we control process leakage (quitC is closed once join is finished (after timeout))
+		return nil
+	case <-chunk.C: // bells are ringing, data have been delivered
+		// glog.V(logger.Detail).Infof("[BZZ] chunk data received")
+	}
+	if len(chunk.SData) == 0 {
+		return nil // chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8]))
+
+	}
+	return chunk
+}
+
+// Read keeps a cursor so cannot be called simulateously, see ReadAt
+func (self *LazyChunkReader) Read(b []byte) (read int, err error) {
+	read, err = self.ReadAt(b, self.off)
+	glog.V(logger.Detail).Infof("[BZZ] read: %v, off: %v, error: %v", read, self.off, err)
+
+	self.off += int64(read)
+	return
+}
+
+// completely analogous to standard SectionReader implementation
+var errWhence = errors.New("Seek: invalid whence")
+var errOffset = errors.New("Seek: invalid offset")
+
+func (s *LazyChunkReader) Seek(offset int64, whence int) (int64, error) {
+	switch whence {
+	default:
+		return 0, errWhence
+	case 0:
+		offset += 0
+	case 1:
+		offset += s.off
+	case 2:
+		if s.chunk == nil {
+			return 0, fmt.Errorf("seek from the end requires rootchunk for size. call Size first")
+		}
+		offset += s.chunk.Size
+	}
+
+	if offset < 0 {
+		return 0, errOffset
+	}
+	s.off = offset
+	return offset, nil
 }

swarm/storage/chunker_test.go

@@ -2,20 +2,33 @@ package storage
 
 import (
 	"bytes"
-	// "fmt"
+	"fmt"
 	"io"
+	"runtime"
+	"sync"
 	"testing"
 	"time"
+
+	"github.com/ethereum/go-ethereum/logger"
+	"github.com/ethereum/go-ethereum/logger/glog"
 )
 
+func init() {
+	glog.SetV(logger.Info)
+	glog.SetToStderr(true)
+}
+
 /*
 Tests TreeChunker by splitting and joining a random byte slice
 */
 
+type test interface {
+	Fatalf(string, ...interface{})
+}
+
 type chunkerTester struct {
-	errors  []error
 	chunks []*Chunk
-	timeout bool
+	t      test
 }
 
 func (self *chunkerTester) checkChunks(t *testing.T, want int) {
@@ -25,77 +38,70 @@ func (self *chunkerTester) checkChunks(t *testing.T, want int) {
 	}
 }
 
-func (self *chunkerTester) Split(chunker *TreeChunker, l int) (key Key, input []byte) {
+func (self *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, chunkC chan *Chunk, swg *sync.WaitGroup) (key Key) {
 	// reset
-	self.errors = nil
 	self.chunks = nil
-	self.timeout = false
-
-	data, slice := testDataReader(l)
-	input = slice
-	key = make([]byte, 32)
-	chunkC := make(chan *Chunk, 1000)
-	errC := chunker.Split(key, data, chunkC, nil)
 	quitC := make(chan bool)
 	timeout := time.After(600 * time.Second)
-
+	if chunkC != nil {
 		go func() {
-	LOOP:
 			for {
 				select {
 				case <-timeout:
-				self.timeout = true
+					self.t.Fatalf("Join timeout error")
-				break LOOP
 
-			case chunk := <-chunkC:
+				case chunk, ok := <-chunkC:
-				if chunk != nil {
-					self.chunks = append(self.chunks, chunk)
-				} else {
-					break LOOP
-				}
-
-			case err, ok := <-errC:
-				if err != nil {
-					self.errors = append(self.errors, err)
-				}
-				// fmt.Printf("err %v", err)
 					if !ok {
-					close(chunkC)
+						// glog.V(logger.Info).Infof("chunkC closed quitting")
-					errC = nil
-				}
-			}
-		}
 						close(quitC)
+						return
+					}
+					// glog.V(logger.Info).Infof("chunk %v received", len(self.chunks))
+					self.chunks = append(self.chunks, chunk)
+					if chunk.wg != nil {
+						chunk.wg.Done()
+					}
+				}
+			}
 		}()
-	<-quitC // waiting for it to finish
+	}
+	key, err := chunker.Split(data, size, chunkC, swg, nil)
+	if err != nil {
+		self.t.Fatalf("Split error: %v", err)
+	}
+	if chunkC != nil {
+		if swg != nil {
+			// glog.V(logger.Info).Infof("Waiting for storage to finish")
+			swg.Wait()
+			// glog.V(logger.Info).Infof("St	orage finished")
+		}
+		close(chunkC)
+	}
+	if chunkC != nil {
+		<-quitC
+	}
 	return
 }
 
-func (self *chunkerTester) Join(chunker *TreeChunker, key Key, c int) SectionReader {
+func (self *chunkerTester) Join(chunker *TreeChunker, key Key, c int, chunkC chan *Chunk, quitC chan bool) LazySectionReader {
 	// reset but not the chunks
-	self.errors = nil
-	self.timeout = false
-	chunkC := make(chan *Chunk, 1000)
 
 	reader := chunker.Join(key, chunkC)
 
-	quitC := make(chan bool)
 	timeout := time.After(600 * time.Second)
 	i := 0
 	go func() {
-	LOOP:
 		for {
 			select {
-			case <-quitC:
-				break LOOP
-
 			case <-timeout:
-				self.timeout = true
+				self.t.Fatalf("Join timeout error")
-				break LOOP
 
-			case chunk := <-chunkC:
+			case chunk, ok := <-chunkC:
+				if !ok {
+					close(quitC)
+					return
+				}
 				i++
-				// dpaLogger.DebugDetailf("TESTER: chunk request %x", chunk.Key[:4])
 				// this just mocks the behaviour of a chunk store retrieval
 				var found bool
 				for _, ch := range self.chunks {
@@ -106,56 +112,58 @@ func (self *chunkerTester) Join(chunker *TreeChunker, key Key, c int) SectionRea
 					}
 				}
 				if !found {
-					// fmt.Printf("TESTER: chunk unknown for %x", chunk.Key[:4])
+					self.t.Fatalf("not found	")
 				}
 				close(chunk.C)
-				// dpaLogger.DebugDetailf("TESTER: chunk request served %x", chunk.Key[:4])
 			}
 		}
 	}()
 	return reader
 }
 
-func testRandomData(chunker *TreeChunker, tester *chunkerTester, n int, chunks int, t *testing.T) {
+func testRandomData(n int, chunks int, t *testing.T) {
-	key, input := tester.Split(chunker, n)
+	chunker := NewTreeChunker(&ChunkerParams{
+		Branches: 128,
+		Hash:     "SHA3",
+	})
+	tester := &chunkerTester{t: t}
+	data, input := testDataReaderAndSlice(n)
 
-	t.Logf(" Key = %x\n", key)
+	chunkC := make(chan *Chunk, 1000)
+	swg := &sync.WaitGroup{}
 
-	tester.checkChunks(t, chunks)
+	splitter := chunker
-	time.Sleep(100 * time.Millisecond)
+	key := tester.Split(splitter, data, int64(n), chunkC, swg)
 
-	reader := tester.Join(chunker, key, 0)
+	// t.Logf(" Key = %v\n", key)
+
+	// tester.checkChunks(t, chunks)
+	chunkC = make(chan *Chunk, 1000)
+	quitC := make(chan bool)
+
+	reader := tester.Join(chunker, key, 0, chunkC, quitC)
 	output := make([]byte, n)
 	r, err := reader.Read(output)
 	if r != n || err != io.EOF {
-		t.Errorf("read error  read: %v  n = %v  err = %v\n", r, n, err)
+		t.Fatalf("read error  read: %v  n = %v  err = %v\n", r, n, err)
 	}
-	// t.Logf(" IN: %x\nOUT: %x\n", input, output)
+	if input != nil {
 		if !bytes.Equal(output, input) {
-		t.Errorf("input and output mismatch\n IN: %x\nOUT: %x\n", input, output)
+			t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", input, output)
 		}
+	}
+	close(chunkC)
+	<-quitC
 }
 
 func TestRandomData(t *testing.T) {
-	chunker, tester := chunkerAndTester()
+	testRandomData(60, 1, t)
-	testRandomData(chunker, tester, 60, 1, t)
+	testRandomData(83, 3, t)
-	testRandomData(chunker, tester, 179, 5, t)
+	testRandomData(179, 5, t)
-	testRandomData(chunker, tester, 253, 7, t)
+	testRandomData(253, 7, t)
-	// t.Logf("chunks %v", tester.chunks)
 }
 
-func chunkerAndTester() (chunker *TreeChunker, tester *chunkerTester) {
+func readAll(reader LazySectionReader, result []byte) {
-	chunker = NewTreeChunker(&ChunkerParams{
-		Branches:     2,
-		Hash:         "SHA256",
-		SplitTimeout: 10,
-		JoinTimeout:  10,
-	})
-	tester = &chunkerTester{}
-	return
-}
-
-func readAll(reader SectionReader, result []byte) {
 	size := int64(len(result))
 
 	var end int64
@@ -169,46 +177,98 @@ func readAll(reader SectionReader, result []byte) {
 	}
 }
 
-func benchReadAll(reader SectionReader) {
+func benchReadAll(reader LazySectionReader) {
-	size := reader.Size()
+	size, _ := reader.Size(nil)
 	output := make([]byte, 1000)
 	for pos := int64(0); pos < size; pos += 1000 {
 		reader.ReadAt(output, pos)
 	}
 }
 
-func benchmarkJoinRandomData(n int, chunks int, t *testing.B) {
+func benchmarkJoin(n int, t *testing.B) {
-	t.StopTimer()
 	for i := 0; i < t.N; i++ {
-		// fmt.Printf("round %v\n", i)
+		chunker := NewTreeChunker(&ChunkerParams{
-		chunker, tester := chunkerAndTester()
+			Branches: 128,
-		key, _ := tester.Split(chunker, n)
+			Hash:     "SHA3",
-		// fmt.Printf("split done %v, joining...\n", i)
+		})
+		tester := &chunkerTester{t: t}
+		data := testDataReader(n)
+
+		chunkC := make(chan *Chunk, 1000)
+		swg := &sync.WaitGroup{}
+
+		key := tester.Split(chunker, data, int64(n), chunkC, swg)
 		t.StartTimer()
-		reader := tester.Join(chunker, key, i)
+		chunkC = make(chan *Chunk, 1000)
-		// fmt.Printf("join done %v, reading...\n", i)
+		quitC := make(chan bool)
+		reader := tester.Join(chunker, key, i, chunkC, quitC)
+		t.StopTimer()
 		benchReadAll(reader)
+		close(chunkC)
+		<-quitC
 	}
 }
 
-func benchmarkSplitRandomData(n int, chunks int, t *testing.B) {
+func benchmarkSplitTree(n int, t *testing.B) {
+	t.ReportAllocs()
 	for i := 0; i < t.N; i++ {
-		chunker, tester := chunkerAndTester()
+		chunker := NewTreeChunker(&ChunkerParams{
-		tester.Split(chunker, n)
+			Branches: 128,
+			Hash:     "SHA3",
+		})
+		tester := &chunkerTester{t: t}
+		data := testDataReader(n)
+		// glog.V(logger.Info).Infof("splitting data of length %v", n)
+		tester.Split(chunker, data, int64(n), nil, nil)
 	}
+	stats := new(runtime.MemStats)
+	runtime.ReadMemStats(stats)
+	fmt.Println(stats.Sys)
 }
 
-func BenchmarkJoinRandomData_100_2(t *testing.B)     { benchmarkJoinRandomData(100, 3, t) }
+func benchmarkSplitPyramid(n int, t *testing.B) {
-func BenchmarkJoinRandomData_1000_2(t *testing.B)    { benchmarkJoinRandomData(1000, 3, t) }
+	t.ReportAllocs()
-func BenchmarkJoinRandomData_10000_2(t *testing.B)   { benchmarkJoinRandomData(10000, 3, t) }
+	for i := 0; i < t.N; i++ {
-func BenchmarkJoinRandomData_100000_2(t *testing.B)  { benchmarkJoinRandomData(100000, 3, t) }
+		splitter := NewPyramidChunker(&ChunkerParams{
-func BenchmarkJoinRandomData_1000000_2(t *testing.B) { benchmarkJoinRandomData(1000000, 3, t) }
+			Branches: 128,
+			Hash:     "SHA3",
+		})
+		tester := &chunkerTester{t: t}
+		data := testDataReader(n)
+		// glog.V(logger.Info).Infof("splitting data of length %v", n)
+		tester.Split(splitter, data, int64(n), nil, nil)
+	}
+	stats := new(runtime.MemStats)
+	runtime.ReadMemStats(stats)
+	fmt.Println(stats.Sys)
+}
 
-func BenchmarkSplitRandomData_100_2(t *testing.B)      { benchmarkSplitRandomData(100, 3, t) }
+func BenchmarkJoin_100_2(t *testing.B)     { benchmarkJoin(100, t) }
-func BenchmarkSplitRandomData_1000_2(t *testing.B)     { benchmarkSplitRandomData(1000, 3, t) }
+func BenchmarkJoin_1000_2(t *testing.B)    { benchmarkJoin(1000, t) }
-func BenchmarkSplitRandomData_10000_2(t *testing.B)    { benchmarkSplitRandomData(10000, 3, t) }
+func BenchmarkJoin_10000_2(t *testing.B)   { benchmarkJoin(10000, t) }
-func BenchmarkSplitRandomData_100000_2(t *testing.B)   { benchmarkSplitRandomData(100000, 3, t) }
+func BenchmarkJoin_100000_2(t *testing.B)  { benchmarkJoin(100000, t) }
-func BenchmarkSplitRandomData_1000000_2(t *testing.B)  { benchmarkSplitRandomData(1000000, 3, t) }
+func BenchmarkJoin_1000000_2(t *testing.B) { benchmarkJoin(1000000, t) }
-func BenchmarkSplitRandomData_10000000_2(t *testing.B) { benchmarkSplitRandomData(10000000, 3, t) }
 
-// go test -bench ./bzz -cpuprofile cpu.out -memprofile mem.out
+func BenchmarkSplitTree_2(t *testing.B)  { benchmarkSplitTree(100, t) }
+func BenchmarkSplitTree_2h(t *testing.B) { benchmarkSplitTree(500, t) }
+func BenchmarkSplitTree_3(t *testing.B)  { benchmarkSplitTree(1000, t) }
+func BenchmarkSplitTree_3h(t *testing.B) { benchmarkSplitTree(5000, t) }
+func BenchmarkSplitTree_4(t *testing.B)  { benchmarkSplitTree(10000, t) }
+func BenchmarkSplitTree_4h(t *testing.B) { benchmarkSplitTree(50000, t) }
+func BenchmarkSplitTree_5(t *testing.B)  { benchmarkSplitTree(100000, t) }
+func BenchmarkSplitTree_6(t *testing.B)  { benchmarkSplitTree(1000000, t) }
+func BenchmarkSplitTree_7(t *testing.B)  { benchmarkSplitTree(10000000, t) }
+func BenchmarkSplitTree_8(t *testing.B)  { benchmarkSplitTree(100000000, t) }
+
+func BenchmarkSplitPyramid_2(t *testing.B)  { benchmarkSplitPyramid(100, t) }
+func BenchmarkSplitPyramid_2h(t *testing.B) { benchmarkSplitPyramid(500, t) }
+func BenchmarkSplitPyramid_3(t *testing.B)  { benchmarkSplitPyramid(1000, t) }
+func BenchmarkSplitPyramid_3h(t *testing.B) { benchmarkSplitPyramid(5000, t) }
+func BenchmarkSplitPyramid_4(t *testing.B)  { benchmarkSplitPyramid(10000, t) }
+func BenchmarkSplitPyramid_4h(t *testing.B) { benchmarkSplitPyramid(50000, t) }
+func BenchmarkSplitPyramid_5(t *testing.B)  { benchmarkSplitPyramid(100000, t) }
+func BenchmarkSplitPyramid_6(t *testing.B)  { benchmarkSplitPyramid(1000000, t) }
+func BenchmarkSplitPyramid_7(t *testing.B)  { benchmarkSplitPyramid(10000000, t) }
+func BenchmarkSplitPyramid_8(t *testing.B)  { benchmarkSplitPyramid(100000000, t) }
+
+// godep go test -bench ./swarm/storage -cpuprofile cpu.out -memprofile mem.out

swarm/storage/chunkreader.go

@@ -1,194 +0,0 @@
-package storage
-
-import (
-	"bytes"
-	"errors"
-	"io"
-)
-
-type Bounded interface {
-	Size() int64
-}
-
-type Sliced interface {
-	Slice(int64, int64) (b []byte, err error)
-}
-
-// Size, Seek, Read, ReadAt
-type SectionReader interface {
-	Bounded
-	io.Seeker
-	io.Reader
-	io.ReaderAt
-}
-
-// ChunkReader implements SectionReader on a section
-// of an underlying ReaderAt.
-type ChunkReader struct {
-	r     io.ReaderAt
-	base  int64
-	off   int64
-	limit int64
-}
-
-// NewChunkReader returns a ChunkReader that reads from r
-// starting at offset off and stops with EOF after n bytes.
-func NewChunkReader(r io.ReaderAt, off int64, n int64) *ChunkReader {
-	return &ChunkReader{r: r, base: off, off: off, limit: off + n}
-}
-
-// ByteSliceReader just extends byte.Reader to make base slice accessible
-type ByteSliceReader struct {
-	*bytes.Reader
-	base []byte
-}
-
-func NewByteSliceReader(b []byte) *ByteSliceReader {
-	return &ByteSliceReader{
-		base:   b,
-		Reader: bytes.NewReader(b),
-	}
-}
-
-// ByteSliceReader implements the Sliced interface
-func (self *ByteSliceReader) Slice(from, to int64) (b []byte, err error) {
-	if from < 0 || to >= int64(self.Len()) {
-		err = io.EOF
-	} else {
-		b = self.base[from:to]
-	}
-	return
-}
-
-// NewChunkReaderFromBytes is a convenience shortcut to get a SectionReader over a byte slice
-func NewChunkReaderFromBytes(b []byte) *ChunkReader {
-	return NewChunkReader(NewByteSliceReader(b), 0, int64(len(b)))
-}
-
-/*
-The following is adapted from io.SectionReader
-*/
-
-func (s *ChunkReader) Size() int64 {
-	return s.limit - s.base
-}
-
-var errWhence = errors.New("Seek: invalid whence")
-var errOffset = errors.New("Seek: invalid offset")
-
-func (s *ChunkReader) Seek(offset int64, whence int) (int64, error) {
-	switch whence {
-	default:
-		return 0, errWhence
-	case 0:
-		offset += s.base
-	case 1:
-		offset += s.off
-	case 2:
-		offset += s.limit
-	}
-	if offset < s.base {
-		return 0, errOffset
-	}
-	s.off = offset
-	return offset - s.base, nil
-}
-
-func (s *ChunkReader) Read(p []byte) (n int, err error) {
-	if s.off >= s.limit {
-		return 0, io.EOF
-	}
-	if max := s.limit - s.off; int64(len(p)) > max {
-		p = p[0:max]
-	}
-	n, err = s.r.ReadAt(p, s.off)
-	s.off += int64(n)
-	return
-}
-
-func (s *ChunkReader) ReadAt(p []byte, off int64) (n int, err error) {
-	if off < 0 || off >= s.limit-s.base {
-		return 0, io.EOF
-	}
-	off += s.base
-	if max := s.limit - off; int64(len(p)) > max {
-		p = p[0:max]
-		n, err = s.r.ReadAt(p, off)
-		if err == nil {
-			err = io.EOF
-		}
-		return n, err
-	}
-	n, err = s.r.ReadAt(p, off)
-	return
-}
-
-// added methods to that ChunkReader implements the Sliced interface
-func (s *ChunkReader) Slice(from, to int64) (b []byte, err error) {
-	if from < 0 || to >= s.Size() {
-		err = io.EOF
-	} else {
-		if sl, ok := s.r.(Sliced); ok {
-			b, err = sl.Slice(s.base+from, s.base+to)
-		} else {
-			err = errors.New("not sliceable base")
-		}
-	}
-	return
-}
-
-// added method so that ChunkReader implements the io.WriterTo interface
-// WriteTo method is used by io.Copy
-// This is so that we avoid one extra step of allocation (if the underlying initial Reader implements Sliced
-func (r *ChunkReader) WriteTo(w io.Writer) (n int64, err error) {
-	var b []byte
-	var m int
-	// if b, _ := r.Slice(r.off-r.base, r.limit-r.base); b == nil {
-	// if slices not available we do it with extra allocation
-	b = make([]byte, r.limit-r.off)
-	m, err = r.Read(b)
-	if err != nil {
-		return
-	}
-	// }
-	m, err = w.Write(b)
-	if m > len(b) {
-		panic("bytes.Reader.WriteTo: invalid Write count")
-	}
-	r.off = r.base + int64(m)
-	n = int64(m)
-	if m != len(b) && err == nil {
-		err = io.ErrShortWrite
-	}
-	// w
-	return
-}
-
-func (self *LazyChunkReader) Size() (n int64) {
-	self.ReadAt(nil, 0)
-	return self.size
-}
-
-func (self *LazyChunkReader) Read(b []byte) (read int, err error) {
-	read, err = self.ReadAt(b, self.off)
-	self.off += int64(read)
-	return
-}
-
-func (s *LazyChunkReader) Seek(offset int64, whence int) (int64, error) {
-	switch whence {
-	default:
-		return 0, errWhence
-	case 0:
-		offset += 0
-	case 1:
-		offset += s.off
-	case 2:
-		offset += s.size
-	}
-	if offset < 0 {
-		return 0, errOffset
-	}
-	s.off = offset
-	return offset, nil
-}

swarm/storage/common_test.go

@@ -1,6 +1,7 @@
 package storage
 
 import (
+	"bytes"
 	"crypto/rand"
 	"io"
 	"sync"
@@ -10,61 +11,39 @@ import (
 	"github.com/ethereum/go-ethereum/logger/glog"
 )
 
-func testDataReader(l int) (r *ChunkReader, slice []byte) {
+func testDataReader(l int) (r io.Reader) {
+	return io.LimitReader(rand.Reader, int64(l))
+}
+
+func testDataReaderAndSlice(l int) (r io.Reader, slice []byte) {
 	slice = make([]byte, l)
 	if _, err := rand.Read(slice); err != nil {
 		panic("rand error")
 	}
-	r = NewChunkReaderFromBytes(slice)
+	r = bytes.NewReader(slice)
-	return
-}
-
-func randomChunks(l int64, branches int64, chunkC chan *Chunk) (key Key, errC chan error) {
-	chunker := NewTreeChunker(&ChunkerParams{
-		Branches:     branches,
-		Hash:         defaultHash,
-		SplitTimeout: splitTimeout,
-	})
-	key = make([]byte, 32)
-	b := make([]byte, l)
-	_, err := rand.Read(b)
-	if err != nil {
-		panic("no rand")
-	}
-	wg := &sync.WaitGroup{}
-	errC = chunker.Split(key, NewChunkReaderFromBytes(b), chunkC, wg)
-	wg.Wait()
 	return
 }
 
 func testStore(m ChunkStore, l int64, branches int64, t *testing.T) {
 
 	chunkC := make(chan *Chunk)
-	key, errC := randomChunks(l, branches, chunkC)
+	go func() {
-
+		for chunk := range chunkC {
-SPLIT:
-	for {
-		select {
-		case chunk := <-chunkC:
 			m.Put(chunk)
-		case err, ok := <-errC:
+			if chunk.wg != nil {
-			if err != nil {
+				chunk.wg.Done()
-				t.Errorf("Chunker error: %v", err)
-				return
-			}
-			if !ok {
-				break SPLIT
-			}
 			}
 		}
+	}()
 	chunker := NewTreeChunker(&ChunkerParams{
 		Branches: branches,
 		Hash:     defaultHash,
-		SplitTimeout: splitTimeout,
 	})
+	swg := &sync.WaitGroup{}
+	key, err := chunker.Split(rand.Reader, l, chunkC, swg, nil)
+	swg.Wait()
+	close(chunkC)
 	chunkC = make(chan *Chunk)
-	var r SectionReader
-	r = chunker.Join(key, chunkC)
 
 	quit := make(chan bool)
 
@@ -73,22 +52,26 @@ SPLIT:
 			go func(chunk *Chunk) {
 				storedChunk, err := m.Get(chunk.Key)
 				if err == notFound {
-					glog.V(logger.Detail).Infof("[BZZ] chunk '%x' not found", chunk.Key)
+					glog.V(logger.Detail).Infof("[BZZ] chunk '%v' not found", chunk.Key.Log())
 				} else if err != nil {
-					glog.V(logger.Detail).Infof("[BZZ] error retrieving chunk %x: %v", chunk.Key, err)
+					glog.V(logger.Detail).Infof("[BZZ] error retrieving chunk %v: %v", chunk.Key.Log(), err)
 				} else {
 					chunk.SData = storedChunk.SData
+					chunk.Size = storedChunk.Size
 				}
-				glog.V(logger.Detail).Infof("[BZZ] chunk '%x' not found", chunk.Key[:4])
+				glog.V(logger.Detail).Infof("[BZZ] chunk '%v' not found", chunk.Key.Log())
 				close(chunk.C)
 			}(ch)
 		}
+		close(quit)
 	}()
+	r := chunker.Join(key, chunkC)
 
 	b := make([]byte, l)
 	n, err := r.ReadAt(b, 0)
 	if err != io.EOF {
-		t.Errorf("read error (%v/%v) %v", n, l, err)
+		t.Fatalf("read error (%v/%v) %v", n, l, err)
-		close(quit)
 	}
+	close(chunkC)
+	<-quit
 }

swarm/storage/dpa.go

@@ -2,6 +2,7 @@ package storage
 
 import (
 	"errors"
+	"io"
 	"sync"
 	"time"
 
@@ -72,33 +73,14 @@ func NewDPA(store ChunkStore, params *ChunkerParams) *DPA {
 // FS-aware API and httpaccess
 // Chunk retrieval blocks on netStore requests with a timeout so reader will
 // report error if retrieval of chunks within requested range time out.
-func (self *DPA) Retrieve(key Key) SectionReader {
+func (self *DPA) Retrieve(key Key) LazySectionReader {
 	return self.Chunker.Join(key, self.retrieveC)
 }
 
 // Public API. Main entry point for document storage directly. Used by the
 // FS-aware API and httpaccess
-func (self *DPA) Store(data SectionReader, wg *sync.WaitGroup) (key Key, err error) {
+func (self *DPA) Store(data io.Reader, size int64, wg *sync.WaitGroup) (key Key, err error) {
-	key = make([]byte, self.Chunker.KeySize())
+	return self.Chunker.Split(data, size, self.storeC, nil, wg)
-	errC := self.Chunker.Split(key, data, self.storeC, wg)
-
-SPLIT:
-	for {
-		select {
-		case err, ok := <-errC:
-			if err != nil {
-				glog.V(logger.Error).Infof("[BZZ] chunker split error: %v", err)
-			}
-			if !ok {
-				break SPLIT
-			}
-
-		case <-self.quitC:
-			break SPLIT
-		}
-	}
-	return
-
 }
 
 func (self *DPA) Start() {
@@ -164,7 +146,7 @@ func (self *DPA) storeLoop() {
 			go func(chunk *Chunk) {
 				self.Put(chunk)
 				if chunk.wg != nil {
-					glog.V(logger.Detail).Infof("[BZZ] DPA.storeLoop %v", chunk.Key.Log())
+					glog.V(logger.Detail).Infof("[BZZ] dpa: store loop %v", chunk.Key.Log())
 					chunk.wg.Done()
 				}
 			}(ch)

swarm/storage/dpa_test.go

@@ -29,9 +29,9 @@ func TestDPArandom(t *testing.T) {
 		ChunkStore: localStore,
 	}
 	dpa.Start()
-	reader, slice := testDataReader(testDataSize)
+	reader, slice := testDataReaderAndSlice(testDataSize)
 	wg := &sync.WaitGroup{}
-	key, err := dpa.Store(reader, wg)
+	key, err := dpa.Store(reader, testDataSize, wg)
 	if err != nil {
 		t.Errorf("Store error: %v", err)
 	}
@@ -85,9 +85,9 @@ func TestDPA_capacity(t *testing.T) {
 		ChunkStore: localStore,
 	}
 	dpa.Start()
-	reader, slice := testDataReader(testDataSize)
+	reader, slice := testDataReaderAndSlice(testDataSize)
 	wg := &sync.WaitGroup{}
-	key, err := dpa.Store(reader, wg)
+	key, err := dpa.Store(reader, testDataSize, wg)
 	if err != nil {
 		t.Errorf("Store error: %v", err)
 	}

swarm/storage/localstore.go

@@ -1,5 +1,9 @@
 package storage
 
+import (
+	"encoding/binary"
+)
+
 // LocalStore is a combination of inmemory db over a disk persisted db
 // implements a Get/Put with fallback (caching) logic using any 2 ChunkStores
 type LocalStore struct {
@@ -48,6 +52,7 @@ func (self *LocalStore) Get(key Key) (chunk *Chunk, err error) {
 	if err != nil {
 		return
 	}
+	chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8]))
 	self.memStore.Put(chunk)
 	return
 }

swarm/storage/memstore.go

@@ -3,7 +3,6 @@
 package storage
 
 import (
-	"bytes"
 	"sync"
 )
 
@@ -44,41 +43,6 @@ func NewMemStore(d *DbStore, capacity uint) (m *MemStore) {
 	return
 }
 
-func (x Key) Size() uint {
-	return uint(len(x))
-}
-
-func (x Key) isEqual(y Key) bool {
-	return bytes.Compare(x, y) == 0
-}
-
-func (h Key) bits(i, j uint) uint {
-	ii := i >> 3
-	jj := i & 7
-	if ii >= h.Size() {
-		return 0
-	}
-
-	if jj+j <= 8 {
-		return uint((h[ii] >> jj) & ((1 << j) - 1))
-	}
-
-	res := uint(h[ii] >> jj)
-	jj = 8 - jj
-	j -= jj
-	for j != 0 {
-		ii++
-		if j < 8 {
-			res += uint(h[ii]&((1<<j)-1)) << jj
-			return res
-		}
-		res += uint(h[ii]) << jj
-		jj += 8
-		j -= 8
-	}
-	return res
-}
-
 type memTree struct {
 	subtree   []*memTree
 	parent    *memTree

swarm/storage/netstore.go

@@ -91,7 +91,8 @@ func (self *NetStore) Put(entry *Chunk) {
 	} else {
 		glog.V(logger.Detail).Infof("[BZZ] NetStore.Put: localStore.Put %v stored locally", entry.Key.Log())
 		// handle propagating store requests
-		go self.cloud.Store(entry)
+		// go self.cloud.Store(entry)
+		self.cloud.Store(entry)
 	}
 }
 

swarm/storage/pyramid.go

@@ -0,0 +1,165 @@
+package storage
+
+import (
+	"io"
+	"math"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/logger"
+	"github.com/ethereum/go-ethereum/logger/glog"
+)
+
+const (
+	processors = 8
+)
+
+type Tree struct {
+	Chunks int64
+	Levels []map[int64]*Node
+	Lock   sync.RWMutex
+}
+
+type Node struct {
+	Pending  int64
+	Children []common.Hash
+	Last     bool
+}
+
+type Task struct {
+	Index int64  // Index of the chunk being processed
+	Data  []byte // Binary blob of the chunk
+	Last  bool
+}
+
+type PyramidChunker struct {
+	hashFunc    Hasher
+	chunkSize   int64
+	hashSize    int64
+	branches    int64
+	workerCount int
+}
+
+func NewPyramidChunker(params *ChunkerParams) (self *PyramidChunker) {
+	self = &PyramidChunker{}
+	self.hashFunc = MakeHashFunc(params.Hash)
+	self.branches = params.Branches
+	self.hashSize = int64(self.hashFunc().Size())
+	self.chunkSize = self.hashSize * self.branches
+	self.workerCount = 1
+	return
+}
+
+func (self *PyramidChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, swg, wwg *sync.WaitGroup) (Key, error) {
+
+	chunks := (size + self.chunkSize - 1) / self.chunkSize
+	depth := int(math.Ceil(math.Log(float64(chunks))/math.Log(float64(self.branches)))) + 1
+	glog.V(logger.Detail).Infof("chunks: %v, depth: %v", chunks, depth)
+
+	results := Tree{
+		Chunks: chunks,
+		Levels: make([]map[int64]*Node, depth),
+	}
+	for i := 0; i < depth; i++ {
+		results.Levels[i] = make(map[int64]*Node)
+	}
+	// Create a pool of workers to crunch through the file
+	tasks := make(chan *Task, 2*processors)
+	pend := new(sync.WaitGroup)
+	abortC := make(chan bool)
+	for i := 0; i < processors; i++ {
+		pend.Add(1)
+		go self.processor(pend, tasks, &results)
+	}
+	// Feed the chunks into the task pool
+	for index := 0; ; index++ {
+		buffer := make([]byte, self.chunkSize+8)
+		n, err := io.ReadFull(data, buffer)
+		last := err == io.ErrUnexpectedEOF
+		if err != nil && !last {
+			glog.V(logger.Info).Infof("error: %v", err)
+
+			close(abortC)
+		}
+		pend.Add(1)
+		// glog.V(logger.Info).Infof("-> task %v (%v)", index, n)
+		select {
+		case tasks <- &Task{Index: int64(index), Data: buffer[:n+8], Last: last}:
+		case <-abortC:
+			return nil, err
+		}
+		if last {
+			// glog.V(logger.Info).Infof("last task %v (%v)", index, n)
+			break
+		}
+	}
+	// Wait for the workers and return
+	close(tasks)
+	pend.Wait()
+
+	// glog.V(logger.Info).Infof("len: %v", results.Levels[0][0])
+	key := results.Levels[0][0].Children[0][:]
+	return key, nil
+}
+
+func (self *PyramidChunker) processor(pend *sync.WaitGroup, tasks chan *Task, results *Tree) {
+	defer pend.Done()
+
+	// glog.V(logger.Info).Infof("processor started")
+	// Start processing leaf chunks ad infinitum
+	hasher := self.hashFunc()
+	for task := range tasks {
+		depth, pow := len(results.Levels)-1, self.branches
+		// glog.V(logger.Info).Infof("task: %v, last: %v", task.Index, task.Last)
+
+		var node *Node
+		for depth >= 0 {
+			// New chunk received, reset the hasher and start processing
+			hasher.Reset()
+
+			if node == nil { // Leaf node, hash the data chunk
+				hasher.Write(task.Data)
+			} else { // Internal node, hash the children
+				for _, hash := range node.Children {
+					hasher.Write(hash[:])
+				}
+			}
+			hash := hasher.Sum(nil)
+			last := task.Last || (node != nil) && node.Last
+			// Insert the subresult into the memoization tree
+			results.Lock.Lock()
+			if node = results.Levels[depth][task.Index/pow]; node == nil {
+				// Figure out the pending tasks
+				pending := self.branches
+				if task.Index/pow == results.Chunks/pow {
+					pending = (results.Chunks + pow/self.branches - 1) / (pow / self.branches) % self.branches
+				}
+				node = &Node{pending, make([]common.Hash, pending), last}
+				results.Levels[depth][task.Index/pow] = node
+			}
+			node.Pending--
+			i := task.Index / (pow / self.branches) % self.branches
+			if last {
+				node.Pending -= self.branches - i
+				node.Children = node.Children[:i+1]
+				node.Last = true
+			}
+			copy(node.Children[i][:], hash)
+			left := node.Pending
+
+			if depth+1 < len(results.Levels) {
+				delete(results.Levels[depth+1], task.Index/(pow/self.branches))
+			}
+			results.Lock.Unlock()
+			// If there's more work to be done, leave for others
+			// glog.V(logger.Info).Infof("left %v", left)
+			if left > 0 {
+				break
+			}
+			// We're the last ones in this batch, merge the children together
+			depth--
+			pow *= self.branches
+		}
+		pend.Done()
+	}
+}

swarm/storage/types.go

@@ -5,6 +5,7 @@ import (
 	"crypto"
 	"fmt"
 	"hash"
+	"io"
 	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -13,10 +14,47 @@ import (
 
 type Hasher func() hash.Hash
 
+// Peer is the recorded as Source on the chunk
+// should probably not be here? but network should wrap chunk object
 type Peer interface{}
 
 type Key []byte
 
+func (x Key) Size() uint {
+	return uint(len(x))
+}
+
+func (x Key) isEqual(y Key) bool {
+	return bytes.Compare(x, y) == 0
+}
+
+func (h Key) bits(i, j uint) uint {
+	ii := i >> 3
+	jj := i & 7
+	if ii >= h.Size() {
+		return 0
+	}
+
+	if jj+j <= 8 {
+		return uint((h[ii] >> jj) & ((1 << j) - 1))
+	}
+
+	res := uint(h[ii] >> jj)
+	jj = 8 - jj
+	j -= jj
+	for j != 0 {
+		ii++
+		if j < 8 {
+			res += uint(h[ii]&((1<<j)-1)) << jj
+			return res
+		}
+		res += uint(h[ii]) << jj
+		jj += 8
+		j -= 8
+	}
+	return res
+}
+
 func IsZeroKey(key Key) bool {
 	return len(key) == 0 || bytes.Equal(key, ZeroKey)
 }
@@ -127,7 +165,7 @@ After getting notified that all the data has been split (the error channel is cl
 
 When calling Join with a root key, the caller gets returned a seekable lazy reader. The caller again provides a channel on which the caller receives placeholder chunks with missing data. The DPA is supposed to forward this to the chunk stores and notify the chunker if the data has been delivered (i.e. retrieved from memory cache, disk-persisted db or cloud based swarm delivery). As the seekable reader is used, the chunker then puts these together the relevant parts on demand.
 */
-type Chunker interface {
+type Splitter interface {
 	/*
 	   When splitting, data is given as a SectionReader, and the key is a hashSize long byte slice (Key), the root hash of the entire content will fill this once processing finishes.
 	   New chunks to store are coming to caller via the chunk storage channel, which the caller provides.
@@ -135,7 +173,10 @@ type Chunker interface {
 	   The caller gets returned an error channel, if an error is encountered during splitting, it is fed to errC error channel.
 	   A closed error signals process completion at which point the key can be considered final if there were no errors.
 	*/
-	Split(key Key, data SectionReader, chunkC chan *Chunk, wg *sync.WaitGroup) chan error
+	Split(io.Reader, int64, chan *Chunk, *sync.WaitGroup, *sync.WaitGroup) (Key, error)
+}
+
+type Joiner interface {
 	/*
 	   Join reconstructs original content based on a root key.
 	   When joining, the caller gets returned a Lazy SectionReader, which is
@@ -148,8 +189,28 @@ type Chunker interface {
 	   The chunks are not meant to be validated by the chunker when joining. This
 	   is because it is left to the DPA to decide which sources are trusted.
 	*/
-	Join(key Key, chunkC chan *Chunk) SectionReader
+	Join(key Key, chunkC chan *Chunk) LazySectionReader
-
+}
-	// returns the key length
+
-	KeySize() int64
+type Chunker interface {
+	Joiner
+	Splitter
+	// returns the key length
+	// KeySize() int64
+}
+
+// Size, Seek, Read, ReadAt
+type LazySectionReader interface {
+	Size(chan bool) (int64, error)
+	io.Seeker
+	io.Reader
+	io.ReaderAt
+}
+
+type LazyTestSectionReader struct {
+	*io.SectionReader
+}
+
+func (self *LazyTestSectionReader) Size(chan bool) (int64, error) {
+	return self.SectionReader.Size(), nil
 }

swarm/test/connections/00.sh

@@ -1,33 +1,25 @@
 #!/bin/bash
 
-dir=`dirname $0`
-source $dir/../../cmd/swarm/test.sh
-
 swarm init 4
 echo "expect each node to have 3 peers"
-cmd="'net.peerCount'"
+cmd="net.peerCount"
 sleep 5
-swarm attach 00 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
+swarm execute 00  "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
-swarm attach 01 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
+swarm execute 01  "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
-swarm attach 02 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
+swarm execute 02  "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
-swarm attach 03 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
+swarm execute 03  "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
 
 swarm stop all
 
-echo "after static nodes is deleted, connections are recovered from kaddb in bzz-peers.json"
+echo "connections are recovered from kaddb in bzz-peers.json"
-# echo rm -rf $DATA_ROOT/enodes\*
-# echo rm -rf $DATA_ROOT/data/\*/static-nodes.json
-rm -rf $DATA_ROOT/enodes*
-rm -rf $DATA_ROOT/data/*/static-nodes.json
 
 swarm cluster 4
 echo "expect each node to have 3 peers"
-cmd="'net.peerCount'"
+sleep 5
-sleep 10
+swarm execute 00  "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
-swarm attach 00 --exec "$cmd" |tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
+swarm execute 01  "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
-swarm attach 01 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
+swarm execute 02  "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
-swarm attach 02 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
+swarm execute 03  "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
-swarm attach 03 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
 
 swarm stop all
 

swarm/test/swap/00.sh

@@ -4,7 +4,7 @@ echo " two nodes that do not sync and do not have any funds"
 echo " cannot retrieve content from each other"
 
 dir=`dirname $0`
-source $dir/../../cmd/swarm/test.sh
+source $dir/../test.sh
 
 FILE_00=/tmp/1K.0
 randomfile 1 > $FILE_00

swarm/test/swap/01.sh

@@ -3,19 +3,20 @@ echo " two nodes that do not sync but have enough funds"
 echo " can retrieve content from each other"
 
 dir=`dirname $0`
-source $dir/../../cmd/swarm/test.sh
+source $dir/..s/test.sh
 
 file=/tmp/test.file
 mininginterval=120
 key=/tmp/key
-logargs="--verbosity=0 --vmodule='swarm/*=6'"
+# logargs="--verbosity=0 --vmodule='swarm/*=6'"
-# logargs='--verbosity=6'
+logargs='--verbosity=6'
 
+# swarm init 2 --mine --bzznosync --bzznoswap=false $logargsc
+# echo "Mining some ether..."
+# sleep $mininginterval
 
-swarm init 2 --mine --bzznosync $logargs
+swarm cluster 2 --mine --bzznosync --bzznoswap=false $logargsc
 
-echo "Mining some ether..."
-sleep $mininginterval
 
 randomfile 10 > $file
 swarm up 00 $file|tail -n1 > $key

swarm/test/syncing/00.sh

@@ -4,7 +4,7 @@ echo " two nodes that sync (no swap and do not have any funds)"
 echo " can be in sync content with each other"
 
 dir=`dirname $0`
-source $dir/../../cmd/swarm/test.sh
+source $dir/../test.sh
 
 mkdir -p /tmp/swarm-test-files
 FILE_00=/tmp/swarm-test-files/00
@@ -28,7 +28,6 @@ swarm needs 00 $key $FILE_00
 swarm needs 01 $key $FILE_00
 swarm stop 01
 
-# exit 1;
 
 swarm up 00 $FILE_01|tail -n1 > $key
 swarm needs 00 $key $FILE_01
@@ -46,7 +45,8 @@ swarm needs 00 $key $FILE_03
 swarm stop 00
 swarm up 01 $FILE_04|tail -n1 > $key
 swarm needs 01 $key $FILE_04
-swarm start 00 #--bzznoswap
+swarm start 00
+sleep $wait
 swarm needs 00 $key $FILE_04
 
 swarm stop all

swarm/test/syncing/01.sh

@@ -4,7 +4,7 @@ echo " two nodes that do not have any funds"
 echo " can still sync content with each other"
 
 dir=`dirname $0`
-source $dir/../../cmd/swarm/test.sh
+source $dir/../test.sh
 key=/tmp/key
 
 long=/tmp/10M

swarm/test/syncing/02.sh

@@ -5,26 +5,26 @@ echo " two nodes that sync (no swap and do not have any funds)"
 echo " can sync content with each other even with intermittent network connection"
 
 dir=`dirname $0`
-source $dir/../../cmd/swarm/test.sh
+source $dir/../test.sh
 
 long=/tmp/10M
 key=/tmp/key
-randomfile 10000 > $long
+randomfile 100000 > $long
 ls -l $long
 
-swarm init 2
+swarm init 2 --vmodule='swarm/*=5'
-sleep $wait
 swarm up 00 $long |tail -n1 > $key &
-sleep $wait
+sleep 1
-swarm attach 01 -exec "'bzz.blockNetworkRead(true)'"
+swarm execute 01 'bzz.blockNetworkRead(true)'
-sleep $wait
+sleep 3
-swarm attach 01 -exec "'bzz.blockNetworkRead(false)'"
+swarm execute 01 'bzz.blockNetworkRead(false)'
-sleep $wait
+# sleep $wait
-swarm attach 01 -exec "'bzz.blockNetworkRead(true)'"
+# swarm attach 01 -exec "'bzz.blockNetworkRead(true)'"
-sleep $wait
+# sleep $wait
 swarm stop 01
 
-swarm start 01
+# swarm start 01
-swarm needs 01 $key $long
+# sleep $wait
-
+# swarm needs 01 $key $long
+# sleep 3
 swarm stop all

swarm/test/test.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+TEST_DIR=`dirname $0`
+TEST_NAME=`basename $0 .sh`
+TEST_TYPE=`basename $TEST_DIR`
+export IP_ADDR="[::]"
+
+
+export SWARM_NETWORK_ID=322$TEST_NAME
+export SWARM_DIR=~/bzz/test/$TEST_TYPE
+
+rm -rf $SWARM_DIR/$SWARM_NETWORK_ID
+
+wait=1
+
+
+
+function randomfile {
+  dd if=/dev/urandom of=/dev/stdout bs=1024 count=$1 2>/dev/null
+}