forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-25 06:06:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge ad6f7d5d09 into e2853948da

Browse source
This commit is contained in:
Viktor Trón 2016-07-08 03:45:01 +00:00 committed by GitHub
commit 132c1d0e31
41 changed files with 2247 additions and 1628 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
swarm/api/api.go
View file

@ -43,12 +43,12 @@ func NewApi(dpa *storage.DPA, dns Resolver) (self *Api) {
}
// DPA reader API
func (self *Api) Retrieve(key storage.Key) storage.SectionReader {
func (self *Api) Retrieve(key storage.Key) storage.LazySectionReader {
return self.dpa.Retrieve(key)
}
func (self *Api) Store(data storage.SectionReader, wg *sync.WaitGroup) (key storage.Key, err error) {
return self.dpa.Store(data, wg)
func (self *Api) Store(data io.Reader, size int64, wg *sync.WaitGroup) (key storage.Key, err error) {
return self.dpa.Store(data, size, wg)
}
type ErrResolve error
@ -105,15 +105,15 @@ func (self *Api) parseAndResolve(uri string, nameresolver bool) (contentHash sto
// Put provides singleton manifest creation on top of dpa store
func (self *Api) Put(content, contentType string) (string, error) {
sr := io.NewSectionReader(strings.NewReader(content), 0, int64(len(content)))
r := strings.NewReader(content)
wg := &sync.WaitGroup{}
key, err := self.dpa.Store(sr, wg)
key, err := self.dpa.Store(r, int64(len(content)), wg)
if err != nil {
return "", err
}
manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType)
sr = io.NewSectionReader(strings.NewReader(manifest), 0, int64(len(manifest)))
key, err = self.dpa.Store(sr, wg)
r = strings.NewReader(manifest)
key, err = self.dpa.Store(r, int64(len(manifest)), wg)
if err != nil {
return "", err
}
@ -124,11 +124,11 @@ func (self *Api) Put(content, contentType string) (string, error) {
// Get uses iterative manifest retrieval and prefix matching
// to resolve path to content using dpa retrieve
// it returns a section reader, mimeType, status and an error
func (self *Api) Get(uri string, nameresolver bool) (reader storage.SectionReader, mimeType string, status int, err error) {
func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionReader, mimeType string, status int, err error) {
key, _, path, err := self.parseAndResolve(uri, nameresolver)
trie, err := loadManifest(self.dpa, key)
quitC := make(chan bool)
trie, err := loadManifest(self.dpa, key, quitC)
if err != nil {
glog.V(logger.Warn).Infof("[BZZ] Swarm: loadManifestTrie error: %v", err)
return
@ -151,7 +151,8 @@ func (self *Api) Get(uri string, nameresolver bool) (reader storage.SectionReade
func (self *Api) Modify(uri, contentHash, contentType string, nameresolver bool) (newRootHash string, err error) {
root, _, path, err := self.parseAndResolve(uri, nameresolver)
trie, err := loadManifest(self.dpa, root)
quitC := make(chan bool)
trie, err := loadManifest(self.dpa, root, quitC)
if err != nil {
return
}
@ -162,9 +163,9 @@ func (self *Api) Modify(uri, contentHash, contentType string, nameresolver bool)
Hash: contentHash,
ContentType: contentType,
}
trie.addEntry(entry)
trie.addEntry(entry, quitC)
} else {
trie.deleteEntry(path)
trie.deleteEntry(path, quitC)
}
err = trie.recalcAndStore()

23
swarm/api/api_test.go
View file

@ -1,11 +1,13 @@
package api
import (
// "bytes"
"io"
"io/ioutil"
"os"
"testing"
"github.com/ethereum/go-ethereum/logger"
"github.com/ethereum/go-ethereum/logger/glog"
"github.com/ethereum/go-ethereum/swarm/storage"
)
@ -27,7 +29,7 @@ func testApi(t *testing.T, f func(*Api)) {
}
type testResponse struct {
reader storage.SectionReader
reader storage.LazySectionReader
*Response
}
@ -51,13 +53,14 @@ func checkResponse(t *testing.T, resp *testResponse, exp *Response) {
resp.Content = string(content)
}
if resp.Content != exp.Content {
// if !bytes.Equal(resp.Content, exp.Content) {
// if !bytes.Equal(resp.Content, exp.Content)
t.Errorf("incorrect content. expected '%s...', got '%s...'", string(exp.Content), string(resp.Content))
}
}
// func expResponse(content []byte, mimeType string, status int) *Response {
func expResponse(content string, mimeType string, status int) *Response {
glog.V(logger.Detail).Infof("expected content (%v): %v ", len(content), content)
return &Response{mimeType, status, int64(len(content)), content}
}
@ -67,7 +70,19 @@ func testGet(t *testing.T, api *Api, bzzhash string) *testResponse {
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
return &testResponse{reader, &Response{mimeType, status, reader.Size(), ""}}
quitC := make(chan bool)
size, err := reader.Size(quitC)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
glog.V(logger.Detail).Infof("reader size: %v ", size)
s := make([]byte, size)
_, err = reader.Read(s)
if err != io.EOF {
t.Fatalf("unexpected error: %v", err)
}
reader.Seek(0, 0)
return &testResponse{reader, &Response{mimeType, status, size, string(s)}}
// return &testResponse{reader, &Response{mimeType, status, reader.Size(), nil}}
}

13
swarm/api/config_test.go
View file

@ -19,16 +19,15 @@ var (
"CacheCapacity": 5000,
"Radius": 0,
"Branches": 128,
"Hash": "SHA256",
"JoinTimeout": 120,
"SplitTimeout": 120,
"CallInterval": 10000000000,
"Hash": "SHA3",
"CallInterval": 3000000000,
"KadDbPath": "` + filepath.Join("TMPDIR", "0d2f62485607cf38d9d795d93682a517661e513e", "bzz-peers.json") + `",
"MaxProx": 8,
"ProxBinSize": 4,
"BucketSize": 3,
"ProxBinSize": 2,
"BucketSize": 4,
"PurgeInterval": 151200000000000,
"InitialRetryInterval": 4200000000,
"InitialRetryInterval": 42000000,
"MaxIdleInterval": 4200000000,
"ConnRetryExp": 2,
"Swap": {
"BuyAt": 20000000000,

94
swarm/api/filesystem.go
View file

@ -86,27 +86,29 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
errors := make([]error, cnt)
done := make(chan bool, maxParallelFiles)
dcnt := 0
awg := &sync.WaitGroup{}
for i, entry := range list {
if i >= dcnt+maxParallelFiles {
<-done
dcnt++
}
awg.Add(1)
go func(i int, entry *manifestTrieEntry, done chan bool) {
f, err := os.Open(entry.Path)
if err == nil {
stat, _ := f.Stat()
sr := io.NewSectionReader(f, 0, stat.Size())
wg := &sync.WaitGroup{}
var hash storage.Key
hash, err = self.api.dpa.Store(sr, wg)
wg := &sync.WaitGroup{}
hash, err = self.api.dpa.Store(f, stat.Size(), wg)
if hash != nil {
list[i].Hash = hash.String()
}
wg.Wait()
awg.Done()
if err == nil {
first512 := make([]byte, 512)
fread, _ := sr.ReadAt(first512, 0)
fread, _ := f.ReadAt(first512, 0)
if fread > 0 {
mimeType := http.DetectContentType(first512[:fread])
if filepath.Ext(entry.Path) == ".css" {
@ -129,6 +131,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
trie := &manifestTrie{
dpa: self.api.dpa,
}
quitC := make(chan bool)
for i, entry := range list {
if errors[i] != nil {
return "", errors[i]
@ -140,9 +143,9 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
Hash: entry.Hash,
ContentType: entry.ContentType,
}
trie.addEntry(ientry)
trie.addEntry(ientry, quitC)
}
trie.addEntry(entry)
trie.addEntry(entry, quitC)
}
err2 := trie.recalcAndStore()
@ -150,6 +153,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
if err2 == nil {
hs = trie.hash.String()
}
awg.Wait()
return hs, err2
}
@ -170,11 +174,13 @@ func (self *FileSystem) Download(bzzpath, localpath string) error {
if err != nil {
return err
}
// if len(path) > 0 {
// path += "/"
// }
trie, err := loadManifest(self.api.dpa, key)
if len(path) > 0 {
path += "/"
}
quitC := make(chan bool)
trie, err := loadManifest(self.api.dpa, key, quitC)
if err != nil {
glog.V(logger.Warn).Infof("[BZZ] fs.Download: loadManifestTrie error: %v", err)
return err
@ -186,46 +192,47 @@ func (self *FileSystem) Download(bzzpath, localpath string) error {
}
var list []*downloadListEntry
var mde, mderr error
var mde error
prevPath := lpath
err = trie.listWithPrefix(path, func(entry *manifestTrieEntry, suffix string) { // TODO: paralellize
err = trie.listWithPrefix(path, quitC, func(entry *manifestTrieEntry, suffix string) {
glog.V(logger.Detail).Infof("[BZZ] fs.Download: %#v", entry)
key := common.Hex2Bytes(entry.Hash)
key = common.Hex2Bytes(entry.Hash)
path := lpath + "/" + suffix
dir := filepath.Dir(path)
if dir != prevPath {
mde = os.MkdirAll(dir, os.ModePerm)
if mde != nil {
mderr = mde
}
prevPath = dir
}
if (mde == nil) && (path != dir+"/") {
list = append(list, &downloadListEntry{key: key, path: path})
}
})
if err == nil {
err = mderr
if err != nil {
return err
}
cnt := len(list)
errors := make([]error, cnt)
wg := sync.WaitGroup{}
errC := make(chan error)
done := make(chan bool, maxParallelFiles)
dcnt := 0
for i, entry := range list {
if i >= dcnt+maxParallelFiles {
<-done
dcnt++
select {
case done <- true:
wg.Add(1)
case <-quitC:
return fmt.Errorf("aborted")
}
go func(i int, entry *downloadListEntry, done chan bool) {
go func(i int, entry *downloadListEntry) {
defer wg.Done()
f, err := os.Create(entry.path) // TODO: path separators
if err == nil {
reader := self.api.dpa.Retrieve(entry.key)
writer := bufio.NewWriter(f)
_, err = io.CopyN(writer, reader, reader.Size()) // TODO: handle errors
size, err := reader.Size(quitC)
if err == nil {
_, err = io.CopyN(writer, reader, size) // TODO: handle errors
err2 := writer.Flush()
if err == nil {
err = err2
@ -235,23 +242,26 @@ func (self *FileSystem) Download(bzzpath, localpath string) error {
err = err2
}
}
errors[i] = err
done <- true
}(i, entry, done)
}
for dcnt < cnt {
<-done
dcnt++
}
if err != nil {
select {
case errC <- err:
case <-quitC:
}
return
}
<-done
}(i, entry)
}
go func() {
wg.Wait()
close(errC)
}()
select {
case err = <-errC:
return err
case <-quitC:
return fmt.Errorf("aborted")
}
for i, _ := range list {
if errors[i] != nil {
return errors[i]
}
}
return err
}

29
swarm/api/filesystem_test.go
View file

@ -1,10 +1,12 @@
package api
import (
"bytes"
"io/ioutil"
"os"
"path"
"runtime"
"sync"
"testing"
)
@ -26,9 +28,9 @@ func testFileSystem(t *testing.T, f func(*FileSystem)) {
}
func readPath(t *testing.T, parts ...string) string {
// func readPath(t *testing.T, parts ...string) []byte {
file := path.Join(parts...)
content, err := ioutil.ReadFile(file)
if err != nil {
t.Fatalf("unexpected error reading '%v': %v", file, err)
}
@ -36,14 +38,12 @@ func readPath(t *testing.T, parts ...string) string {
}
func TestApiDirUpload0(t *testing.T) {
// t.Skip("FIXME")
testFileSystem(t, func(fs *FileSystem) {
api := fs.api
bzzhash, err := fs.Upload(path.Join(testDir, "test0"), "")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
content := readPath(t, testDir, "test0", "index.html")
resp := testGet(t, api, bzzhash+"/index.html")
exp := expResponse(content, "text/html; charset=utf-8", 0)
@ -54,17 +54,12 @@ func TestApiDirUpload0(t *testing.T) {
exp = expResponse(content, "text/css", 0)
checkResponse(t, resp, exp)
content = readPath(t, testDir, "test0", "img", "logo.png")
resp = testGet(t, api, bzzhash+"/img/logo.png")
exp = expResponse(content, "image/png", 0)
_, _, _, err = api.Get(bzzhash, true)
if err == nil {
t.Fatalf("expected error: %v", err)
}
downloadDir := path.Join(testDownloadDir, "test0")
os.RemoveAll(downloadDir)
defer os.RemoveAll(downloadDir)
err = fs.Download(bzzhash, downloadDir)
if err != nil {
@ -77,12 +72,10 @@ func TestApiDirUpload0(t *testing.T) {
if bzzhash != newbzzhash {
t.Fatalf("download %v reuploaded has incorrect hash, expected %v, got %v", downloadDir, bzzhash, newbzzhash)
}
})
}
func TestApiDirUploadModify(t *testing.T) {
// t.Skip("FIXME")
testFileSystem(t, func(fs *FileSystem) {
api := fs.api
bzzhash, err := fs.Upload(path.Join(testDir, "test0"), "")
@ -96,12 +89,24 @@ func TestApiDirUploadModify(t *testing.T) {
t.Errorf("unexpected error: %v", err)
return
}
bzzhash, err = api.Modify(bzzhash+"/index2.html", "9ea1f60ebd80786d6005f6b256376bdb494a82496cd86fe8c307cdfb23c99e71", "text/html; charset=utf-8", true)
index, err := ioutil.ReadFile(path.Join(testDir, "test0", "index.html"))
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
bzzhash, err = api.Modify(bzzhash+"/img/logo.png", "9ea1f60ebd80786d6005f6b256376bdb494a82496cd86fe8c307cdfb23c99e71", "text/html; charset=utf-8", true)
wg := &sync.WaitGroup{}
hash, err := api.Store(bytes.NewReader(index), int64(len(index)), wg)
wg.Wait()
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
bzzhash, err = api.Modify(bzzhash+"/index2.html", hash.Hex(), "text/html; charset=utf-8", true)
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
bzzhash, err = api.Modify(bzzhash+"/img/logo.png", hash.Hex(), "text/html; charset=utf-8", true)
if err != nil {
t.Errorf("unexpected error: %v", err)
return

17
swarm/api/http/server.go
View file

@ -86,8 +86,10 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
return
}
}
if len(proto) > 4 {
raw = proto[1:5] == "bzzr"
nameresolver = proto[1:5] != "bzzi"
}
glog.V(logger.Debug).Infof(
"[BZZ] Swarm: %s request over protocol %s '%s' received.",
@ -96,10 +98,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
switch {
case r.Method == "POST" || r.Method == "PUT":
key, err := a.Store(io.NewSectionReader(&sequentialReader{
reader: r.Body,
ahead: make(map[int64]chan bool),
}, 0, r.ContentLength), nil)
key, err := a.Store(r.Body, r.ContentLength, nil)
if err == nil {
glog.V(logger.Debug).Infof("[BZZ] Swarm: Content for %v stored", key.Log())
} else {
@ -165,7 +164,9 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
// retrieving content
reader := a.Retrieve(key)
glog.V(logger.Debug).Infof("[BZZ] Swarm: Reading %d bytes.", reader.Size())
quitC := make(chan bool)
size, err := reader.Size(quitC)
glog.V(logger.Debug).Infof("[BZZ] Swarm: Reading %d bytes.", size)
// setting mime type
qv := requestURL.Query()
@ -176,7 +177,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
w.Header().Set("Content-Type", mimeType)
http.ServeContent(w, r, uri, forever(), reader)
glog.V(logger.Debug).Infof("[BZZ] Swarm: Serve raw content '%s' (%d bytes) as '%s'", uri, reader.Size(), mimeType)
glog.V(logger.Debug).Infof("[BZZ] Swarm: Serve raw content '%s' (%d bytes) as '%s'", uri, size, mimeType)
// retrieve path via manifest
} else {
@ -203,7 +204,9 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
} else {
status = 200
}
glog.V(logger.Debug).Infof("[BZZ] Swarm: Served '%s' (%d bytes) as '%s' (status code: %v)", uri, reader.Size(), mimeType, status)
quitC := make(chan bool)
size, err := reader.Size(quitC)
glog.V(logger.Debug).Infof("[BZZ] Swarm: Served '%s' (%d bytes) as '%s' (status code: %v)", uri, size, mimeType, status)
http.ServeContent(w, r, path, forever(), reader)

77
swarm/api/manifest.go
View file

@ -4,7 +4,6 @@ import (
"bytes"
"encoding/json"
"fmt"
"io"
"sync"
"github.com/ethereum/go-ethereum/common"
@ -35,24 +34,24 @@ type manifestTrieEntry struct {
subtrie *manifestTrie
}
func loadManifest(dpa *storage.DPA, hash storage.Key) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
func loadManifest(dpa *storage.DPA, hash storage.Key, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
glog.V(logger.Detail).Infof("[BZZ] manifest lookup key: '%v'.", hash.Log())
// retrieve manifest via DPA
manifestReader := dpa.Retrieve(hash)
return readManifest(manifestReader, hash, dpa)
return readManifest(manifestReader, hash, dpa, quitC)
}
func readManifest(manifestReader storage.SectionReader, hash storage.Key, dpa *storage.DPA) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dpa *storage.DPA, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
// TODO check size for oversized manifests
manifestData := make([]byte, manifestReader.Size())
var size int
size, err = manifestReader.Read(manifestData)
if int64(size) < manifestReader.Size() {
size, err := manifestReader.Size(quitC)
manifestData := make([]byte, size)
read, err := manifestReader.Read(manifestData)
if int64(read) < size {
glog.V(logger.Detail).Infof("[BZZ] Manifest %v not found.", hash.Log())
if err == nil {
err = fmt.Errorf("Manifest retrieval cut short: read %v, expect %v", size, manifestReader.Size())
err = fmt.Errorf("Manifest retrieval cut short: read %v, expect %v", read, size)
}
return
}
@ -72,12 +71,12 @@ func readManifest(manifestReader storage.SectionReader, hash storage.Key, dpa *s
dpa: dpa,
}
for _, entry := range man.Entries {
trie.addEntry(entry)
trie.addEntry(entry, quitC)
}
return
}
func (self *manifestTrie) addEntry(entry *manifestTrieEntry) {
func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
self.hash = nil // trie modified, hash needs to be re-calculated on demand
if len(entry.Path) == 0 {
@ -98,11 +97,11 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry) {
}
if (oldentry.ContentType == manifestType) && (cpl == len(oldentry.Path)) {
if self.loadSubTrie(oldentry) != nil {
if self.loadSubTrie(oldentry, quitC) != nil {
return
}
entry.Path = entry.Path[cpl:]
oldentry.subtrie.addEntry(entry)
oldentry.subtrie.addEntry(entry, quitC)
oldentry.Hash = ""
return
}
@ -114,8 +113,8 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry) {
}
entry.Path = entry.Path[cpl:]
oldentry.Path = oldentry.Path[cpl:]
subtrie.addEntry(entry)
subtrie.addEntry(oldentry)
subtrie.addEntry(entry, quitC)
subtrie.addEntry(oldentry, quitC)
self.entries[b] = &manifestTrieEntry{
Path: commonPrefix,
@ -135,7 +134,7 @@ func (self *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) {
return
}
func (self *manifestTrie) deleteEntry(path string) {
func (self *manifestTrie) deleteEntry(path string, quitC chan bool) {
self.hash = nil // trie modified, hash needs to be re-calculated on demand
if len(path) == 0 {
@ -155,10 +154,10 @@ func (self *manifestTrie) deleteEntry(path string) {
epl := len(entry.Path)
if (entry.ContentType == manifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) {
if self.loadSubTrie(entry) != nil {
if self.loadSubTrie(entry, quitC) != nil {
return
}
entry.subtrie.deleteEntry(path[epl:])
entry.subtrie.deleteEntry(path[epl:], quitC)
entry.Hash = ""
// remove subtree if it has less than 2 elements
cnt, lastentry := entry.subtrie.getCountLast()
@ -198,24 +197,24 @@ func (self *manifestTrie) recalcAndStore() error {
return err
}
sr := io.NewSectionReader(bytes.NewReader(manifest), 0, int64(len(manifest)))
sr := bytes.NewReader(manifest)
wg := &sync.WaitGroup{}
key, err2 := self.dpa.Store(sr, wg)
key, err2 := self.dpa.Store(sr, int64(len(manifest)), wg)
wg.Wait()
self.hash = key
return err2
}
func (self *manifestTrie) loadSubTrie(entry *manifestTrieEntry) (err error) {
func (self *manifestTrie) loadSubTrie(entry *manifestTrieEntry, quitC chan bool) (err error) {
if entry.subtrie == nil {
hash := common.Hex2Bytes(entry.Hash)
entry.subtrie, err = loadManifest(self.dpa, hash)
entry.subtrie, err = loadManifest(self.dpa, hash, quitC)
entry.Hash = "" // might not match, should be recalculated
}
return
}
func (self *manifestTrie) listWithPrefixInt(prefix, rp string, cb func(entry *manifestTrieEntry, suffix string)) (err error) {
func (self *manifestTrie) listWithPrefixInt(prefix, rp string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) error {
plen := len(prefix)
var start, stop int
if plen == 0 {
@ -227,6 +226,11 @@ func (self *manifestTrie) listWithPrefixInt(prefix, rp string, cb func(entry *ma
}
for i := start; i <= stop; i++ {
select {
case <-quitC:
return fmt.Errorf("aborted")
default:
}
entry := self.entries[i]
if entry != nil {
epl := len(entry.Path)
@ -236,11 +240,13 @@ func (self *manifestTrie) listWithPrefixInt(prefix, rp string, cb func(entry *ma
l = epl
}
if prefix[:l] == entry.Path[:l] {
sterr := self.loadSubTrie(entry)
if sterr == nil {
entry.subtrie.listWithPrefixInt(prefix[l:], rp+entry.Path[l:], cb)
} else {
err = sterr
err := self.loadSubTrie(entry, quitC)
if err != nil {
return err
}
err = entry.subtrie.listWithPrefixInt(prefix[l:], rp+entry.Path[l:], quitC, cb)
if err != nil {
return err
}
}
} else {
@ -250,14 +256,14 @@ func (self *manifestTrie) listWithPrefixInt(prefix, rp string, cb func(entry *ma
}
}
}
return
return nil
}
func (self *manifestTrie) listWithPrefix(prefix string, cb func(entry *manifestTrieEntry, suffix string)) (err error) {
return self.listWithPrefixInt(prefix, "", cb)
func (self *manifestTrie) listWithPrefix(prefix string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) (err error) {
return self.listWithPrefixInt(prefix, "", quitC, cb)
}
func (self *manifestTrie) findPrefixOf(path string) (entry *manifestTrieEntry, pos int) {
func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *manifestTrieEntry, pos int) {
glog.V(logger.Detail).Infof("[BZZ] findPrefixOf(%s)", path)
@ -275,10 +281,10 @@ func (self *manifestTrie) findPrefixOf(path string) (entry *manifestTrieEntry, p
if (len(path) >= epl) && (path[:epl] == entry.Path) {
glog.V(logger.Detail).Infof("[BZZ] entry.ContentType = %v", entry.ContentType)
if entry.ContentType == manifestType {
if self.loadSubTrie(entry) != nil {
if self.loadSubTrie(entry, quitC) != nil {
return nil, 0
}
entry, pos = entry.subtrie.findPrefixOf(path[epl:])
entry, pos = entry.subtrie.findPrefixOf(path[epl:], quitC)
if entry != nil {
pos += epl
}
@ -308,6 +314,7 @@ func RegularSlashes(path string) (res string) {
func (self *manifestTrie) getEntry(spath string) (entry *manifestTrieEntry, fullpath string) {
path := RegularSlashes(spath)
var pos int
entry, pos = self.findPrefixOf(path)
quitC := make(chan bool)
entry, pos = self.findPrefixOf(path, quitC)
return entry, path[:pos]
}

7
swarm/api/manifest_test.go
View file

@ -10,18 +10,19 @@ import (
"github.com/ethereum/go-ethereum/swarm/storage"
)
func manifest(paths ...string) (manifestReader storage.SectionReader) {
func manifest(paths ...string) (manifestReader storage.LazySectionReader) {
var entries []string
for _, path := range paths {
entry := fmt.Sprintf(`{"path":"%s"}`, path)
entries = append(entries, entry)
}
manifest := fmt.Sprintf(`{"entries":[%s]}`, strings.Join(entries, ","))
return io.NewSectionReader(strings.NewReader(manifest), 0, int64(len(manifest)))
return &storage.LazyTestSectionReader{io.NewSectionReader(strings.NewReader(manifest), 0, int64(len(manifest)))}
}
func testGetEntry(t *testing.T, path, match string, paths ...string) *manifestTrie {
trie, err := readManifest(manifest(paths...), nil, nil)
quitC := make(chan bool)
trie, err := readManifest(manifest(paths...), nil, nil, quitC)
if err != nil {
t.Errorf("unexpected error making manifest: %v", err)
}

8
swarm/api/storage.go
View file

@ -34,7 +34,11 @@ func (self *Storage) Get(bzzpath string) (*Response, error) {
if err != nil {
return nil, err
}
expsize := reader.Size()
quitC := make(chan bool)
expsize, err := reader.Size(quitC)
if err != nil {
return nil, err
}
body := make([]byte, expsize)
size, err := reader.Read(body)
if int64(size) == expsize {
@ -43,6 +47,8 @@ func (self *Storage) Get(bzzpath string) (*Response, error) {
return &Response{mimeType, status, expsize, string(body[:size])}, err
}
// Modify(rootHash, path, contentHash, contentType) takes th e manifest trie rooted in rootHash,
// and merge on to it. creating an entry w conentType (mime)
func (self *Storage) Modify(rootHash, path, contentHash, contentType string) (newRootHash string, err error) {
return self.api.Modify(rootHash+"/"+path, contentHash, contentType, true)
}

229
swarm/cmd/README.md Normal file
View file

@ -0,0 +1,229 @@
# install and setup swarm
swarm is developed on a branch of the ethereum/go-ethereum repo
at this stage of the project there is no packages or binary distro, you need to a dev environment and compile from source.
[This document spells out a complete server setup on ubuntu linux](https://gist.github.com/zelig/74eb365752ceaacf15e860fb80eacb3e) including git/ssh/screen config, golang and compilation, node/npm and network monitoring (might contain a few bits that are tangential to swarm).
Assuming you got your setup working, you will use the `swarm` command line tool to control your swarm of instances.
This command line tool is at the moment geared towards developers and testing.
It is likely that it will be replaced by two different tools, one for devel/testing and one for end users
The command can be used to update the code
```shell
swarm update upstream/swarm
```
Then compile with
```shell
godep go build -v ./cmd/geth
```
Make sure you have `GOPATH` variable set and also that the `swarm` executable is in your PATH.
These environment variables are relevant and set to the following defaults.
Make sure you are happy with them, otherwise change them, in which case best to put these lines in your `~/.profile`.
```
export GETH_DIR=$GOPATH/src/github.com/ethereum/go-ethereum
export GETH=$GETH_DIR/geth
export SWARM_DIR=~/bzz
export SWARM_NETWORK_ID=322
```
* `GETH_DIR` points to your git working copy (given `GOPATH` its standardly under `$GOPATH/src/github.com/ethereum/go-ethereum`)
* `GETH` points to the `geth` executable compiled from the swarm branch. If you have systemwide install or use multiple geths you may need to change this, otherwise it is assumed you compile to the working copy of the repo.
* `SWARM_DIR` is the root directory for all swarm related stuff: logs, configs, as well as geth datadirs, make sure this dir is on a device with sufficient disk space
* `SWARM_NETWORK_ID`: this is by default the network id of the swarm testnet. If you run your own swarm, you need to change it, choose a number that is not likely chosen by others to avoid others joining you.
# Deploying and remote control
the swarm command supports remote update and remote control of your instances.
In our setting we assume you want to run a cluster of potentially remote swarm nodes each running a local cluster of instances
The only assumption is that you have (passwordless) ssh access set up to your swarm servers.
Assume `nodes.lst` is a list of nodes in the format of `username@ip` one per line. blank lines and lines commented out with `#` are ignored.
This copies the scripts found in `swarm/cmd/swarm` on all remote nodes listed in `nodes.lst`
```
swarm remote-update-scripts nodes.lst
```
If you just want to deploy a locally compiled binaries to all your remote nodes, this will fail if the remote instances are running, so make sure you stop them beforehand
```shell
swarm remote-run nodes.lst swarm stop all
swarm remote-update-bin nodes.lst
```
Once you deployed the executables to the nodes, you can control them all with one command. For instance the following line initialises a cluster of two test swarm instances on each remote node.
Watch out, this will wipe your storage and all swarm related data
```shell
swarm remote-run nodes.lst swarm init 2
```
To (re) start a particular instance on a specific remote node with alternative options (for instance mining and different logging verbosity), you can just:
```shell
swarm remote-run cicada@3.3.0.1 'swarm restart 01 --mine --verbosity=0 --vmodule=swarm/*=5'
```
# Logging
To check logs
```shell
swarm log 00 # taillog flow
swarm remote-run cicada@3.3.0.1 swarm log 00
```
You can view the log with a pager for an instance with
```
swarm viewlog 00
```
Logs are preserved and viewable with the above commands even when nodes are offline
Each new run logs to a different file
To purge logs
```
swarm cleanlog 01
```
To remove all logs on all nodes:
```
swarm remote-run nodes.lst swarm cleanlog all
```
# upload and dowload
upload and download via a running local instance
```shell
swarm up 00 /path/to/file/or/directory
swarm down 01 hash /path/to/destination
```
upload via remote swarm proxy or public gateway
```shell
swarm remote-up gateway-url /path/to/file/or/directory
wget -O- gateway-url/bzz:/swarm-url
```
# Further examples
```shell
# start with updaing
swarm update chambers
# display CLI options given to geth used to launch swarm instance 02
swarm options 02
# restart swarm instance 00 with alternatiev options
swarm restart 00 --mine --bzznosync --verbosity=0 --vmodule=swarm/*=6
# attach console to a running swarm instance
swarm attach 00
# execute a command; e.g., start mining on a running instance
swarm execute 00 'miner.stop(1)'
# display static info about a instance (even if its offline)
swarm info 00
# displays the enode url of a running instance
swarm enode 01
# add peers to a running swarm instance
swarm addpeers 00 "enode://1033c1cada...@3.3.0.1:30301"
# to compile a list of enodes from all instances on all remote nodes:
swarm remote-run nodes.lst 'swarm enode all' > enodes.lst
# to add all peers to all instances on each node
for node in `cat nodes.lst|grep -v '^#'`; do scp enodes.lst $node:; done
swarm remote-run nodes.lst 'swarm addpeers enodes.lst'
# if you run a local network and your nodes do not listen to external IPs
swarm remote-run pivot.lst 'swarm restart all'
# to add just one or a few guardians and let the network bootstrap
# swarm remote-run 'swarm enode all'
swarm addpeers pivot.lst
# or directly
swarm addpeers all <(IP_ADDR='[::]' swarm enode 01|tr -d '"')
# stop all running instances on the node
swarm stop all
# stop all running instances on all remote nodes
swarm remote-run nodes.lst swarm stop all
# display peer connection table of running instance 00
swarm hive 00
# display peer connection table for a running instance and continually refresh every 4 seconds
swarm monitor 00 4
# display peer connection table for all running instance on a remote node and continually refresh every 10 seconds
swarm monitor cicada@3.3.0.1 all 10
# configure eth-net-intelligence-api network monitoring client API for a node (the name argument appears as a prefix for all instances in your cluster)
swarm netstatconf cicada-sworm
# restart the net monitor client API
swarm netstatun
# configure eth-net-intelligence-api network monitoring client API and (re)start the monitor tool on all remote nodes
swarm remote-run nodes.lst 'swarm netstatconf cicada-sworm; swarm netstatrun'
swarm remote nodes.lst 'swarm netstatconf cicada-sworm; swarm netstatrun'
```
see also:
* https://github.com/ethereum/go-ethereum/tree/swarm/swarm/test
* https://github.com/ethereum/go-ethereum/tree/swarm/swarm/cmd
# ethereum netstats client setup
## install
nodejs and npm are prerequisites
```shell
# MAC
brew install node npm
# ubuntu
sudo apt-get install npm nodejs
```
clone the git repo and install:
```
git clone git@github.com:cubedro/eth-net-intelligence-api.git
cd eth-net-intelligence-api
npm install
npm install -g pm2
```
## configure and run netstats client for each node
```shell
swarm remote-run nodes.lst 'swarm netstatconf cicada-sworm; swarm netstatrun'
```

11
swarm/cmd/bzzhash/bzzhash.go
View file

@ -3,7 +3,6 @@ package main
import (
"fmt"
"io"
"os"
"runtime"
@ -24,15 +23,11 @@ func main() {
}
stat, _ := f.Stat()
sr := io.NewSectionReader(f, 0, stat.Size())
chunker := storage.NewTreeChunker(storage.NewChunkerParams())
hash := make([]byte, chunker.KeySize())
errC := chunker.Split(hash, sr, nil, nil)
err, ok := <-errC
key, err := chunker.Split(f, stat.Size(), nil, nil, nil)
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
}
if !ok {
fmt.Printf("%064x\n", hash)
} else {
fmt.Printf("%v\n", key)
}
}

6
swarm/cmd/swarm/env.sh Normal file
View file

@ -0,0 +1,6 @@
export PATH=$HOME/bin:$PATH
export GETH_DIR=$HOME/bin
export SWARM_DIR=
export NVM_DIR=$HOME/.nvm
[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" # This loads nvm

145
swarm/cmd/swarm/gethup.sh
View file

@ -1,145 +0,0 @@
#!/bin/bash
# Usage:
# bash /path/to/eth-utils/gethup.sh <datadir> <instance_name> <ip_addr>
root=$1 # base directory to use for datadir and logs
shift
id=$1 # double digit instance id like 00 01 02
shift
ip_addr=$1 # ip address to substitute
shift
# logs are output to a date-tagged file for each run , while a link is
# created to the latest, so that monitoring be easier with the same filename
# TODO: use this if GETH not set
# GETH=geth
# echo "ls -l $GETH"
# ls -l $GETH
# geth CLI params e.g., (dd=04, run=09)
datetag=`date "+%c%y%m%d-%H%M%S"|cut -d ' ' -f 5`
datadir=$root/data/$id # /tmp/eth/04
log=$root/log/$id.$datetag.log # /tmp/eth/04.09.log
linklog=$root/log/$id.log # /tmp/eth/04.09.log
password=$id # 04
port=303$id # 34504
bzzport=322$id # 32204
rpcport=302$id # 3204
mkdir -p $root/data
mkdir -p $root/enodes
mkdir -p $root/pids
mkdir -p $root/log
# if we do not have an account, create one
# will not prompt for password, we use the double digit instance id as passwd
# NEVER EVER USE THESE ACCOUNTS FOR INTERACTING WITH A LIVE CHAIN
keystoredir="$datadir/keystore/"
# echo "KeyStore dir: $keystoredir"
if [ ! -d "$keystoredir" ]; then
# echo "create an account with password $id [DO NOT EVER USE THIS ON LIVE]"
# mkdir -p $datadir/keystore
$GETH --datadir $datadir --password <(echo -n $id) account new >/dev/null 2>&1
# create account with password 00, 01, ...
# note that the account key will be stored also separately outside
# datadir
# this way you can safely clear the data directory and still keep your key
# under `<rootdir>/keystore/dd
# LS=`ls $datadir/keystore`
# echo $LS
while [ ! -d "$keystoredir" ]; do
echo "."
((i++))
if ((i>10)); then break; fi
sleep 1
done
# echo "copying keys $datadir/keystore $root/keystore/$id"
mkdir -p $root/keystore/$id
cp -R "$datadir/keystore/" $root/keystore/$id
fi
# # mkdir -p $datadir/keystore
# if [ ! -d "$datadir/keystore" ]; then
# echo "copying keys $root/keystore/$id $datadir/keystore"
# cp -R $root/keystore/$id/keystore/ $datadir/keystore/
# fi
# query node's enode url
if [ $ip_addr="" ]; then
pattern='\d+\.\d+\.\d+\.\d+'
ip_addr="[::]"
else
pattern='\[\:\:\]'
fi
geth="$GETH --datadir $datadir --port $port"
# echo -n "enode for instance $id... "
if [ ! "$GETH" = "" ] && [ ! -f $root/enodes/$id.enode ]; then
cmd="$geth js <(echo 'console.log(admin.nodeInfo.enode); exit();') "
# echo $cmd '2>/dev/null |grep enode | perl -pe "s/'$pattern'/'$ip_addr'/g" | perl -pe "s/^/\"/; s/\s*$/\"/;" > '$root/enodes/$id.enode
eval $cmd 2>/dev/null |grep enode | perl -pe "s/$pattern/$ip_addr/g" | perl -pe "s/^/\"/; s/\s*\$/\"/;" > $root/enodes/$id.enode
fi
# cat $root/enodes/$id.enode
echo
# copy cluster enodes list to node's static node list
# echo "copy cluster enodes list to node's static node list"
if [ -f $root/enodes.all ]; then
cp $root/enodes.all $datadir/static-nodes.json
fi
if [ ! -f $root/pids/$id.pid ]; then
# bring up node `dd` (double digit)
# - using <rootdir>/<dd>
# - listening on port 303dd, (like 30300, 30301, ...)
# - with the account unlocked
# - launching json-rpc server on port 81dd (like 8100, 8101, 8102, ...)
# echo "BZZKEY=$geth account list|head -n1|perl -ne '/([a-f0-9]{40})/ && print \$1'"
BZZKEY=`$geth account list|head -n1|perl -ne '/([a-f0-9]{40})/ && print \$1'`
echo -n "starting instance $id ($BZZKEY @ $datadir )..."
# echo "$geth \
# --identity=$id \
# --bzzaccount=$BZZKEY --bzzport=$bzzport \
# --unlock=$BZZKEY \
# --password=<(echo -n $id) \
# --rpc --rpcport=$rpcport --rpccorsdomain='*' $* \
# 2>&1 | tee "$stablelog" > "$log" & # comment out if you pipe it to a tty etc.
# " >&2
$GETH --datadir=$datadir \
--identity=$id \
--bzzaccount=$BZZKEY --bzzport=$bzzport \
--port=$port \
--unlock=$BZZKEY \
--password=<(echo -n $id) \
--rpc --rpcport=$rpcport --rpccorsdomain='*' $* \
> "$log" 2>&1 & # comment out if you pipe it to a tty etc.
ln -sf "$log" "$linklog"
# wait until ready
# pid=`ps auxwww|grep geth|grep "ty=$id"|grep -v grep|awk '{print $2}'`
# echo "pid: $pid"
# ps auxwww|grep geth|grep "ty=$id"|grep -v grep
# echo $pid > $root/pids/$id.pid
#echo $! > $root/pids/$id.pid
while true; do
$GETH --exec="net" attach ipc:$datadir/geth.ipc > /dev/null 2>&1 && break
sleep 1
echo -n "."
if ((i++>10)); then
echo "instance $id failed to start"
exit 1
fi
done
echo -n "started - "
pid=`ps auxwww|grep geth|grep "ty=$id"|grep -v grep|awk '{print $2}'`
echo "pid: $pid"
# ps auxwww|grep geth|grep "ty=$id"|grep -v grep
echo $pid > $root/pids/$id.pid
fi
# to bring up logs, uncomment
# tail -f $log

759
swarm/cmd/swarm/swarm Executable file
View file

@ -0,0 +1,759 @@
#!/bin/bash
if [ "$GETH_DIR" = "" ]; then
if [ "$GOPATH" = "" ]; then echo "either GETH_DIR or GOPATH environment variable must be set"; exit 1; fi
export GETH_DIR=$GOPATH/src/github.com/ethereum/go-ethereum
fi
if [ "$GETH" = "" ]; then
export GETH=$GETH_DIR/geth
fi
if [ "$SWARM_NETWORK_ID" = "" ]; then export SWARM_NETWORK_ID=322; fi
if [ "$SWARM_DIR" = "" ]; then export SWARM_DIR=$HOME/bzz; fi
if [ "$IP_ADDR" = "" ]; then
# export IP_ADDR=`curl ipecho.net/plain 2>/dev/null;echo f`
export IP_ADDR=
fi
root=$SWARM_DIR
network_id=$SWARM_NETWORK_ID
cmd=$1
shift
dir="$root/$network_id"
tmpdir=/tmp
function randomfile {
dd if=/dev/urandom of=/dev/stdout bs=1024 count=$1 2>/dev/null
}
# swarm attach 00 brings up a console attached to a running instance
function attach {
id=$1
shift
echo "attaching console to instance $id"
cmd="$GETH --datadir=$dir/data/$id $* attach ipc:$dir/data/$id/geth.ipc"
# echo $cmd
eval $cmd
}
# swarm attach 00 brings up a console attached to a running instance
function execute {
id=$1
shift
# attach $id --exec "'$*' "
cmd="$GETH --datadir=$dir/data/$id --exec '$*' attach ipc:$dir/data/$id/geth.ipc"
# echo $cmd
eval $cmd
}
# swarm hive 00 displays the kademlia table of the given running instance
function hive {
if [ "$1" = "all" ]; then
N=`ls -1 -d $dir/data/* |wc -l`
for ((i=0;i<N;++i)); do
instance=`printf "%02d" $i`
hive $instance
done
else
# echo "kademlia table of instance $id"
execute $1 'console.log(bzz.hive)'|grep -v undefined
fi
}
# swarm log 00 shows the running tail logs of an instance
function log {
id=$1
shift
echo "streaming logs for instance $id"
cmd="tail -f $dir/log/$id.log"
echo $cmd
eval $cmd
}
# swarm cleanlog 00 removes the old log files for a given instance (all for every instance)
function cleanlog {
id=$1
shift
if [ $id = "all" ]; then
echo "remove logs for all instances"
rm -rf "$dir/log/"
else
echo "remove logs for instance $id"
rm -rf $dir/log/$id*
fi
}
# display kademlia tables of istances
function monitor {
if [ "$3" = "" ]; then
id=$1
period=$2
while true; do
hive $id
sleep $period
done
else
node=$1
id=$2
period=$3
while true; do
remote-run $node swarm hive $id
sleep $period
done
fi
}
# swarm cleanbzz 00 removes the bzz subdirectory for a given instance (all for every instance)
function cleanbzz {
id=$1
shift
if [ $id = "all" ]; then
echo "remove bzz data for all instances"
rm -rf $dir/data/*/bzz
else
echo "remove bzz data for instance $id"
rm -rf "$dir/data/$id"
fi
}
# swarm less/viewlogd 00 displays the last (current) log for the given instance (in a pager)
function viewlog {
id=$1
shift
echo "viewing logs for instance $id"
cmd="/usr/bin/less $dir/log/$id.log"
echo $cmd
eval $cmd
}
# display the swawrm base account for an instance
function key {
id=$1
shift
mkdir -p $dir/data/$id/
$GETH --datadir=$dir/data/$id account list|head -n1|perl -ne '/([a-f0-9]{40})/ && print $1'
}
# swarm options 00 displays the geth command line options used to start the swarm
function rawoptions {
id=$1
shift
globaloptions="
--dev
--maxpeers=40
--shh=false
--nodiscover
--networkid=$network_id
--bzznoswap
--verbosity=0
--vmodule=swarm/*=5"
datadir=$dir/data/$id
password=$id
port=303$id
bzzport=322$id
rpcport=302$id
key=`swarm key $id`
instanceoptions="
--datadir=$datadir
--identity=$id
--bzzaccount=$key
--unlock=$key
--bzzport=$bzzport
--port=$port
--rpc
--rpcport=$rpcport
--rpccorsdomain='*'"
echo "$globaloptions"
echo "$instanceoptions"
echo "$*"
}
function options {
echo "The command line options passed to geth are the following:"
echo "use 'geth help' to see further options"
rawoptions $*
}
function start {
id=$1
shift
if [ -f $dir/pids/$id.pid ]; then
echo "instance $id already running"
return
fi
datetag=`date "+%Y-%m-%d-%H:%M:%S"`
log=$dir/log/$id.$datetag.log
linklog=$dir/log/$id.log
opts=`swarm rawoptions $id $*|tr '\r' ' '`
# echo; echo "$GETH $opts > $log 2>&1 &"
$GETH $opts --password=<(echo -n $id) > "$log" 2>&1 & # comment out if you pipe it to a tty etc.
ln -sf "$log" "$linklog"
# wait until ready
((j=0))
while true; do
execute $id "net" > /dev/null 2>&1 && break
sleep 1
echo -n "."
if ((j++>10)); then
echo "instance $id failed to start"
exit 1
fi
done
echo -n "started - "
pid=`ps auxwww|grep geth|grep "ty=$id"|grep -v grep|awk '{print $2}'`
echo "pid: $pid"
echo $pid > $dir/pids/$id.pid
}
# setup 00 creates the direcories for instance
function setup {
id=$1
shift
mkdir -p $dir/data/$id
mkdir -p $dir/enodes
mkdir -p $dir/pids
mkdir -p $dir/log
}
# creates the swarm base account for an instance
function create-account {
id=$1
datadir=$dir/data/$id
# if we do not have an account, create one
# will not prompt for password, we use the double digit instance id as passwd
# NEVER EVER USE THESE ACCOUNTS FOR INTERACTING WITH A LIVE CHAIN
keystoredir="$datadir/keystore/"
# echo "KeyStore dir: $keystoredir"
if [ ! -d "$keystoredir" ]; then
# echo "create an account with password $id [DO NOT EVER USE THIS ON LIVE]"
# mkdir -p $datadir/keystore
$GETH --datadir=$datadir --password=<(echo -n $id) account new >/dev/null 2>&1
# create account with password 00, 01, ...
# note that the account key will be stored also separately outside
# datadir
# this way you can safely clear the data directory and still keep your key
# under <rootdir>/keystore/dd
# LS=$(ls $datadir/keystore)
# echo $LS
while [ ! -d "$keystoredir" ]; do
echo "."
((i++))
if ((i>10)); then break; fi
sleep 1
done
# echo "copying keys $datadir/keystore $root/keystore/$id"
mkdir -p $dir/keystore/$id
cp -R "$keystoredir" $dir/keystore/$id
fi
}
# shuts down a running instance, cleans the pid
function stop {
id=$1
shift
if [ $id = "all" ]; then
procs=`cat $dir/pids/*.pid 2>/dev/null |perl -pe 's/^\s+//;s/\s+\\$//;s/\s+/\n/g'`
# echo "stopping processes $procs"
for p in $procs; do
shutdown $p
done
rm -rf $dir/pids/*
else
pid=$dir/pids/$id.pid
if [ -f $pid ]; then
echo "stopping instance $id, pid="`cat $pid`
shutdown `cat $pid`
rm $pid
fi
fi
}
# shutdown kills the node with interrupt 2 - if it resits falls back to -9 after 10s
function shutdown {
echo -n "stopping $1..."
kill -2 $1
while true; do
ps auxwww|grep geth|grep -v grep|awk '{print $2}'|grep -ql $1 || break
if ((i++>5)); then
echo "not stopping. killing it"
kill -QUIT $1
break
fi
echo '.'
sleep 1
done
echo "stopped"
}
# swarm restart 00 calls stop and start
function restart {
id=$1
shift
if [ $id = "all" ]; then
stop all
N=`ls -d1 $dir/data/*|wc -l`
cluster $N $*
else
stop $id
start $id $*
fi
}
# swarm init X sets up and starts a new client instance
##########################################################
#
# IT WIPES THE DATABASE
#
##########################################################
function init {
killall geth
reset all
cluster $*
enode all
connect all
}
# reset wipes the datadirs of the instance
function reset {
id=$1
shift
if [ $id = "all" ]; then
rm -rf $dir
else
rm -rf$dir/*/$id*
fi
}
# enode displays the instance's enode address
# swarm enode all writes all instances' enodes in a file
function enode {
id=$1
shift
if [ $id = "all" ]; then
json=$dir/static-nodes.json
enodes=$dir/enodes.lst
cmd=$dir/connect.js
rm -f $enodes $json $cmd
# build a static nodes(-like) list of all enodes of the local cluster
echo "[" >> $json
N=`ls -1 -d $dir/data/* |wc -l`
for ((i=0;i<N;++i)); do
id=`printf "%02d" $i`
f=$dir/enodes/$id.enode
enode $id > $f
echo -n "admin.addPeer(" >> $cmd
cat "$f" | perl -pe 's/\s*$//' >> $cmd
echo ");" >> $cmd
cat $f |perl -pe 's/"//g'>> $enodes
cat $f >> $json
echo "," >> $json
done
echo "\"\"]" >> $json
cat $enodes
else
# echo "local IP: $ip_addr "
execute $id 'admin.nodeInfo.enode' |perl -pe "s/\[\:\:\]/$IP_ADDR/ "
fi
}
# connect sources the local or remote set of peers and connects the node to the peers
function connect {
id=$1
shift
if [ $id = "all" ]; then
N=`ls -1 -d $dir/data/* |wc -l`
for ((i=0;i<N;++i)); do
id=`printf "%02d" $i`
connect $id
done
else
echo -n 'peer count: '
attach $id --preload $dir/connect.js --exec net.peerCount
fi
}
# swarm cluster N launches N nodes; 00 01 02 ...
function cluster {
N=$1
shift
echo "launching cluster of $N instances"
for ((i=0;i<N;++i)); do
id=`printf "%02d" $i`
setup $id $*
create-account $id
echo "launching node $i/$N.."
start $id $*
# info $id
enode $id
done
}
# swarm needs instance keyfile destination
# tests if the content for the key is available for the intance
#
function needs {
id=$1
keyfile=$2
target=$3
dest=$tmpdir/down
mkdir -p $dest
file=$dest/`basename $target`
rm -f $file
echo -n "waiting for root hash in '$keyfile'..."
while true; do
if [ ! -z $keyfile ]; then
break
fi
sleep 1
echo -n "."
done
key=`cat $keyfile|tr -d \"`
echo " => $key"
download $id $key $dest && cmp --silent $file $target && echo "PASS" || echo "FAIL"
}
# swarm up 00 file uploads file via instances CLI
function up { #port, file
echo "Upload file '$2' to node $1... " 1>&2
file=`basename $2`
/usr/bin/time -f "latency: %e" swarm execute $1 "bzz.upload(\"$2\", \"$file\")"|tail -n1> /tmp/key
cat /tmp/key
}
# swarm download 00 file download file via instances CLI
function download {
echo "download '$2' from node $1 to '$3'"
execute $1 "bzz.download(\"$2\", \"$3\")" >/dev/null
}
# swarm down issues bzz.get to download the content 10 attempts
function down {
echo -n "Download hash '$2' from node $1... "
while true; do
execute $1 "bzz.get(\"$2\")" 2> /dev/null |grep -qil "status" && break
sleep 1
echo -n "."
if ((i++>10)); then
echo "not found"
return
fi
done
echo "found OK"
}
# static info about an instance (available even if node is off)
function info {
echo "swarm node information"
echo "ROOTDIR: $root"
echo "DATADIR: $dir/data/$1"
echo "LOGFILE: $dir/log/$1.log"
echo "HTTPAPI: http://localhost:322$1"
echo "ETHPORT: 303$1"
echo "RPCPORT: 302$1"
echo "ACCOUNT:" 0x`ls -1 $dir/data/$1/bzz`
echo "CHEQUEB:" `cat $dir/data/$1/bzz/*/config.json|grep Contract|awk -F\" '{print $4}'`
echo "ROOTDIR: $root"
echo "DATADIR: $dir/data/$1"
echo "LOGFILE: $dir/log/$1.log"
}
# live into about an instance
function status {
echo -n "account balance: "
execute $1 'eth.getBalance(eth.accounts[0])'
echo -n "swap contract balance: "
execute $1 "eth.getBalance(bzz.info.Swap.Contract)"
echo -n "chequebook balance: "
execute $1 "chequebook.balance"
echo -n "peer count: "
execute $1 'net.peerCount'
echo -n "latest block number: "
execute $1 "eth.blockNumber"
}
# display peers for an instance
function peers {
execute $1 'admin.peers'
}
# add peers into an instance (connection not guaranteed)
function addpeers {
id=$1
peers=$2
if [ $id = "all" ]; then
N=`ls -1 -d $dir/data/* |wc -l`
for ((i=0;i<N;++i)); do
id=`printf "%02d" $i`
addpeers $id $peers
done
else
echo "addpeer to instance $id"
for peer in `cat $peers|grep -v '^#'`; do
execute $id "admin.addPeer(\"$peer\")"
done
fi
}
# configures the eth-net-intelligence-api network monitor
function netstatconf {
group=$1
N=`ls -1 -d $dir/data/* |wc -l`
ip=`curl ipecho.net/plain 2>/dev/null;echo `
ws_server="ws://146.185.130.117:3000"
ws_secret=BZZ322
conf="$dir/$group-$ip.netstat.json"
echo "writing netstat conf for cluster $group-$ip ($N instances) -> $conf"
echo -e "[" > $conf
for ((i=0;i<$N;++i)); do
id=`printf "%02d" $i`
single_template=" {\n \"name\" : \"$group-$ip-$i\",\n \"cwd\" : \".\",\n \"script\" : \"app.js\",\n \"log_date_format\" : \"YYYY-MM-DD HH:mm Z\",\n \"merge_logs\" : false,\n \"watch\" : false,\n \"exec_interpreter\" : \"node\",\n \"exec_mode\" : \"fork_mode\",\n \"env\":\n {\n \"NODE_ENV\" : \"production\",\n \"RPC_HOST\" : \"localhost\",\n \"RPC_PORT\" : \"302$id\",\n \"INSTANCE_NAME\" : \"$group-$ip-$i\",\n \"WS_SERVER\" : \"$ws_server\",\n \"WS_SECRET\" : \"$ws_secret\",\n }\n }"
endline=""
if ((i<$N-1)); then
# if [ "$i" -ne "$N" ]; then
endline=","
fi
echo -e "$single_template$endline" >> $conf
done
echo "]" >> $conf
}
# (re)starts the eth-net-intelligence-api network monitor
function netstatrun {
cd $GETH_DIR/../eth-net-intelligence-api
pm2 kill
pm2 start $dir/*.netstat.json
}
# kills the eth-net-intelligence-api network monitor
function netstatkill {
cd $GETH_DIR/../eth-net-intelligence-api
pm2 kill
}
# copies the swarm control script to the remote node(s)
function remote-update-scripts {
scriptdir=$GETH_DIR/swarm/cmd/swarm/
remotes=$1
for remote in `cat $remotes|grep -v '^#'`; do echo "updating scripts on $remote..."; ssh $remote mkdir -p bin && scp -r $scriptdir/* $remote:bin/; done
}
# copies the geth executable to the remote nodes
function remote-update-bin {
remotes=$1
# remote-update-scripts $remotes
for remote in `cat $remotes|grep -v '^#'`; do echo "updating binary on $remote..."; scp -r $GETH_DIR/geth $remote:bin/; done
}
# runs a command on remote node or nodes from a file
function remote-run {
remotes=$1
shift
if `echo "$remotes" | grep -qil @`; then
ip=`echo "$remotes"|cut -d@ -f2`
ssh $remotes "export IP_ADDR=$ip;" '. $HOME/bin/env.sh;' "$*"
else
for remote in `cat $remotes|grep -v '^#'`; do echo "running on $remote..."; remote-run $remote "$*"; done
fi
}
# updates the code from the given branch
function update {
branch=$1
echo "cd $GETH_DIR && git remote update && git reset --hard $branch"
(cd $GETH_DIR && git remote update && git reset --hard $branch)
}
function checksum {
tar -cf - $1 | md5sum|awk '{print $1}'
}
function checkaccess {
nodes=$1
target=`basename $2`
chsum=`md5sum $2|cut -f1 -d' '`
master=`head -1 $nodes`
echo "uploading target on $master (md5sum $chsum, size: `du -b -d0 $2|cut -f1`)"
scp $2 $master:$target
hash=`swarm remote-run $master "swarm up 00 $target $file"|tr -d '"'`
remote-run $nodes swarm checkdownload all $hash $chsum
}
function checkdownload {
id=$1
if [ "$id" = "all" ]; then
shift
N=`ls -1 -d $dir/data/* |wc -l`
for ((i=0;i<N;++i)); do
id=`printf "%02d" $i`
checkdownload $id $*
done
else
hash=$2
target=$3
datetag=`date "+%Y-%m-%d-%H:%M:%S"`
file="swarm-$datetag"
rm -rf $tmpdir/$file
/usr/bin/time -o $tmpdir/$file.log -f "%e" swarm download $id $hash $tmpdir/$file > /dev/null
echo
if [ -f $target ]; then
cmp --silent $target $tmpdir/$file/* && echo PASS || echo FAIL
elif [ -r $target ]; then
diff -r $target $tmpdir/$file/ >/dev/null && echo PASS || echo FAIL
else
exp=`md5sum $tmpdir/$file/*|cut -f1 -d' '`
if [ "$exp" = "$target" ]; then
echo -n PASS
else
echo FAIL "$exp = $target"
fi
fi
echo " latency: " `cat $tmpdir/$file.log`
fi
}
function meminfo {
pid="$dir/pids/$1.pid"
if [ -f "$pid" ]; then
# cd /proc/`cat "$pid"` && cat status
ps aux|awk -v PID=`cat $pid` '$2 == PID {print $5 "Kb (" $4 "%)" }'
fi
}
function cpuinfo {
pid="$dir/pids/$1.pid"
if [ -f "$pid" ]; then
# cd /proc/`cat "$pid"` && cat status
ps aux|awk -v PID=`cat $pid` '$2 == PID {print $3 "%" }'
fi
}
function diskusage {
if [ "$1" == "" ]; then
echo "DISK USAGE:" `df -m |grep '/$'|awk '{print $(NF-2) "Mb (" $(NF-1) ")"} '`
else
du -m -d0 $*|cut -f1
fi
}
function diskinfo {
echo "DISK USAGE $1:"
echo "blockchain:" `diskusage $dir/data/$1/chaindata`
echo "chunkstore:" `diskusage $dir/data/$1/bzz/*/chunks`
echo "overall: /" `diskusage`
}
case $cmd in
"info" )
info $*;;
"enode" )
enode $*;;
"status" )
status $*;;
"peers" )
peers $*;;
"addpeers" )
addpeers $*;;
"clean" )
clean $*;;
"needs" )
needs $*;;
"up" )
up $*;;
"key" )
key $*;;
"down" )
down $*;;
"download" )
download $*;;
"init" )
init $*;;
"exec" )
execute $*;;
"hive" )
hive $*;;
"start" )
start $*;;
"stop" )
stop $* ;;
"restart" )
restart $*;;
"reset" )
reset $*;;
"cluster" )
cluster $*;;
"attach" )
attach $*;;
"execute" )
execute $*;;
"exec" )
execute $*;;
"cleanbzz" )
cleanbzz $*;;
"cleanlog" )
cleanlog $*;;
"log" )
log $*;;
"viewlog" )
viewlog $*;;
"less" )
viewlog $*;;
"connect" )
connect $*;;
"monitor" )
monitor $*;;
"remote-update-scripts" )
remote-update-scripts $*;;
"remote-update-bin" )
remote-update-bin $*;;
"update-src" )
update-src $*;;
"remote-run" )
remote-run $*;;
"netstatconf" )
netstatconf $*;;
"netstatkill" )
netstatkill $*;;
"netstatrun" )
netstatrun $*;;
"options" )
options $*;;
"rawoptions" )
rawoptions $*;;
"randomfile" )
randomfile $*;;
"diskinfo" )
diskinfo $*;;
"meminfo" )
meminfo $*;;
"cpuinfo" )
cpuinfo $*;;
"setup" )
setup $* ;;
"create-account" )
create-account $*;;
"checkaccess" )
checkaccess $* ;;
"checkdownload" )
checkdownload $* ;;
esac

360
swarm/cmd/swarm/swarm.sh
View file

@ -1,360 +0,0 @@
# !/bin/bash
# bash cluster <root> <network_id> <number_of_nodes> <runid> <local_IP> [[params]...]
# https://github.com/ethereum/go-ethereum/wiki/Setting-up-monitoring-on-local-cluster
# sets up a local ethereum network cluster of nodes
# - <number_of_nodes> is the number of nodes in cluster
# - <root> is the root directory for the cluster, the nodes are set up
# with datadir `<root>/<network_id>/00`, `<root>/ <network_id>/01`, ...
# - new accounts are created for each node
# - they launch on port 30300, 30301, ...
# - they star rpc on port 8100, 8101, ...
# - by collecting the nodes nodeUrl, they get connected to each other
# - if enode has no IP, `<local_IP>` is substituted
# - if `<network_id>` is not 0, they will not connect to a default client,
# resulting in a private isolated network
# - the nodes log into `<root>/<network_id>/00.<runid>.log`, `<root>/<network_id>/01.<runid>.log`, ...
# - The nodes launch in mining mode
# - the cluster can be killed with `killall geth` (FIXME: should record PIDs)
# and restarted from the same state
# - if you want to interact with the nodes, use rpc
# - you can supply additional params on the command line which will be passed
# to each node, for instance `-mine`
if [ "$GETH" = "" ]; then
echo "env var GETH not set "
exit 1
fi
srcdir=`dirname $0`
root=$1
shift
network_id=$1
shift
cmd=$1
shift
# ip_addr=`curl ipecho.net/plain 2>/dev/null;echo `
# echo "external IP: $ip_addr"
swarmoptions='--dev --maxpeers=20 --shh=false --nodiscover'
tmpdir=/tmp
function attach {
id=$1
shift
echo "attaching console to instance $id"
cmd="$GETH $* attach ipc:$root/$network_id/data/$id/geth.ipc"
# echo $cmd
eval $cmd
}
function log {
id=$1
shift
echo "streaming logs for instance $id"
cmd="tail -f $root/$network_id/log/$id.log"
echo $cmd
eval $cmd
}
function less {
id=$1
shift
echo "viewing logs for instance $id"
cmd="/usr/bin/less $root/$network_id/log/$id.log"
echo $cmd
eval $cmd
}
function start {
id=$1
shift
# echo -n "starting instance $id - "
cmd="bash $srcdir/gethup.sh $root/$network_id/ $id '$ip_addr' --networkid=$network_id $swarmoptions $*"
# echo "pid="`cat $root/$network_id/pids/$id.pid`
# echo $cmd
eval $cmd
}
function stop {
id=$1
shift
if [ $id = "all" ]; then
procs=`cat $root/$network_id/pids/*.pid 2>/dev/null |perl -pe 's/^\s+//;s/\s+\\$//;s/\s+/\n/g'`
# echo "stopping processes $procs"
for p in $procs; do
shutdown $p
done
rm -rf $root/$network_id/pids/*
else
pid=$root/$network_id/pids/$id.pid
if [ -f $pid ]; then
echo "stopping instance $id, pid="`cat $pid`
shutdown `cat $pid`
rm $pid
fi
fi
# ps auxwww|grep geth|grep bzz|grep -v grep
}
function shutdown {
echo -n "stopping $1..."
kill -2 $1
while true ;do
ps auxwww|grep geth|grep -v grep|awk '{print $2}'|grep -ql $1 || break
sleep 1
done
echo "stopped"
}
function restart {
id=$1
shift
stop $id
start $id $*
}
function init {
stop all
killall geth
reset all
cluster $*
enode all
connect all
}
function reset {
id=$1
shift
if [ $id = "all" ]; then
rm -rf $root/$network_id
else
rm -rf$root/$network_id/*/$id*
fi
}
function enode {
dir=$root/$network_id
id=$1
shift
if [ $id = "all" ]; then
N=`ls -1 $dir/enodes/|wc -l`
enodes=$dir/enodes.all
rm -f $enodes
# build a static nodes(-like) list of all enodes of the local cluster
echo "[" >> $enodes
for ((i=0;i<N;++i)); do
id=`printf "%02d" $i`
enode=$dir/enodes/$id.enode
enode $id
if [ -f "$enode" ] && [ ! -z "$enode" ]; then
cat "$enode" >> $enodes
echo "," >> $enodes
fi
done
echo "\"\"]" >> $enodes
cmd=$dir/connect.js
for ((i=0;i<N;++i)); do
id=`printf "%02d" $i`
enode=$dir/enodes/$id.enode
if [ -f "$enode" ] && [ ! -z "$enode" ]; then
echo -n "admin.addPeer(" >> $cmd
cat "$enode" >> $cmd
echo ");" >> $cmd
fi
done
else
enode=$dir/enodes/$id.enode
attach $id --exec "'console.log(admin.nodeInfo.enode)'" |head -2 |tail -1| perl -pe 's/^/"/;s/$/"/'|perl -pe 's/\s*$//' > $enode
# cat $enode
fi
}
function connect {
dir=$root/$network_id
id=$1
shift
if [ $id = "all" ]; then
for ((i=0;i<N;++i)); do
id=`printf "%02d" $i`
connect $id
done
else
cmd="$GETH --preload $dir/connect.js --exec '\"admin.peers\"' attach ipc:$root/$network_id/data/$id/geth.ipc $dir/connect.js"
# echo $cmd
eval $cmd
cat $dir/connect.js
fi
}
function cluster {
N=$1
shift
echo "launching cluster of $N instances"
# cmd="bash $srcdir/gethcluster.sh $root $network_id $N '' $swarmoptions $*"
# echo $cmd
# eval $cmd
dir=$root/$network_id
mkdir -p $dir/data
mkdir -p $dir/enodes
mkdir -p $dir/pids
mkdir -p $dir/log
for ((i=0;i<N;++i)); do
id=`printf "%02d" $i`
mkdir -p $dir/data/$id
echo "launching node $i/$N ---> tail -f $dir/log/$id.log"
start $id $vmodule $*
done
}
function needs {
id=$1
keyfile=$2
target=$3
dir=`dirname $3`
dest=$tmpdir/down
mkdir -p $dest
file=$dest/`basename $target`
rm -f $file
echo -n "waiting for root hash in '$keyfile'..."
while true; do
if [ -f $keyfile ] && [ ! -z $keyfile ]; then
break
fi
sleep 1
echo -n "."
done
key=`cat $keyfile|tr -d \"`
echo " => $key"
download $id $key $dest && cmp --silent $file $target && echo "PASS" || echo "FAIL"
# && ls -l $keyfile $file $target
}
function up { #port, file
echo "Upload file '$2' to node $1... " 1>&2
file=`basename $2`
attach $1 "--exec 'bzz.upload(\"$2\", \"$file\")'"|tail -n1> /tmp/key
# key=`bash swarm/cmd/bzzup.sh $2 86$1`
cat /tmp/key
}
function download {
echo "download '$2' from node $1 to '$3'"
# echo attach $1 "--exec 'bzz.download(\"$2\", \"$3\")'"
attach $1 "--exec 'bzz.download(\"$2\", \"$3\")'" > /dev/null
}
function down {
echo -n "Download hash '$2' from node $1... "
# echo "wget -O- http://localhost:86$1/$2 > /dev/null 2>&1 && echo 'got it' || echo 'not found'"
# wget -O- http://localhost:86$1/$2 > /dev/null 2>&1 && echo "got it" || echo "not found"
while true; do
attach $1 "--exec 'bzz.get(\"$2\")'" 2> /dev/null |grep -qil "status" && break
sleep 1
echo -n "."
if ((i++>10)); then
echo "not found"
return
fi
done
echo "found OK"
}
function clean { #index
echo "Clean up for $1"
rm -rf $root/$network_id/data/$1/{bzz/*/chunks,bzz/*/requests/,bzz/*/bzz-peers.json,chaindata,nodes}
}
function info {
echo "swarm node information"
echo "ROOTDIR: $root"
echo "DATADIR: $root/$network_id/data/$1"
echo "LOGFILE: $root/$network_id/log/$1.log"
echo "HTTPAPI: http://localhost:322$1"
echo "ETHPORT: 303$1"
echo "RPCPORT: 302$1"
echo "ACCOUNT:" 0x`ls -1 $root/$network_id/data/$1/bzz`
echo "CHEQUEB:" `cat $root/$network_id/data/$1/bzz/*/config.json|grep Contract|awk -F\" '{print $4}'`
echo "ROOTDIR: $root"
echo "DATADIR: $root/$network_id/data/$1"
echo "LOGFILE: $root/$network_id/log/$1.log"
}
function status {
attach 00 -exec "'console.log(eth.getBalance(eth.accounts[0])); console.log(eth.getBalance(bzz.info().Swap.Contract)); console.log(chequebook.balance)'"
}
function netstatconf {
begin=$1
N=$2
name_prefix=$3
ws_server=$4
ws_secret=$5
conf="$root/$network_id/$name_prefix.netstat.json"
echo "writing netstat conf for cluster $name_prefix to $conf"
echo -e "[" > $conf
for ((i=$begin;i<$start+$N;++i)); do
id=`printf "%02d" $i`
single_template=" {\n \"name\" : \"$name_prefix-$i\",\n \"cwd\" : \".\",\n \"script\" : \"app.js\",\n \"log_date_format\" : \"YYYY-MM-DD HH:mm Z\",\n \"merge_logs\" : false,\n \"watch\" : false,\n \"exec_interpreter\" : \"node\",\n \"exec_mode\" : \"fork_mode\",\n \"env\":\n {\n \"NODE_ENV\" : \"production\",\n \"RPC_HOST\" : \"localhost\",\n \"RPC_PORT\" : \"302$id\",\n \"INSTANCE_NAME\" : \"$name_prefix-$i\",\n \"WS_SERVER\" : \"$ws_server\",\n \"WS_SECRET\" : \"$ws_secret\",\n }\n }"
endline=""
if (($i<$N-1)); then
# if [ "$i" -ne "$N" ]; then
endline=","
fi
echo -e "$single_template$endline" >> $conf
done
echo "]" >> $conf
}
case $cmd in
"info" )
info $*;;
"enode" )
enode $*;;
"connect" )
connect $*;;
"status" )
status $*;;
"clean" )
clean $*;;
"needs" )
needs $*;;
"up" )
up $*;;
"down" )
down $*;;
"init" )
init $*;;
"start" )
start $*;;
"stop" )
stop $* ;;
"restart" )
restart $*;;
"reset" )
reset $*;;
"cluster" )
cluster $*;;
"attach" )
attach $*;;
"log" )
log $*;;
"less" )
less $*;;
"netstatconf" )
netstatconf $*;;
esac

28
swarm/cmd/swarm/test.sh
View file

@ -1,28 +0,0 @@
#!/bin/bash
TEST_DIR=`dirname $0`
TEST_NAME=`basename $0 .sh`
TEST_TYPE=`basename $TEST_DIR`
export SWARM_BIN=$TEST_DIR/../../cmd/swarm
export GETH=$SWARM_BIN/../../../geth
export NETWORKID=322$TEST_NAME
export TMPDIR=~/BZZ/test/$TEST_TYPE
export DATA_ROOT=$TMPDIR/$NETWORKID
# alias swarm='bash $SWARM_BIN/swarm.sh $DATA_ROOT $NETWORKID'
EXTRA_ARGS=$*
rm -rf $DATA_ROOT
wait=1
function swarm {
# echo bash $SWARM_BIN/swarm.sh $TMPDIR $NETWORKID $* $EXTRA_ARGS
bash $SWARM_BIN/swarm.sh $TMPDIR $NETWORKID $* $EXTRA_ARGS
}
function randomfile {
dd if=/dev/urandom of=/dev/stdout bs=1024 count=$1 2>/dev/null
}

56
swarm/network/hive.go
View file

@ -30,6 +30,7 @@ type Hive struct {
addr kademlia.Address
kad *kademlia.Kademlia
path string
quit chan bool
toggle chan bool
more chan bool
@ -106,6 +107,7 @@ func (self *Hive) Addr() kademlia.Address {
func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPeer func(string) error) (err error) {
self.toggle = make(chan bool)
self.more = make(chan bool)
self.quit = make(chan bool)
self.id = id
self.listenAddr = listenAddr
err = self.kad.Load(self.path, nil)
@ -123,15 +125,15 @@ func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPee
// to attempt to write to more (remove Peer when shutting down)
return
}
node, need, proxLimit := self.kad.FindBest()
node, need, proxLimit := self.kad.Suggest()
glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: select candidate peer")
if node != nil && len(node.Url) > 0 {
glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: call for bee %v", node.Url)
glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: call known bee %v", node.Url)
// enode or any lower level connection address is unnecessary in future
// discovery table is used to look it up.
connectPeer(node.Url)
} else if need {
}
if need {
// a random peer is taken from the table
peers := self.kad.FindClosest(kademlia.RandomAddressAt(self.addr, rand.Intn(self.kad.MaxProx)), 1)
if len(peers) > 0 {
@ -140,16 +142,19 @@ func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPee
req := &retrieveRequestMsgData{
Key: storage.Key(randAddr[:]),
}
glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: call any bee in area %v messenger bee %v", randAddr, peers[0])
glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: call any bee near %v (PO%03d) - messenger bee: %v", randAddr, proxLimit, peers[0])
peers[0].(*peer).retrieve(req)
} else {
glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: no peer")
glog.V(logger.Warn).Infof("[BZZ] KΛÐΞMLIΛ hive: no peer")
}
self.toggle <- true
glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: buzz kept alive")
glog.V(logger.Detail).Infof("[BZZ] KΛÐΞMLIΛ hive: buzz kept alive")
} else {
glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: no need for more bees")
self.toggle <- false
glog.V(logger.Info).Infof("[BZZ] KΛÐΞMLIΛ hive: no need for more bees")
}
select {
case self.toggle <- need:
case <-self.quit:
return
}
glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: queen's address: %v, population: %d (%d)", self.addr, self.kad.Count(), self.kad.DBCount())
}
@ -174,11 +179,7 @@ func (self *Hive) keepAlive() {
default:
}
}
case need, alive := <-self.toggle:
if !alive {
self.more <- false
return
}
case need := <-self.toggle:
if alarm == nil && need {
alarm = time.NewTicker(time.Duration(self.callInterval)).C
}
@ -186,20 +187,31 @@ func (self *Hive) keepAlive() {
alarm = nil
}
case <-self.quit:
return
}
}
}
func (self *Hive) Stop() error {
// closing toggle channel quits the updateloop
close(self.toggle)
close(self.quit)
return self.kad.Save(self.path, saveSync)
}
// called at the end of a successful protocol handshake
func (self *Hive) addPeer(p *peer) {
func (self *Hive) addPeer(p *peer) error {
defer func() {
select {
case self.more <- true:
default:
}
}()
glog.V(logger.Detail).Infof("[BZZ] KΛÐΞMLIΛ hive: hi new bee %v", p)
self.kad.On(p, loadSync)
err := self.kad.On(p, loadSync)
if err != nil {
return err
}
// self lookup (can be encoded as nil/zero key since peers addr known) + no id ()
// the most common way of saying hi in bzz is initiation of gossip
// let me know about anyone new from my hood , here is the storageradius
@ -207,10 +219,8 @@ func (self *Hive) addPeer(p *peer) {
// we do not record as request or forward it, just reply with peers
p.retrieve(&retrieveRequestMsgData{})
glog.V(logger.Detail).Infof("[BZZ] KΛÐΞMLIΛ hive: 'whatsup wheresdaparty' sent to %v", p)
select {
case self.more <- true:
default:
}
return nil
}
// called after peer disconnected
@ -342,7 +352,7 @@ func (self *Hive) peers(req *retrieveRequestMsgData) {
for _, peer := range self.getPeers(key, int(req.MaxPeers)) {
addrs = append(addrs, peer.remoteAddr)
}
glog.V(logger.Debug).Infof("[BZZ] Hive sending %d peer addresses to %v. req.Id: %v, req.Key: %x", len(addrs), req.from, req.Id, req.Key.Log())
glog.V(logger.Debug).Infof("[BZZ] Hive sending %d peer addresses to %v. req.Id: %v, req.Key: %v", len(addrs), req.from, req.Id, req.Key.Log())
peersData := &peersMsgData{
Peers: addrs,

53
swarm/network/kademlia/kaddb.go
View file

@ -26,7 +26,6 @@ type NodeRecord struct {
Meta *json.RawMessage // arbitrary metadata saved for a peer
node Node
connected bool
}
func (self *NodeRecord) setSeen() {
@ -45,7 +44,7 @@ type KadDb struct {
Nodes [][]*NodeRecord
index map[Address]*NodeRecord
cursors []int
lock sync.Mutex
lock sync.RWMutex
purgeInterval time.Duration
initialRetryInterval time.Duration
connRetryExp int
@ -161,10 +160,10 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe
var interval time.Duration
var found bool
var count int
var purge []int
var purge []bool
var delta time.Duration
var cursor int
var count int
var after time.Time
// iterate over columns maximum bucketsize times
@ -181,7 +180,7 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe
proxLimit = po
need = true
}
cursor = self.cursors[po]
purge = make([]bool, len(dbrow))
// there is a missing slot - finding a node to connect to
// select a node record from the relavant kaddb row (of identical prox order)
@ -191,7 +190,7 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe
node = dbrow[cursor]
// skip already connected nodes
if node.connected {
if node.node != nil {
glog.V(logger.Debug).Infof("[KΛÐ]: kaddb record %v (PO%03d:%d/%d) already connected", node.Addr, po, cursor, len(dbrow))
continue ROW
}
@ -208,7 +207,7 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe
}
if delta > self.purgeInterval {
// remove node
purge = append(purge, cursor)
purge[cursor] = true
glog.V(logger.Debug).Infof("[KΛÐ]: kaddb record %v (PO%03d:%d) unreachable since %v. Removed", node.Addr, po, cursor, node.Seen)
continue ROW
}
@ -222,41 +221,38 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe
glog.V(logger.Debug).Infof("[KΛÐ]: kaddb record %v (PO%03d:%d) selected as candidate connection %v. seen at %v (%v ago), selectable since %v, retry after %v (in %v)", node.Addr, po, cursor, rounds, node.Seen, delta, node.After, after, interval)
node.After = after
found = true
break ROW
} // ROW
self.cursors[po] = cursor
self.delete(po, purge...)
self.delete(po, purge)
if found {
return node, true, proxLimit
return node, need, proxLimit
}
} // ROUND
if need {
return nil, true, proxLimit
}
} // ROUNDS
return nil, false, proxLimit
return nil, need, proxLimit
}
// deletes the noderecords of a kaddb row corresponding to the indexes
// caller must hold the dblock
// the call is unsafe, no index checks
func (self *KadDb) delete(row int, indexes ...int) {
var prev int
func (self *KadDb) delete(row int, purge []bool) {
var nodes []*NodeRecord
dbrow := self.Nodes[row]
for _, next := range indexes {
// need to adjust dbcursor
if next > 0 {
if next <= self.cursors[row] {
self.cursors[row]--
for i, del := range purge {
if i == self.cursors[row] {
//reset cursor
self.cursors[row] = len(nodes)
}
nodes = append(nodes, dbrow[prev:next]...)
// delete the entry to be purged
if del {
delete(self.index, dbrow[i].Addr)
continue
}
prev = next + 1
delete(self.index, dbrow[next].Addr)
// otherwise append to new list
nodes = append(nodes, dbrow[i])
}
self.Nodes[row] = append(nodes, dbrow[prev:]...)
self.Nodes[row] = nodes
}
// save persists kaddb on disk (written to file on path in json format.
@ -306,14 +302,15 @@ func (self *KadDb) load(path string, cb func(*NodeRecord, Node) error) (err erro
return
}
var n int
var purge []int
var purge []bool
for po, b := range self.Nodes {
purge = make([]bool, len(b))
ROW:
for i, node := range b {
if cb != nil {
err = cb(node, node.node)
if err != nil {
purge = append(purge, i)
purge[i] = true
continue ROW
}
}
@ -323,7 +320,7 @@ func (self *KadDb) load(path string, cb func(*NodeRecord, Node) error) (err erro
}
self.index[node.Addr] = node
}
self.delete(po, purge...)
self.delete(po, purge)
}
glog.V(logger.Info).Infof("[KΛÐ] loaded kaddb with %v nodes from %v", n, path)

267
swarm/network/kademlia/kademlia.go
View file

@ -12,16 +12,17 @@ import (
)
const (
bucketSize = 3
proxBinSize = 4
bucketSize = 4
proxBinSize = 2
maxProx = 8
connRetryExp = 2
maxPeers = 100
)
var (
purgeInterval = 42 * time.Hour
initialRetryInterval = 42 * 100 * time.Millisecond
maxIdleInterval = 42 * 10 * time.Second
initialRetryInterval = 42 * time.Millisecond
maxIdleInterval = 42 * 100 * time.Millisecond
)
type KadParams struct {
@ -31,6 +32,7 @@ type KadParams struct {
BucketSize int
PurgeInterval time.Duration
InitialRetryInterval time.Duration
MaxIdleInterval time.Duration
ConnRetryExp int
}
@ -41,6 +43,7 @@ func NewKadParams() *KadParams {
BucketSize: bucketSize,
PurgeInterval: purgeInterval,
InitialRetryInterval: initialRetryInterval,
MaxIdleInterval: maxIdleInterval,
ConnRetryExp: connRetryExp,
}
}
@ -52,7 +55,7 @@ type Kademlia struct {
proxLimit int // state, the PO of the first row of the most proximate bin
proxSize int // state, the number of peers in the most proximate bin
count int // number of active peers (w live connection)
buckets []*bucket // the actual bins
buckets [][]Node // the actual bins
db *KadDb // kaddb, node record database
lock sync.RWMutex // mutex to access buckets
}
@ -68,14 +71,9 @@ type Node interface {
// add is the base address of the table
// params is KadParams configuration
func New(addr Address, params *KadParams) *Kademlia {
buckets := make([]*bucket, params.MaxProx+1)
for i, _ := range buckets {
buckets[i] = &bucket{size: params.BucketSize} // will initialise bucket{int(0),[]Node(nil),sync.Mutex}
}
glog.V(logger.Info).Infof("[KΛÐ] base address %v", addr)
buckets := make([][]Node, params.MaxProx+1)
glog.V(logger.Info).Infof("[KΛÐ] base address %v", addr.String()[:6])
// ! temporary hack fixme:
params.ProxBinSize = 4
return &Kademlia{
addr: addr,
KadParams: params,
@ -109,9 +107,6 @@ func (self *Kademlia) On(node Node, cb func(*NodeRecord, Node) error) (err error
index := self.proximityBin(node.Addr())
record := self.db.findOrCreate(index, node.Addr(), node.Url())
// callback on add node
// setting the node on the record, set it checked (for connectivity)
record.node = node
if cb != nil {
err = cb(record, node)
@ -119,33 +114,46 @@ func (self *Kademlia) On(node Node, cb func(*NodeRecord, Node) error) (err error
if err != nil {
return fmt.Errorf("unable to add node %v, callback error: %v", node.Addr(), err)
}
glog.V(logger.Info).Infof("[KΛÐ]: add node record %v with node %v", record, node)
glog.V(logger.Debug).Infof("[KΛÐ]: add node record %v with node %v", record, node)
}
record.connected = true
// insert in kademlia table of active nodes
bucket := self.buckets[index]
// if bucket is full insertion replaces the worst node
// TODO: give priority to peers with active traffic
replaced, err := bucket.insert(node)
if err != nil {
glog.V(logger.Debug).Infof("[KΛÐ]: node %v not needed: %v", node, err)
return err
// no prox adjustment needed
// do not change count
if len(bucket) >= self.BucketSize { // >= allows us to add peers beyond the bucketsize limitation
// always rotate peers
idle := self.MaxIdleInterval
var pos int
var replaced Node
for i, p := range bucket {
idleInt := time.Since(p.LastActive())
if idleInt > idle {
idle = idleInt
pos = i
replaced = p
}
if replaced != nil {
glog.V(logger.Debug).Infof("[KΛÐ]: node %v replaced by %v ", replaced, node)
return
}
// new node added
glog.V(logger.Info).Infof("[KΛÐ]: add node %v to table", node)
if replaced == nil {
glog.V(logger.Debug).Infof("[KΛÐ]: all peers wanted, PO%03d bucket full", index)
return fmt.Errorf("bucket full")
}
glog.V(logger.Debug).Infof("[KΛÐ]: node %v replaced by %v (idle for %v > %v)", replaced, node, idle, self.MaxIdleInterval)
replaced.Drop()
self.buckets[index] = append(bucket[:pos], bucket[(pos+1):]...)
// there is no change in bucket cardinalities so no prox limit adjustment is needed
return nil
} else {
self.buckets[index] = append(bucket, node)
glog.V(logger.Debug).Infof("[KΛÐ]: add node %v to table", node)
self.count++
self.setProxLimit(index, false)
return
self.setProxLimit(index, true)
}
record.node = node
return nil
}
// is the entrypoint called when a node is taken offline
// Off is the called when a node is taken offline (from the protocol main loop exit)
func (self *Kademlia) Off(node Node, cb func(*NodeRecord, Node)) (err error) {
self.lock.Lock()
defer self.lock.Unlock()
@ -153,70 +161,71 @@ func (self *Kademlia) Off(node Node, cb func(*NodeRecord, Node)) (err error) {
var found bool
index := self.proximityBin(node.Addr())
bucket := self.buckets[index]
for i := 0; i < len(bucket.nodes); i++ {
if node.Addr() == bucket.nodes[i].Addr() {
for i := 0; i < len(bucket); i++ {
if node.Addr() == bucket[i].Addr() {
found = true
bucket.nodes = append(bucket.nodes[:i], bucket.nodes[(i+1):]...)
self.buckets[index] = append(bucket[:i], bucket[(i+1):]...)
break
}
}
if !found {
return
// gracefully return without error if peer already unregistered
glog.V(logger.Warn).Infof("[KΛÐ]: remove node %v not in table, population now is %v", node, self.count)
return nil
}
glog.V(logger.Info).Infof("[KΛÐ]: remove node %v from table", node)
self.count--
if len(bucket.nodes) < bucket.size {
err = fmt.Errorf("insufficient nodes (%v) in bucket %v", len(bucket.nodes), index)
}
glog.V(logger.Debug).Infof("[KΛÐ]: remove node %v from table, population now is %v", node, self.count)
self.setProxLimit(index, false)
self.setProxLimit(index, true)
r := self.db.index[node.Addr()]
record := self.db.index[node.Addr()]
// callback on remove
if cb != nil {
cb(r, r.node)
cb(record, record.node)
}
r.node = nil
r.connected = false
record.node = nil
return
}
// proxLimit is dynamically adjusted so that
// 1) there is no empty buckets in bin < proxLimit and
// 2) the sum of all items sare the maximpossible but lower than ProxBinSize
// 2) the sum of all items are the minimum possible but higher than ProxBinSize
// adjust Prox (proxLimit and proxSize after an insertion/removal of nodes)
// caller holds the lock
func (self *Kademlia) setProxLimit(r int, off bool) {
// glog.V(logger.Info).Infof("[KΛÐ]: adjust proxbin for (bin: %v, off: %v)", r, off)
if r < self.proxLimit && len(self.buckets[r].nodes) > 0 {
func (self *Kademlia) setProxLimit(r int, on bool) {
// if the change is outside the core (PO lower)
// and the change does not leave a bucket empty then
// no adjustment needed
if r < self.proxLimit && len(self.buckets[r]) > 0 {
return
}
glog.V(logger.Detail).Infof("[KΛÐ]: set proxbin (size: %v, limit: %v, bin: %v, off: %v)", self.proxSize, self.proxLimit, r, off)
if off {
// if on=a node was added, then r must be within prox limit so increment cardinality
if on {
self.proxSize++
curr := len(self.buckets[self.proxLimit])
// if now core is big enough without the furthest bucket, then contract
// this can never result in more than one bucket change
if self.proxSize >= self.ProxBinSize+curr && curr > 0 {
self.proxSize -= curr
self.proxLimit++
glog.V(logger.Detail).Infof("[KΛÐ]: proxbin contraction (size: %v, limit: %v, bin: %v)", self.proxSize, self.proxLimit, r)
}
return
}
// otherwise
if r >= self.proxLimit {
self.proxSize--
}
// expand core by lowering prox limit until hit zero or cover the empty bucket or reached target cardinality
for (self.proxSize < self.ProxBinSize || r < self.proxLimit) &&
self.proxLimit > 0 {
//
self.proxLimit--
self.proxSize += len(self.buckets[self.proxLimit].nodes)
glog.V(logger.Detail).Infof("[KΛÐ]: proxbin expansion (size: %v, limit: %v, bin: %v, off: %v)", self.proxSize, self.proxLimit, r, off)
self.proxSize += len(self.buckets[self.proxLimit])
glog.V(logger.Detail).Infof("[KΛÐ]: proxbin expansion (size: %v, limit: %v, bin: %v)", self.proxSize, self.proxLimit, r)
}
// glog.V(logger.Detail).Infof("%v", self)
return
}
self.proxSize++
for self.proxLimit < self.MaxProx &&
len(self.buckets[self.proxLimit].nodes) > 0 &&
self.proxSize-len(self.buckets[self.proxLimit].nodes) >= self.ProxBinSize {
//
self.proxSize -= len(self.buckets[self.proxLimit].nodes)
self.proxLimit++
glog.V(logger.Detail).Infof("[KΛÐ]: proxbin contraction (size: %v, limit: %v, bin: %v, off: %v)", self.proxSize, self.proxLimit, r, off)
}
// glog.V(logger.Detail).Infof("%v", self)
}
/*
@ -227,60 +236,55 @@ proxLimit and MaxProx.
func (self *Kademlia) FindClosest(target Address, max int) []Node {
defer self.lock.RUnlock()
self.lock.RLock()
r := nodesByDistance{
target: target,
}
index := self.proximityBin(target)
start := index
var down bool
if index >= self.proxLimit {
po := self.proximityBin(target)
index := po
step := 1
// set proxbin iteration full
if index > self.proxLimit {
index = self.proxLimit
start = self.MaxProx
down = true
}
var n int
// if max is set to 0, just want a full bucket, dynamic number
min := max
// set limit to max
limit := max
if max == 0 {
limit = 1000
min = 1
limit = maxPeers
}
for {
bucket := self.buckets[start].nodes
for i := 0; i < len(bucket); i++ {
r.push(bucket[i], limit)
var n int
for index >= 0 {
// add entire bucket
for _, p := range self.buckets[index] {
r.push(p, limit)
n++
}
if max == 0 && start <= index && (n > 0 || start == 0) || max > 0 && down && start <= index && (n >= limit || n == self.count || start == 0) {
// terminate if index reached the bottom or enough peers > min
if n >= min && (step < 0 || max == 0) {
break
}
if down {
start--
} else {
if start == self.MaxProx {
if index == 0 {
break
// reach top most non-empty PO bucket, turn around
if index == self.MaxProx {
index = po
step = -1
}
start = index - 1
down = true
} else {
start++
index += step
}
}
}
glog.V(logger.Detail).Infof("[KΛÐ]: serve %d (=<%d) nodes for target lookup %v (PO%d)", n, self.MaxProx, target, index)
glog.V(logger.Detail).Infof("[KΛÐ]: serve %d (<=%d) nodes for target lookup %v (PO%03d)", n, max, target, po)
return r.nodes
}
func (self *Kademlia) binsize(p int) int {
b := self.buckets[p]
defer b.lock.RUnlock()
b.lock.RLock()
return len(b.nodes)
}
func (self *Kademlia) FindBest() (*NodeRecord, bool, int) {
return self.db.findBest(self.BucketSize, self.binsize)
func (self *Kademlia) Suggest() (*NodeRecord, bool, int) {
defer self.lock.RUnlock()
self.lock.RLock()
return self.db.findBest(self.BucketSize, func(i int) int { return len(self.buckets[i]) })
}
// adds node records to kaddb (persisted node record db)
@ -288,13 +292,6 @@ func (self *Kademlia) Add(nrs []*NodeRecord) {
self.db.add(nrs, self.proximityBin)
}
// in situ mutable bucket
type bucket struct {
size int
nodes []Node
lock sync.RWMutex
}
// nodesByDistance is a list of nodes, ordered by distance to target.
type nodesByDistance struct {
nodes []Node
@ -331,27 +328,6 @@ func (h *nodesByDistance) push(node Node, max int) {
}
}
// insert adds a peer to a bucket either by appending to existing items if
// bucket length does not exceed bucketSize, or by replacing the worst
// Node in the bucket
func (self *bucket) insert(node Node) (replaced Node, err error) {
self.lock.Lock()
defer self.lock.Unlock()
if len(self.nodes) >= self.size { // >= allows us to add peers beyond the bucketsize limitation
// dev p2p kicks out nodes idle for > 30 s, so here we never replace nodes if
// bucket is full
return nil, fmt.Errorf("bucket full")
}
self.nodes = append(self.nodes, node)
return
}
func (self *bucket) length(node Node) int {
self.lock.Lock()
defer self.lock.Unlock()
return len(self.nodes)
}
/*
Taking the proximity order relative to a fix point x classifies the points in
the space (n byte long byte sequences) into bins. Items in each are at
@ -396,43 +372,46 @@ func (self *Kademlia) Load(path string, cb func(*NodeRecord, Node) error) (err e
}
// kademlia table + kaddb table displayed with ascii
// callerholds the lock
func (self *Kademlia) String() string {
defer self.lock.RUnlock()
self.lock.RLock()
defer self.db.lock.RUnlock()
self.db.lock.RLock()
var rows []string
rows = append(rows, "=========================================================================")
rows = append(rows, fmt.Sprintf("KΛÐΞMLIΛ hive: queen's address: %v, population: %d (%d)", self.addr, self.count, self.DBCount()))
rows = append(rows, fmt.Sprintf("%v : MaxProx: %d, ProxBinSize: %d, BucketSize: %d, proxLimit: %d, proxSize: %d", time.Now(), self.MaxProx, self.ProxBinSize, self.BucketSize, self.proxLimit, self.proxSize))
rows = append(rows, fmt.Sprintf("%v KΛÐΞMLIΛ hive: queen's address: %v", time.Now().UTC().Format(time.UnixDate), self.addr.String()[:6]))
rows = append(rows, fmt.Sprintf("population: %d (%d), proxLimit: %d, proxSize: %d", self.count, len(self.db.index), self.proxLimit, self.proxSize))
rows = append(rows, fmt.Sprintf("MaxProx: %d, ProxBinSize: %d, BucketSize: %d", self.MaxProx, self.ProxBinSize, self.BucketSize))
for i, b := range self.buckets {
for i, bucket := range self.buckets {
if i == self.proxLimit {
rows = append(rows, fmt.Sprintf("===================== PROX LIMIT: %d =================================", i))
rows = append(rows, fmt.Sprintf("============ PROX LIMIT: %d ==========================================", i))
}
row := []string{fmt.Sprintf("%03d", i), fmt.Sprintf("%2d", len(b.nodes))}
row := []string{fmt.Sprintf("%03d", i), fmt.Sprintf("%2d", len(bucket))}
var k int
c := self.db.cursors[i]
for ; k < len(b.nodes); k++ {
p := b.nodes[(c+k)%len(b.nodes)]
row = append(row, fmt.Sprintf("%s", p.Addr().String()[:8]))
if k == 3 {
for ; k < len(bucket); k++ {
p := bucket[(c+k)%len(bucket)]
row = append(row, p.Addr().String()[:6])
if k == 4 {
break
}
}
for ; k < 3; k++ {
for ; k < 4; k++ {
row = append(row, " ")
}
row = append(row, fmt.Sprintf("| %2d %2d", len(self.db.Nodes[i]), self.db.cursors[i]))
for j, p := range self.db.Nodes[i] {
row = append(row, fmt.Sprintf("%08x", p.Addr[:4]))
if j == 2 {
row = append(row, p.Addr.String()[:6])
if j == 3 {
break
}
}
rows = append(rows, strings.Join(row, " "))
if i == self.MaxProx {
break
}
}
rows = append(rows, "=========================================================================")

32
swarm/network/kademlia/kademlia_test.go
View file

@ -78,11 +78,11 @@ func TestBootstrap(t *testing.T) {
n := 0
for n < 100 {
err = kad.On(node, nil)
if err != nil && err.Error() != "bucket full" {
if err != nil {
t.Fatalf("backend not accepting node: %v", err)
}
record, need, _ := kad.FindBest()
record, need, _ := kad.Suggest()
if !need {
break
}
@ -150,7 +150,7 @@ func TestFindClosest(t *testing.T) {
// check that the result nodes have minimum distance to target.
farthestResult := nodes[len(nodes)-1].Addr()
for i, b := range kad.buckets {
for j, n := range b.nodes {
for j, n := range b {
if contains(nodes, n.Addr()) {
continue // don't run the check below for nodes in result
}
@ -235,12 +235,12 @@ func TestSaveLoad(t *testing.T) {
nodes := kad.FindClosest(self, 100)
path := "/tmp/bzz.peers"
err = kad.Save(path, nil)
if err != nil {
if err != nil && err.Error() != "bucket full" {
t.Fatalf("unepected error saving kaddb: %v", err)
}
kad = New(self, params)
err = kad.Load(path, nil)
if err != nil {
if err != nil && err.Error() != "bucket full" {
t.Fatalf("unepected error loading kaddb: %v", err)
}
for _, b := range kad.db.Nodes {
@ -260,30 +260,30 @@ func TestSaveLoad(t *testing.T) {
}
func (self *Kademlia) proxCheck(t *testing.T) bool {
var sum, i int
var b *bucket
for i, b = range self.buckets {
l := len(b.nodes)
var sum int
for i, b := range self.buckets {
l := len(b)
// if we are in the high prox multibucket
if i >= self.proxLimit {
sum += l
} else if l == 0 {
t.Errorf("bucket %d empty, yet proxLimit is %d\n%v", len(b.nodes), self.proxLimit, self)
t.Errorf("bucket %d empty, yet proxLimit is %d\n%v", len(b), self.proxLimit, self)
return false
}
}
// check if merged high prox bucket does not exceed size
if sum > 0 {
// if sum > self.ProxBinSize {
// t.Errorf("bucket %d is empty, yet proxSize is %d\n%v", i, self.proxSize, self)
// return false
// }
if sum != self.proxSize {
t.Errorf("proxSize incorrect, expected %v, got %v", sum, self.proxSize)
return false
}
if self.proxLimit > 0 && sum+len(self.buckets[self.proxLimit-1].nodes) < self.ProxBinSize {
t.Errorf("proxBinSize incorrect, expected %v got %v", sum, self.proxSize)
last := len(self.buckets[self.proxLimit])
if last > 0 && sum >= self.ProxBinSize+last {
t.Errorf("proxLimit %v incorrect, redundant non-empty bucket %d added to proxBin with %v (target %v)", self.proxLimit, last, sum-last, self.ProxBinSize)
return false
}
if self.proxLimit > 0 && sum < self.ProxBinSize {
t.Errorf("proxLimit %v incorrect. proxSize %v is less than target %v, yet there is more peers", self.proxLimit, sum, self.ProxBinSize)
return false
}
}

15
swarm/network/protocol.go
View file

@ -49,6 +49,7 @@ const (
ErrExtraStatusMsg
ErrSwap
ErrSync
ErrUnwanted
)
var errorToString = map[int]string{
@ -61,6 +62,7 @@ var errorToString = map[int]string{
ErrExtraStatusMsg: "Extra status message",
ErrSwap: "SWAP error",
ErrSync: "Sync error",
ErrUnwanted: "Unwanted peer",
}
// bzz represents the swarm wire protocol
@ -179,7 +181,8 @@ func run(requestDb *storage.LDBDatabase, depo StorageHandler, backend bind.Backe
// the main forever loop that handles incoming requests
for {
if self.hive.blockRead {
time.Sleep(1 * time.Second)
glog.V(logger.Warn).Infof("[BZZ] Cannot read network")
time.Sleep(100 * time.Millisecond)
continue
}
err = self.handle()
@ -223,7 +226,7 @@ func (self *bzz) handle() error {
if err := msg.Decode(&req); err != nil {
return self.protoError(ErrDecode, "<- %v: %v", msg, err)
}
glog.V(logger.Debug).Infof("[BZZ] incoming store request: %s", req.String())
glog.V(logger.Detail).Infof("[BZZ] incoming store request: %s", req.String())
// swap accounting is done within forwarding
self.storage.HandleStoreRequestMsg(&req, &peer{bzz: self})
@ -254,7 +257,7 @@ func (self *bzz) handle() error {
return self.protoError(ErrDecode, "<- %v: %v", msg, err)
}
req.from = &peer{bzz: self}
glog.V(logger.Debug).Infof("[BZZ] <- peer addresses: %v", req)
glog.V(logger.Detail).Infof("[BZZ] <- peer addresses: %v", req)
self.hive.HandlePeersMsg(&req, &peer{bzz: self})
case syncRequestMsg:
@ -366,7 +369,10 @@ func (self *bzz) handleStatus() (err error) {
}
glog.V(logger.Info).Infof("[BZZ] Peer %08x is [bzz] capable (%d/%d)", self.remoteAddr.Addr[:4], status.Version, status.NetworkId)
self.hive.addPeer(&peer{bzz: self})
err = self.hive.addPeer(&peer{bzz: self})
if err != nil {
return self.protoError(ErrUnwanted, "%v", err)
}
// hive sets syncstate so sync should start after node added
glog.V(logger.Info).Infof("[BZZ] syncronisation request sent with %v", self.syncState)
@ -516,7 +522,6 @@ func (self *bzz) send(msg uint64, data interface{}) error {
if self.hive.blockWrite {
return fmt.Errorf("network write blocked")
}
// self.messages = append(self.messages, "")
glog.V(logger.Detail).Infof("[BZZ] -> %v: %v (%T) to %v", msg, data, data, self)
err := p2p.Send(self.rw, msg, data)
if err != nil {

30
swarm/network/syncdb.go
View file

@ -126,7 +126,7 @@ LOOP:
// if syncdb is stopped. In this case we need to save the item to the db
more = deliver(req, self.quit)
if !more {
glog.V(logger.Debug).Infof("[BZZ] syncDb[%v] quit: switching to db. session tally (db/total): %v/%v", self.priority, self.dbTotal, self.total)
glog.V(logger.Debug).Infof("[BZZ] syncDb[%v/%v] quit: switching to db. session tally (db/total): %v/%v", self.key.Log(), self.priority, self.dbTotal, self.total)
// received quit signal, save request currently waiting delivery
// by switching to db mode and closing the buffer
buffer = nil
@ -136,12 +136,12 @@ LOOP:
break // break from select, this item will be written to the db
}
self.total++
glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] deliver (db/total): %v/%v", self.priority, self.dbTotal, self.total)
glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] deliver (db/total): %v/%v", self.key.Log(), self.priority, self.dbTotal, self.total)
// by the time deliver returns, there were new writes to the buffer
// if buffer contention is detected, switch to db mode which drains
// the buffer so no process will block on pushing store requests
if len(buffer) == cap(buffer) {
glog.V(logger.Debug).Infof("[BZZ] syncDb[%v] buffer full %v: switching to db. session tally (db/total): %v/%v", self.priority, cap(buffer), self.dbTotal, self.total)
glog.V(logger.Debug).Infof("[BZZ] syncDb[%v/%v] buffer full %v: switching to db. session tally (db/total): %v/%v", self.key.Log(), self.priority, cap(buffer), self.dbTotal, self.total)
buffer = nil
db = self.buffer
}
@ -154,18 +154,18 @@ LOOP:
binary.BigEndian.PutUint64(counterValue, counter)
batch.Put(self.counterKey, counterValue) // persist counter in batch
self.writeSyncBatch(batch) // save batch
glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] quitting: save current batch to db", self.priority)
glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] quitting: save current batch to db", self.key.Log(), self.priority)
break LOOP
}
self.dbTotal++
self.total++
// otherwise break after selec
// otherwise break after select
case dbSize = <-self.batch:
// explicit request for batch
if inBatch == 0 && quit != nil {
// there was no writes since the last batch so db depleted
// switch to buffer mode
glog.V(logger.Debug).Infof("[BZZ] syncDb[%v] empty db: switching to buffer", self.priority)
glog.V(logger.Debug).Infof("[BZZ] syncDb[%v/%v] empty db: switching to buffer", self.key.Log(), self.priority)
db = nil
buffer = self.buffer
dbSize <- 0 // indicates to 'caller' that batch has been written
@ -174,7 +174,7 @@ LOOP:
}
binary.BigEndian.PutUint64(counterValue, counter)
batch.Put(self.counterKey, counterValue)
glog.V(logger.Debug).Infof("[BZZ] syncDb[%v] write batch %v/%v - %x - %x", self.priority, inBatch, counter, self.counterKey, counterValue)
glog.V(logger.Debug).Infof("[BZZ] syncDb[%v/%v] write batch %v/%v - %x - %x", self.key.Log(), self.priority, inBatch, counter, self.counterKey, counterValue)
batch = self.writeSyncBatch(batch)
dbSize <- inBatch // indicates to 'caller' that batch has been written
inBatch = 0
@ -186,7 +186,7 @@ LOOP:
db = self.buffer
buffer = nil
quit = nil
glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] quitting: save buffer to db", self.priority)
glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] quitting: save buffer to db", self.key.Log(), self.priority)
close(db)
continue LOOP
}
@ -194,15 +194,15 @@ LOOP:
// only get here if we put req into db
entry, err = self.newSyncDbEntry(req, counter)
if err != nil {
glog.V(logger.Warn).Infof("[BZZ] syncDb[%v] saving request %v (#%v/%v) failed: %v", self.priority, req, inBatch, inDb, err)
glog.V(logger.Warn).Infof("[BZZ] syncDb[%v/%v] saving request %v (#%v/%v) failed: %v", self.key.Log(), self.priority, req, inBatch, inDb, err)
continue LOOP
}
batch.Put(entry.key, entry.val)
glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] to batch %v '%v' (#%v/%v/%v)", self.priority, req, entry, inBatch, inDb, counter)
glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] to batch %v '%v' (#%v/%v/%v)", self.key.Log(), self.priority, req, entry, inBatch, inDb, counter)
// if just switched to db mode and not quitting, then launch dbRead
// in a parallel go routine to send deliveries from db
if inDb == 0 && quit != nil {
glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] start dbRead")
glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] start dbRead")
go self.dbRead(true, counter, deliver)
}
inDb++
@ -221,7 +221,7 @@ LOOP:
func (self *syncDb) writeSyncBatch(batch *leveldb.Batch) *leveldb.Batch {
err := self.db.Write(batch)
if err != nil {
glog.V(logger.Warn).Infof("[BZZ] syncDb[%v] saving batch to db failed: %v", self.priority, err)
glog.V(logger.Warn).Infof("[BZZ] syncDb[%v/%v] saving batch to db failed: %v", self.key.Log(), self.priority, err)
return batch
}
return new(leveldb.Batch)
@ -295,7 +295,7 @@ func (self *syncDb) dbRead(useBatches bool, counter uint64, fun func(interface{}
continue
}
del = new(leveldb.Batch)
glog.V(logger.Detail).Infof("[BZZ] syncDb[%v]: new iterator: %x (batch %v, count %v)", self.priority, key, batches, cnt)
glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v]: new iterator: %x (batch %v, count %v)", self.key.Log(), self.priority, key, batches, cnt)
for n = 0; !useBatches || n < cnt; it.Next() {
copy(key, it.Key())
@ -307,11 +307,11 @@ func (self *syncDb) dbRead(useBatches bool, counter uint64, fun func(interface{}
val := make([]byte, 40)
copy(val, it.Value())
entry = &syncDbEntry{key, val}
// glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] - %v, batches: %v, total: %v, session total from db: %v/%v", self.priority, self.key.Log(), batches, total, self.dbTotal, self.total)
// glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] - %v, batches: %v, total: %v, session total from db: %v/%v", self.key.Log(), self.priority, self.key.Log(), batches, total, self.dbTotal, self.total)
more = fun(entry, self.quit)
if !more {
// quit received when waiting to deliver entry, the entry will not be deleted
glog.V(logger.Detail).Infof("[BZZ] syncDb[%v] batch %v quit after %v/%v items", self.priority, batches, n, cnt)
glog.V(logger.Detail).Infof("[BZZ] syncDb[%v/%v] batch %v quit after %v/%v items", self.key.Log(), self.priority, batches, n, cnt)
break
}
// since subsequent batches of the same db session are indexed incrementally

28
swarm/network/syncer.go
View file

@ -298,6 +298,7 @@ func (self *syncer) sync() {
if state.LastSeenAt < state.SessionAt {
state.Last = state.SessionAt
glog.V(logger.Debug).Infof("[BZZ] syncer[%v]: start syncronising history since last disconnect at %v up until session start at %v: %v", self.key.Log(), state.LastSeenAt, state.SessionAt, state)
// blocks until state syncing is finished
self.syncState(state)
}
glog.V(logger.Info).Infof("[BZZ] syncer[%v]: syncing all history complete", self.key.Log())
@ -358,7 +359,9 @@ func (self *syncer) syncHistory(state *syncState) chan interface{} {
IT:
for {
key := it.Next()
if key != nil {
if key == nil {
break IT
}
select {
// blocking until history channel is read from
case history <- storage.Key(key):
@ -368,9 +371,6 @@ func (self *syncer) syncHistory(state *syncState) chan interface{} {
case <-self.quit:
return
}
} else {
break IT
}
}
glog.V(logger.Debug).Infof("[BZZ] syncer[%v]: finished syncing history between %v - %v for chunk addresses %v - %v (at %v) (chunks = %v)", self.key.Log(), state.First, state.Last, state.Start, state.Stop, state.Latest, n)
}()
@ -416,26 +416,32 @@ LOOP:
// are checked first - integrity can only be guaranteed if writing
// is locked while selecting
if priority != High || len(keys) == 0 {
// selection is not needed if the High priority queue has items
keys = nil
PRIORITIES:
for priority = High; priority >= 0; priority-- {
// the first priority channel that is non-empty will be assigned to keys
if len(self.keys[priority]) > 0 {
glog.V(logger.Detail).Infof("[BZZ] syncer[%v]: reading request with priority %v", self.key.Log(), priority)
keys = self.keys[priority]
break
break PRIORITIES
}
glog.V(logger.Debug).Infof("[BZZ] syncer[%v/%v]: queue: [%v, %v, %v]", self.key.Log(), priority, len(self.keys[High]), len(self.keys[Medium]), len(self.keys[Low]))
// if the input queue is empty on this level, resort to history if there is any
if uint(priority) == histPrior && history != nil {
glog.V(logger.Detail).Infof("[BZZ] syncer[%v]: reading history for %v", self.key.Log(), self.key)
keys = history
break
break PRIORITIES
}
}
}
// if peer ready to receive but nothing to send
if keys == nil && deliveryRequest == nil {
// if no items left and switch to waiting mode
glog.V(logger.Detail).Infof("[BZZ] syncer[%v]: buffers consumed. Waiting", self.key.Log())
newUnsyncedKeys = self.newUnsyncedKeys
}
}
// send msg iff
// * peer is ready to receive keys AND (
@ -447,7 +453,7 @@ LOOP:
len(unsynced) > 0 && keys == nil ||
len(unsynced) == int(self.SyncBatchSize)) {
justSynced = false
// listen to requests again
// listen to requests
deliveryRequest = self.deliveryRequest
newUnsyncedKeys = nil // not care about data until next req comes in
// set sync to current counter
@ -458,11 +464,11 @@ LOOP:
// send the unsynced keys
stateCopy := *state
err := self.unsyncedKeys(unsynced, &stateCopy)
self.state = state
glog.V(logger.Debug).Infof("[BZZ] syncer[%v]: --> %v keys sent: (total: %v (%v), history: %v), sent sync state: %v", self.key.Log(), len(unsynced), keyCounts, keyCount, historyCnt, stateCopy)
if err != nil {
glog.V(logger.Warn).Infof("[BZZ] syncer[%v]: unable to send unsynced keys: %v", err)
}
self.state = state
glog.V(logger.Debug).Infof("[BZZ] syncer[%v]: --> %v keys sent: (total: %v (%v), history: %v), sent sync state: %v", self.key.Log(), len(unsynced), keyCounts, keyCount, historyCnt, stateCopy)
unsynced = nil
keys = nil
}
@ -579,7 +585,7 @@ func (self *syncer) syncDeliveries() {
total++
msg, err = self.newStoreRequestMsgData(req)
if err != nil {
glog.V(logger.Warn).Infof("[BZZ] syncer[%v]: failed to deliver %v: %v", self.key.Log(), req, err)
glog.V(logger.Warn).Infof("[BZZ] syncer[%v]: failed to create store request for %v: %v", self.key.Log(), req, err)
} else {
err = self.store(msg)
if err != nil {

431
swarm/storage/chunker.go
View file

@ -2,10 +2,11 @@ package storage
import (
"encoding/binary"
"errors"
"fmt"
"hash"
"io"
"sync"
"time"
"github.com/ethereum/go-ethereum/logger"
"github.com/ethereum/go-ethereum/logger/glog"
@ -37,11 +38,9 @@ data_{i} := size(subtree_{i}) || key_{j} || key_{j+1} .... || key_{j+n-1}
*/
const (
// defaultHash = "SHA3" // http://golang.org/pkg/hash/#Hash
defaultHash = "SHA256" // http://golang.org/pkg/hash/#Hash
defaultHash = "SHA3" // http://golang.org/pkg/hash/#Hash
// defaultHash = "SHA256" // http://golang.org/pkg/hash/#Hash
defaultBranches int64 = 128
joinTimeout = 120 // second
splitTimeout = 120 // second
// hashSize int64 = hasherfunc.New().Size() // hasher knows about its own length in bytes
// chunksize int64 = branches * hashSize // chunk is defined as this
)
@ -57,128 +56,105 @@ The hashing itself does use extra copies and allocation though, since it does ne
type ChunkerParams struct {
Branches int64
Hash string
JoinTimeout time.Duration
SplitTimeout time.Duration
}
func NewChunkerParams() *ChunkerParams {
return &ChunkerParams{
Branches: defaultBranches,
Hash: defaultHash,
JoinTimeout: joinTimeout,
SplitTimeout: splitTimeout,
}
}
type TreeChunker struct {
branches int64
hashFunc Hasher
joinTimeout time.Duration
splitTimeout time.Duration
// calculated
hashSize int64 // self.hashFunc.New().Size()
chunkSize int64 // hashSize* branches
workerCount int
}
func NewTreeChunker(params *ChunkerParams) (self *TreeChunker) {
self = &TreeChunker{}
self.hashFunc = MakeHashFunc(params.Hash)
self.branches = params.Branches
self.joinTimeout = params.JoinTimeout * time.Second
self.splitTimeout = params.SplitTimeout * time.Second
self.hashSize = int64(self.hashFunc().Size())
self.chunkSize = self.hashSize * self.branches
self.workerCount = 1
return
}
func (self *TreeChunker) KeySize() int64 {
return self.hashSize
}
// func (self *TreeChunker) KeySize() int64 {
// return self.hashSize
// }
// String() for pretty printing
func (self *Chunk) String() string {
return fmt.Sprintf("Key: %v TreeSize: %v Chunksize: %v", self.Key.Log(), self.Size, len(self.SData))
}
// The treeChunkers own Hash hashes together
// - the size (of the subtree encoded in the Chunk)
// - the Chunk, ie. the contents read from the input reader
func (self *TreeChunker) Hash(input []byte) []byte {
hasher := self.hashFunc()
hasher.Write(input)
return hasher.Sum(nil)
type hashJob struct {
key Key
chunk []byte
size int64
parentWg *sync.WaitGroup
}
func (self *TreeChunker) Split(key Key, data SectionReader, chunkC chan *Chunk, swg *sync.WaitGroup) (errC chan error) {
if swg != nil {
swg.Add(1)
defer swg.Done()
}
func (self *TreeChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, swg, wwg *sync.WaitGroup) (Key, error) {
if self.chunkSize <= 0 {
panic("chunker must be initialised")
}
if int64(len(key)) != self.hashSize {
panic(fmt.Sprintf("root key buffer must be allocated byte slice of length %d", self.hashSize))
}
jobC := make(chan *hashJob, 2*processors)
wg := &sync.WaitGroup{}
errC = make(chan error)
rerrC := make(chan error)
timeout := time.After(self.splitTimeout)
errC := make(chan error)
wg.Add(1)
go func() {
// wwg = workers waitgroup keeps track of hashworkers spawned by this split call
if wwg != nil {
wwg.Add(1)
}
go self.hashWorker(jobC, chunkC, errC, swg, wwg)
depth := 0
treeSize := self.chunkSize
size := data.Size()
// takes lowest depth such that chunksize*HashCount^(depth+1) > size
// power series, will find the order of magnitude of the data size in base hashCount or numbers of levels of branching in the resulting tree.
for ; treeSize < size; treeSize *= self.branches {
depth++
}
key := make([]byte, self.hashFunc().Size())
// glog.V(logger.Detail).Infof("[BZZ] split request received for data (%v bytes, depth: %v)", size, depth)
//launch actual recursive function passing the workgroup
self.split(depth, treeSize/self.branches, key, data, chunkC, rerrC, wg, swg)
}()
// this waitgroup member is released after the root hash is calculated
wg.Add(1)
//launch actual recursive function passing the waitgroups
go self.split(depth, treeSize/self.branches, key, data, size, jobC, chunkC, errC, wg, swg, wwg)
// closes internal error channel if all subprocesses in the workgroup finished
go func() {
// waiting for all threads to finish
wg.Wait()
close(rerrC)
}()
// waiting for request to end with wg finishing, error, or timeout
go func() {
select {
case err := <-rerrC:
if err != nil {
errC <- err
} // otherwise splitting is complete
case <-timeout:
errC <- fmt.Errorf("split time out")
// if storage waitgroup is non-nil, we wait for storage to finish too
if swg != nil {
// glog.V(logger.Detail).Infof("Waiting for storage to finish")
swg.Wait()
}
close(errC)
}()
return
select {
case err := <-errC:
if err != nil {
return nil, err
}
//
}
return key, nil
}
func (self *TreeChunker) split(depth int, treeSize int64, key Key, data SectionReader, chunkC chan *Chunk, errc chan error, parentWg *sync.WaitGroup, swg *sync.WaitGroup) {
defer parentWg.Done()
size := data.Size()
var newChunk *Chunk
var hash Key
// glog.V(logger.Detail).Infof("[BZZ] depth: %v, max subtree size: %v, data size: %v", depth, treeSize, size)
func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reader, size int64, jobC chan *hashJob, chunkC chan *Chunk, errC chan error, parentWg, swg, wwg *sync.WaitGroup) {
for depth > 0 && size < treeSize {
treeSize /= self.branches
@ -187,17 +163,16 @@ func (self *TreeChunker) split(depth int, treeSize int64, key Key, data SectionR
if depth == 0 {
// leaf nodes -> content chunks
chunkData := make([]byte, data.Size()+8)
chunkData := make([]byte, size+8)
binary.LittleEndian.PutUint64(chunkData[0:8], uint64(size))
data.ReadAt(chunkData[8:], 0)
hash = self.Hash(chunkData)
// glog.V(logger.Detail).Infof("[BZZ] content chunk: max subtree size: %v, data size: %v", treeSize, size)
newChunk = &Chunk{
Key: hash,
SData: chunkData,
Size: size,
data.Read(chunkData[8:])
select {
case jobC <- &hashJob{key, chunkData, size, parentWg}:
case <-errC:
}
// glog.V(logger.Detail).Infof("[BZZ] read %v", size)
return
}
} else {
// intermediate chunk containing child nodes hashes
branchCnt := int64((size + treeSize - 1) / treeSize)
// glog.V(logger.Detail).Infof("[BZZ] intermediate node: setting branches: %v, depth: %v, max subtree size: %v, data size: %v", branches, depth, treeSize, size)
@ -216,156 +191,205 @@ func (self *TreeChunker) split(depth int, treeSize int64, key Key, data SectionR
} else {
secSize = treeSize
}
// take the section of the data encoded in the subTree
subTreeData := NewChunkReader(data, pos, secSize)
// the hash of that data
subTreeKey := chunk[8+i*self.hashSize : 8+(i+1)*self.hashSize]
childrenWg.Add(1)
go self.split(depth-1, treeSize/self.branches, subTreeKey, subTreeData, chunkC, errc, childrenWg, swg)
self.split(depth-1, treeSize/self.branches, subTreeKey, data, secSize, jobC, chunkC, errC, childrenWg, swg, wwg)
i++
pos += treeSize
}
// wait for all the children to complete calculating their hashes and copying them onto sections of the chunk
// parentWg.Add(1)
// go func() {
childrenWg.Wait()
if len(jobC) > self.workerCount && self.workerCount < processors {
if wwg != nil {
wwg.Add(1)
}
self.workerCount++
go self.hashWorker(jobC, chunkC, errC, swg, wwg)
}
select {
case jobC <- &hashJob{key, chunk, size, parentWg}:
case <-errC:
}
}
func (self *TreeChunker) hashWorker(jobC chan *hashJob, chunkC chan *Chunk, errC chan error, swg, wwg *sync.WaitGroup) {
hasher := self.hashFunc()
if wwg != nil {
defer wwg.Done()
}
for {
select {
case job, ok := <-jobC:
if !ok {
return
}
// now we got the hashes in the chunk, then hash the chunks
hash = self.Hash(chunk)
newChunk = &Chunk{
Key: hash,
SData: chunk,
Size: size,
hasher.Reset()
self.hashChunk(hasher, job, chunkC, swg)
// glog.V(logger.Detail).Infof("[BZZ] hash chunk (%v)", job.size)
case <-errC:
return
}
}
}
// The treeChunkers own Hash hashes together
// - the size (of the subtree encoded in the Chunk)
// - the Chunk, ie. the contents read from the input reader
func (self *TreeChunker) hashChunk(hasher hash.Hash, job *hashJob, chunkC chan *Chunk, swg *sync.WaitGroup) {
hasher.Write(job.chunk)
h := hasher.Sum(nil)
newChunk := &Chunk{
Key: h,
SData: job.chunk,
Size: job.size,
wg: swg,
}
// report hash of this chunk one level up (keys corresponds to the proper subslice of the parent chunk)
copy(job.key, h)
// send off new chunk to storage
if chunkC != nil {
if swg != nil {
swg.Add(1)
}
}
// send off new chunk to storage
job.parentWg.Done()
if chunkC != nil {
chunkC <- newChunk
}
// report hash of this chunk one level up (keys corresponds to the proper subslice of the parent chunk)x
copy(key, hash)
}
func (self *TreeChunker) Join(key Key, chunkC chan *Chunk) SectionReader {
return &LazyChunkReader{
key: key,
chunkC: chunkC,
quitC: make(chan bool),
errC: make(chan error),
chunker: self,
}
}
// LazyChunkReader implements LazySectionReader
type LazyChunkReader struct {
key Key // root key
chunkC chan *Chunk // chunk channel to send retrieve requests on
size int64 // size of the entire subtree
chunk *Chunk // size of the entire subtree
off int64 // offset
quitC chan bool // channel to abort retrieval
errC chan error // error channel to monitor retrieve errors
chunker *TreeChunker // needs TreeChunker params TODO: should just take
// the chunkSize, branches etc as params
chunkSize int64 // inherit from chunker
branches int64 // inherit from chunker
hashSize int64 // inherit from chunker
}
func (self *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) {
self.errC = make(chan error)
chunk := &Chunk{
Key: self.key,
C: make(chan bool), // close channel to signal data delivery
}
self.chunkC <- chunk // submit retrieval request, someone should be listening on the other side (or we will time out globally)
glog.V(logger.Detail).Infof("[BZZ] readAt: reading %v into %d bytes at offset %d.", chunk.Key.Log(), len(b), off)
// implements the Joiner interface
func (self *TreeChunker) Join(key Key, chunkC chan *Chunk) LazySectionReader {
// waiting for the chunk retrieval
return &LazyChunkReader{
key: key,
chunkC: chunkC,
chunkSize: self.chunkSize,
branches: self.branches,
hashSize: self.hashSize,
}
}
// Size is meant to be called on the LazySectionReader
func (self *LazyChunkReader) Size(quitC chan bool) (n int64, err error) {
if self.chunk != nil {
return self.chunk.Size, nil
}
chunk := retrieve(self.key, self.chunkC, quitC)
if chunk == nil {
select {
case <-self.quitC:
// this is how we control process leakage (quitC is closed once join is finished (after timeout))
// glog.V(logger.Detail).Infof("[BZZ] quit")
return
case <-chunk.C: // bells are ringing, data have been delivered
// glog.V(logger.Detail).Infof("[BZZ] chunk data received for %v", chunk.Key.Log())
case <-quitC:
return 0, errors.New("aborted")
default:
return 0, fmt.Errorf("root chunk not found for %v", self.key.Hex())
}
if len(chunk.SData) == 0 {
// glog.V(logger.Detail).Infof("[BZZ] No payload in %v", chunk.Key.Log())
return 0, notFound
}
chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8]))
self.size = chunk.Size
if b == nil {
self.chunk = chunk
return chunk.Size, nil
}
// read at can be called numerous times
// concurrent reads are allowed
// Size() needs to be called synchronously on the LazyChunkReader first
func (self *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) {
// this is correct, a swarm doc cannot be zero length, so no EOF is expected
if len(b) == 0 {
// glog.V(logger.Detail).Infof("[BZZ] Size query for %v", chunk.Key.Log())
return
return 0, nil
}
want := int64(len(b))
if off+want > self.size {
want = self.size - off
quitC := make(chan bool)
size, err := self.Size(quitC)
if err != nil {
return 0, err
}
glog.V(logger.Detail).Infof("readAt: len(b): %v, off: %v, size: %v ", len(b), off, size)
errC := make(chan error)
// glog.V(logger.Detail).Infof("[BZZ] readAt: reading %v into %d bytes at offset %d.", self.chunk.Key.Log(), len(b), off)
// }
// glog.V(logger.Detail).Infof("-> want: %v, off: %v size: %v ", want, off, self.size)
var treeSize int64
var depth int
// calculate depth and max treeSize
treeSize = self.chunker.chunkSize
for ; treeSize < chunk.Size; treeSize *= self.chunker.branches {
treeSize = self.chunkSize
for ; treeSize < size; treeSize *= self.branches {
depth++
}
wg := sync.WaitGroup{}
wg.Add(1)
go self.join(b, off, off+want, depth, treeSize/self.chunker.branches, chunk, &wg)
go self.join(b, off, off+int64(len(b)), depth, treeSize/self.branches, self.chunk, &wg, errC, quitC)
go func() {
wg.Wait()
close(self.errC)
close(errC)
}()
select {
case err = <-self.errC:
err = <-errC
if err != nil {
close(quitC)
return 0, err
}
// glog.V(logger.Detail).Infof("[BZZ] ReadAt received %v", err)
read = len(b)
if off+int64(read) == self.size {
err = io.EOF
glog.V(logger.Detail).Infof("end: len(b): %v, off: %v, size: %v ", len(b), off, size)
if off+int64(len(b)) >= size {
glog.V(logger.Detail).Infof(" len(b): %v EOF", len(b))
return len(b), io.EOF
}
// glog.V(logger.Detail).Infof("[BZZ] ReadAt returning at %d: %v", read, err)
case <-self.quitC:
// glog.V(logger.Detail).Infof("[BZZ] ReadAt aborted at %d: %v", read, err)
}
return
return len(b), nil
}
func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeSize int64, chunk *Chunk, parentWg *sync.WaitGroup) {
func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeSize int64, chunk *Chunk, parentWg *sync.WaitGroup, errC chan error, quitC chan bool) {
defer parentWg.Done()
// return NewDPA(&LocalStore{})
glog.V(logger.Detail).Infof("inh len(b): %v, off: %v eoff: %v ", len(b), off, eoff)
// glog.V(logger.Detail).Infof("[BZZ] depth: %v, loff: %v, eoff: %v, chunk.Size: %v, treeSize: %v", depth, off, eoff, chunk.Size, treeSize)
chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8]))
// chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8]))
// find appropriate block level
for chunk.Size < treeSize && depth > 0 {
treeSize /= self.chunker.branches
treeSize /= self.branches
depth--
}
// leaf chunk found
if depth == 0 {
// glog.V(logger.Detail).Infof("[BZZ] depth: %v, len(b): %v, off: %v, eoff: %v, chunk.Size: %v, treeSize: %v", depth, len(b), off, eoff, chunk.Size, treeSize)
if int64(len(b)) != eoff-off {
//fmt.Printf("len(b) = %v off = %v eoff = %v", len(b), off, eoff)
panic("len(b) does not match")
}
glog.V(logger.Detail).Infof("[BZZ] depth: %v, len(b): %v, off: %v, eoff: %v, chunk.Size: %v, treeSize: %v", depth, len(b), off, eoff, chunk.Size, treeSize)
copy(b, chunk.SData[8+off:8+eoff])
return // simply give back the chunks reader for content chunks
}
// subtree index
// subtree
start := off / treeSize
end := (eoff + treeSize - 1) / treeSize
wg := sync.WaitGroup{}
wg := &sync.WaitGroup{}
defer wg.Wait()
glog.V(logger.Detail).Infof("[BZZ] start %v,end %v", start, end)
for i := start; i < end; i++ {
soff := i * treeSize
roff := soff
seoff := soff + treeSize
@ -376,36 +400,91 @@ func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, tr
if seoff > eoff {
seoff = eoff
}
if depth > 1 {
wg.Wait()
}
wg.Add(1)
go func(j int64) {
childKey := chunk.SData[8+j*self.chunker.hashSize : 8+(j+1)*self.chunker.hashSize]
// glog.V(logger.Detail).Infof("[BZZ] subtree index: %v -> %v", j, childKey.Log())
ch := &Chunk{
Key: childKey,
C: make(chan bool), // close channel to signal data delivery
}
// glog.V(logger.Detail).Infof("[BZZ] chunk data sent for %v (key interval in chunk %v-%v)", ch.Key.Log(), j*self.chunker.hashSize, (j+1)*self.chunker.hashSize)
self.chunkC <- ch // submit retrieval request, someone should be listening on the other side (or we will time out globally)
// waiting for the chunk retrieval
childKey := chunk.SData[8+j*self.hashSize : 8+(j+1)*self.hashSize]
// glog.V(logger.Detail).Infof("[BZZ] subtree ind.ex: %v -> %v", j, childKey.Log())
chunk := retrieve(childKey, self.chunkC, quitC)
if chunk == nil {
select {
case <-self.quitC:
// this is how we control process leakage (quitC is closed once join is finished (after timeout))
case errC <- fmt.Errorf("chunk %v-%v not found", off, off+treeSize):
case <-quitC:
}
return
case <-ch.C: // bells are ringing, data have been delivered
// glog.V(logger.Detail).Infof("[BZZ] chunk data received")
}
if soff < off {
soff = off
}
if len(ch.SData) == 0 {
self.errC <- fmt.Errorf("chunk %v-%v not found", off, off+treeSize)
return
}
self.join(b[soff-off:seoff-off], soff-roff, seoff-roff, depth-1, treeSize/self.chunker.branches, ch, &wg)
self.join(b[soff-off:seoff-off], soff-roff, seoff-roff, depth-1, treeSize/self.branches, chunk, wg, errC, quitC)
}(i)
} //for
wg.Wait()
}
// the helper method submits chunks for a key to a oueue (DPA) and
// block until they time out or arrive
// abort if quitC is readable
func retrieve(key Key, chunkC chan *Chunk, quitC chan bool) *Chunk {
chunk := &Chunk{
Key: key,
C: make(chan bool), // close channel to signal data delivery
}
// glog.V(logger.Detail).Infof("[BZZ] chunk data sent for %v (key interval in chunk %v-%v)", ch.Key.Log(), j*self.chunker.hashSize, (j+1)*self.chunker.hashSize)
// submit chunk for retrieval
select {
case chunkC <- chunk: // submit retrieval request, someone should be listening on the other side (or we will time out globally)
case <-quitC:
return nil
}
// waiting for the chunk retrieval
select { // chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8]))
case <-quitC:
// this is how we control process leakage (quitC is closed once join is finished (after timeout))
return nil
case <-chunk.C: // bells are ringing, data have been delivered
// glog.V(logger.Detail).Infof("[BZZ] chunk data received")
}
if len(chunk.SData) == 0 {
return nil // chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8]))
}
return chunk
}
// Read keeps a cursor so cannot be called simulateously, see ReadAt
func (self *LazyChunkReader) Read(b []byte) (read int, err error) {
read, err = self.ReadAt(b, self.off)
glog.V(logger.Detail).Infof("[BZZ] read: %v, off: %v, error: %v", read, self.off, err)
self.off += int64(read)
return
}
// completely analogous to standard SectionReader implementation
var errWhence = errors.New("Seek: invalid whence")
var errOffset = errors.New("Seek: invalid offset")
func (s *LazyChunkReader) Seek(offset int64, whence int) (int64, error) {
switch whence {
default:
return 0, errWhence
case 0:
offset += 0
case 1:
offset += s.off
case 2:
if s.chunk == nil {
return 0, fmt.Errorf("seek from the end requires rootchunk for size. call Size first")
}
offset += s.chunk.Size
}
if offset < 0 {
return 0, errOffset
}
s.off = offset
return offset, nil
}

260
swarm/storage/chunker_test.go
View file

@ -2,20 +2,33 @@ package storage
import (
"bytes"
// "fmt"
"fmt"
"io"
"runtime"
"sync"
"testing"
"time"
"github.com/ethereum/go-ethereum/logger"
"github.com/ethereum/go-ethereum/logger/glog"
)
func init() {
glog.SetV(logger.Info)
glog.SetToStderr(true)
}
/*
Tests TreeChunker by splitting and joining a random byte slice
*/
type test interface {
Fatalf(string, ...interface{})
}
type chunkerTester struct {
errors []error
chunks []*Chunk
timeout bool
t test
}
func (self *chunkerTester) checkChunks(t *testing.T, want int) {
@ -25,77 +38,70 @@ func (self *chunkerTester) checkChunks(t *testing.T, want int) {
}
}
func (self *chunkerTester) Split(chunker *TreeChunker, l int) (key Key, input []byte) {
func (self *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, chunkC chan *Chunk, swg *sync.WaitGroup) (key Key) {
// reset
self.errors = nil
self.chunks = nil
self.timeout = false
data, slice := testDataReader(l)
input = slice
key = make([]byte, 32)
chunkC := make(chan *Chunk, 1000)
errC := chunker.Split(key, data, chunkC, nil)
quitC := make(chan bool)
timeout := time.After(600 * time.Second)
if chunkC != nil {
go func() {
LOOP:
for {
select {
case <-timeout:
self.timeout = true
break LOOP
self.t.Fatalf("Join timeout error")
case chunk := <-chunkC:
if chunk != nil {
self.chunks = append(self.chunks, chunk)
} else {
break LOOP
}
case err, ok := <-errC:
if err != nil {
self.errors = append(self.errors, err)
}
// fmt.Printf("err %v", err)
case chunk, ok := <-chunkC:
if !ok {
close(chunkC)
errC = nil
}
}
}
// glog.V(logger.Info).Infof("chunkC closed quitting")
close(quitC)
return
}
// glog.V(logger.Info).Infof("chunk %v received", len(self.chunks))
self.chunks = append(self.chunks, chunk)
if chunk.wg != nil {
chunk.wg.Done()
}
}
}
}()
<-quitC // waiting for it to finish
}
key, err := chunker.Split(data, size, chunkC, swg, nil)
if err != nil {
self.t.Fatalf("Split error: %v", err)
}
if chunkC != nil {
if swg != nil {
// glog.V(logger.Info).Infof("Waiting for storage to finish")
swg.Wait()
// glog.V(logger.Info).Infof("St orage finished")
}
close(chunkC)
}
if chunkC != nil {
<-quitC
}
return
}
func (self *chunkerTester) Join(chunker *TreeChunker, key Key, c int) SectionReader {
func (self *chunkerTester) Join(chunker *TreeChunker, key Key, c int, chunkC chan *Chunk, quitC chan bool) LazySectionReader {
// reset but not the chunks
self.errors = nil
self.timeout = false
chunkC := make(chan *Chunk, 1000)
reader := chunker.Join(key, chunkC)
quitC := make(chan bool)
timeout := time.After(600 * time.Second)
i := 0
go func() {
LOOP:
for {
select {
case <-quitC:
break LOOP
case <-timeout:
self.timeout = true
break LOOP
self.t.Fatalf("Join timeout error")
case chunk := <-chunkC:
case chunk, ok := <-chunkC:
if !ok {
close(quitC)
return
}
i++
// dpaLogger.DebugDetailf("TESTER: chunk request %x", chunk.Key[:4])
// this just mocks the behaviour of a chunk store retrieval
var found bool
for _, ch := range self.chunks {
@ -106,56 +112,58 @@ func (self *chunkerTester) Join(chunker *TreeChunker, key Key, c int) SectionRea
}
}
if !found {
// fmt.Printf("TESTER: chunk unknown for %x", chunk.Key[:4])
self.t.Fatalf("not found ")
}
close(chunk.C)
// dpaLogger.DebugDetailf("TESTER: chunk request served %x", chunk.Key[:4])
}
}
}()
return reader
}
func testRandomData(chunker *TreeChunker, tester *chunkerTester, n int, chunks int, t *testing.T) {
key, input := tester.Split(chunker, n)
func testRandomData(n int, chunks int, t *testing.T) {
chunker := NewTreeChunker(&ChunkerParams{
Branches: 128,
Hash: "SHA3",
})
tester := &chunkerTester{t: t}
data, input := testDataReaderAndSlice(n)
t.Logf(" Key = %x\n", key)
chunkC := make(chan *Chunk, 1000)
swg := &sync.WaitGroup{}
tester.checkChunks(t, chunks)
time.Sleep(100 * time.Millisecond)
splitter := chunker
key := tester.Split(splitter, data, int64(n), chunkC, swg)
reader := tester.Join(chunker, key, 0)
// t.Logf(" Key = %v\n", key)
// tester.checkChunks(t, chunks)
chunkC = make(chan *Chunk, 1000)
quitC := make(chan bool)
reader := tester.Join(chunker, key, 0, chunkC, quitC)
output := make([]byte, n)
r, err := reader.Read(output)
if r != n || err != io.EOF {
t.Errorf("read error read: %v n = %v err = %v\n", r, n, err)
t.Fatalf("read error read: %v n = %v err = %v\n", r, n, err)
}
// t.Logf(" IN: %x\nOUT: %x\n", input, output)
if input != nil {
if !bytes.Equal(output, input) {
t.Errorf("input and output mismatch\n IN: %x\nOUT: %x\n", input, output)
t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", input, output)
}
}
close(chunkC)
<-quitC
}
func TestRandomData(t *testing.T) {
chunker, tester := chunkerAndTester()
testRandomData(chunker, tester, 60, 1, t)
testRandomData(chunker, tester, 179, 5, t)
testRandomData(chunker, tester, 253, 7, t)
// t.Logf("chunks %v", tester.chunks)
testRandomData(60, 1, t)
testRandomData(83, 3, t)
testRandomData(179, 5, t)
testRandomData(253, 7, t)
}
func chunkerAndTester() (chunker *TreeChunker, tester *chunkerTester) {
chunker = NewTreeChunker(&ChunkerParams{
Branches: 2,
Hash: "SHA256",
SplitTimeout: 10,
JoinTimeout: 10,
})
tester = &chunkerTester{}
return
}
func readAll(reader SectionReader, result []byte) {
func readAll(reader LazySectionReader, result []byte) {
size := int64(len(result))
var end int64
@ -169,46 +177,98 @@ func readAll(reader SectionReader, result []byte) {
}
}
func benchReadAll(reader SectionReader) {
size := reader.Size()
func benchReadAll(reader LazySectionReader) {
size, _ := reader.Size(nil)
output := make([]byte, 1000)
for pos := int64(0); pos < size; pos += 1000 {
reader.ReadAt(output, pos)
}
}
func benchmarkJoinRandomData(n int, chunks int, t *testing.B) {
t.StopTimer()
func benchmarkJoin(n int, t *testing.B) {
for i := 0; i < t.N; i++ {
// fmt.Printf("round %v\n", i)
chunker, tester := chunkerAndTester()
key, _ := tester.Split(chunker, n)
// fmt.Printf("split done %v, joining...\n", i)
chunker := NewTreeChunker(&ChunkerParams{
Branches: 128,
Hash: "SHA3",
})
tester := &chunkerTester{t: t}
data := testDataReader(n)
chunkC := make(chan *Chunk, 1000)
swg := &sync.WaitGroup{}
key := tester.Split(chunker, data, int64(n), chunkC, swg)
t.StartTimer()
reader := tester.Join(chunker, key, i)
// fmt.Printf("join done %v, reading...\n", i)
chunkC = make(chan *Chunk, 1000)
quitC := make(chan bool)
reader := tester.Join(chunker, key, i, chunkC, quitC)
t.StopTimer()
benchReadAll(reader)
close(chunkC)
<-quitC
}
}
func benchmarkSplitRandomData(n int, chunks int, t *testing.B) {
func benchmarkSplitTree(n int, t *testing.B) {
t.ReportAllocs()
for i := 0; i < t.N; i++ {
chunker, tester := chunkerAndTester()
tester.Split(chunker, n)
chunker := NewTreeChunker(&ChunkerParams{
Branches: 128,
Hash: "SHA3",
})
tester := &chunkerTester{t: t}
data := testDataReader(n)
// glog.V(logger.Info).Infof("splitting data of length %v", n)
tester.Split(chunker, data, int64(n), nil, nil)
}
stats := new(runtime.MemStats)
runtime.ReadMemStats(stats)
fmt.Println(stats.Sys)
}
func BenchmarkJoinRandomData_100_2(t *testing.B) { benchmarkJoinRandomData(100, 3, t) }
func BenchmarkJoinRandomData_1000_2(t *testing.B) { benchmarkJoinRandomData(1000, 3, t) }
func BenchmarkJoinRandomData_10000_2(t *testing.B) { benchmarkJoinRandomData(10000, 3, t) }
func BenchmarkJoinRandomData_100000_2(t *testing.B) { benchmarkJoinRandomData(100000, 3, t) }
func BenchmarkJoinRandomData_1000000_2(t *testing.B) { benchmarkJoinRandomData(1000000, 3, t) }
func benchmarkSplitPyramid(n int, t *testing.B) {
t.ReportAllocs()
for i := 0; i < t.N; i++ {
splitter := NewPyramidChunker(&ChunkerParams{
Branches: 128,
Hash: "SHA3",
})
tester := &chunkerTester{t: t}
data := testDataReader(n)
// glog.V(logger.Info).Infof("splitting data of length %v", n)
tester.Split(splitter, data, int64(n), nil, nil)
}
stats := new(runtime.MemStats)
runtime.ReadMemStats(stats)
fmt.Println(stats.Sys)
}
func BenchmarkSplitRandomData_100_2(t *testing.B) { benchmarkSplitRandomData(100, 3, t) }
func BenchmarkSplitRandomData_1000_2(t *testing.B) { benchmarkSplitRandomData(1000, 3, t) }
func BenchmarkSplitRandomData_10000_2(t *testing.B) { benchmarkSplitRandomData(10000, 3, t) }
func BenchmarkSplitRandomData_100000_2(t *testing.B) { benchmarkSplitRandomData(100000, 3, t) }
func BenchmarkSplitRandomData_1000000_2(t *testing.B) { benchmarkSplitRandomData(1000000, 3, t) }
func BenchmarkSplitRandomData_10000000_2(t *testing.B) { benchmarkSplitRandomData(10000000, 3, t) }
func BenchmarkJoin_100_2(t *testing.B) { benchmarkJoin(100, t) }
func BenchmarkJoin_1000_2(t *testing.B) { benchmarkJoin(1000, t) }
func BenchmarkJoin_10000_2(t *testing.B) { benchmarkJoin(10000, t) }
func BenchmarkJoin_100000_2(t *testing.B) { benchmarkJoin(100000, t) }
func BenchmarkJoin_1000000_2(t *testing.B) { benchmarkJoin(1000000, t) }
// go test -bench ./bzz -cpuprofile cpu.out -memprofile mem.out
func BenchmarkSplitTree_2(t *testing.B) { benchmarkSplitTree(100, t) }
func BenchmarkSplitTree_2h(t *testing.B) { benchmarkSplitTree(500, t) }
func BenchmarkSplitTree_3(t *testing.B) { benchmarkSplitTree(1000, t) }
func BenchmarkSplitTree_3h(t *testing.B) { benchmarkSplitTree(5000, t) }
func BenchmarkSplitTree_4(t *testing.B) { benchmarkSplitTree(10000, t) }
func BenchmarkSplitTree_4h(t *testing.B) { benchmarkSplitTree(50000, t) }
func BenchmarkSplitTree_5(t *testing.B) { benchmarkSplitTree(100000, t) }
func BenchmarkSplitTree_6(t *testing.B) { benchmarkSplitTree(1000000, t) }
func BenchmarkSplitTree_7(t *testing.B) { benchmarkSplitTree(10000000, t) }
func BenchmarkSplitTree_8(t *testing.B) { benchmarkSplitTree(100000000, t) }
func BenchmarkSplitPyramid_2(t *testing.B) { benchmarkSplitPyramid(100, t) }
func BenchmarkSplitPyramid_2h(t *testing.B) { benchmarkSplitPyramid(500, t) }
func BenchmarkSplitPyramid_3(t *testing.B) { benchmarkSplitPyramid(1000, t) }
func BenchmarkSplitPyramid_3h(t *testing.B) { benchmarkSplitPyramid(5000, t) }
func BenchmarkSplitPyramid_4(t *testing.B) { benchmarkSplitPyramid(10000, t) }
func BenchmarkSplitPyramid_4h(t *testing.B) { benchmarkSplitPyramid(50000, t) }
func BenchmarkSplitPyramid_5(t *testing.B) { benchmarkSplitPyramid(100000, t) }
func BenchmarkSplitPyramid_6(t *testing.B) { benchmarkSplitPyramid(1000000, t) }
func BenchmarkSplitPyramid_7(t *testing.B) { benchmarkSplitPyramid(10000000, t) }
func BenchmarkSplitPyramid_8(t *testing.B) { benchmarkSplitPyramid(100000000, t) }
// godep go test -bench ./swarm/storage -cpuprofile cpu.out -memprofile mem.out

194
swarm/storage/chunkreader.go
View file

@ -1,194 +0,0 @@
package storage
import (
"bytes"
"errors"
"io"
)
type Bounded interface {
Size() int64
}
type Sliced interface {
Slice(int64, int64) (b []byte, err error)
}
// Size, Seek, Read, ReadAt
type SectionReader interface {
Bounded
io.Seeker
io.Reader
io.ReaderAt
}
// ChunkReader implements SectionReader on a section
// of an underlying ReaderAt.
type ChunkReader struct {
r io.ReaderAt
base int64
off int64
limit int64
}
// NewChunkReader returns a ChunkReader that reads from r
// starting at offset off and stops with EOF after n bytes.
func NewChunkReader(r io.ReaderAt, off int64, n int64) *ChunkReader {
return &ChunkReader{r: r, base: off, off: off, limit: off + n}
}
// ByteSliceReader just extends byte.Reader to make base slice accessible
type ByteSliceReader struct {
*bytes.Reader
base []byte
}
func NewByteSliceReader(b []byte) *ByteSliceReader {
return &ByteSliceReader{
base: b,
Reader: bytes.NewReader(b),
}
}
// ByteSliceReader implements the Sliced interface
func (self *ByteSliceReader) Slice(from, to int64) (b []byte, err error) {
if from < 0 || to >= int64(self.Len()) {
err = io.EOF
} else {
b = self.base[from:to]
}
return
}
// NewChunkReaderFromBytes is a convenience shortcut to get a SectionReader over a byte slice
func NewChunkReaderFromBytes(b []byte) *ChunkReader {
return NewChunkReader(NewByteSliceReader(b), 0, int64(len(b)))
}
/*
The following is adapted from io.SectionReader
*/
func (s *ChunkReader) Size() int64 {
return s.limit - s.base
}
var errWhence = errors.New("Seek: invalid whence")
var errOffset = errors.New("Seek: invalid offset")
func (s *ChunkReader) Seek(offset int64, whence int) (int64, error) {
switch whence {
default:
return 0, errWhence
case 0:
offset += s.base
case 1:
offset += s.off
case 2:
offset += s.limit
}
if offset < s.base {
return 0, errOffset
}
s.off = offset
return offset - s.base, nil
}
func (s *ChunkReader) Read(p []byte) (n int, err error) {
if s.off >= s.limit {
return 0, io.EOF
}
if max := s.limit - s.off; int64(len(p)) > max {
p = p[0:max]
}
n, err = s.r.ReadAt(p, s.off)
s.off += int64(n)
return
}
func (s *ChunkReader) ReadAt(p []byte, off int64) (n int, err error) {
if off < 0 || off >= s.limit-s.base {
return 0, io.EOF
}
off += s.base
if max := s.limit - off; int64(len(p)) > max {
p = p[0:max]
n, err = s.r.ReadAt(p, off)
if err == nil {
err = io.EOF
}
return n, err
}
n, err = s.r.ReadAt(p, off)
return
}
// added methods to that ChunkReader implements the Sliced interface
func (s *ChunkReader) Slice(from, to int64) (b []byte, err error) {
if from < 0 || to >= s.Size() {
err = io.EOF
} else {
if sl, ok := s.r.(Sliced); ok {
b, err = sl.Slice(s.base+from, s.base+to)
} else {
err = errors.New("not sliceable base")
}
}
return
}
// added method so that ChunkReader implements the io.WriterTo interface
// WriteTo method is used by io.Copy
// This is so that we avoid one extra step of allocation (if the underlying initial Reader implements Sliced
func (r *ChunkReader) WriteTo(w io.Writer) (n int64, err error) {
var b []byte
var m int
// if b, _ := r.Slice(r.off-r.base, r.limit-r.base); b == nil {
// if slices not available we do it with extra allocation
b = make([]byte, r.limit-r.off)
m, err = r.Read(b)
if err != nil {
return
}
// }
m, err = w.Write(b)
if m > len(b) {
panic("bytes.Reader.WriteTo: invalid Write count")
}
r.off = r.base + int64(m)
n = int64(m)
if m != len(b) && err == nil {
err = io.ErrShortWrite
}
// w
return
}
func (self *LazyChunkReader) Size() (n int64) {
self.ReadAt(nil, 0)
return self.size
}
func (self *LazyChunkReader) Read(b []byte) (read int, err error) {
read, err = self.ReadAt(b, self.off)
self.off += int64(read)
return
}
func (s *LazyChunkReader) Seek(offset int64, whence int) (int64, error) {
switch whence {
default:
return 0, errWhence
case 0:
offset += 0
case 1:
offset += s.off
case 2:
offset += s.size
}
if offset < 0 {
return 0, errOffset
}
s.off = offset
return offset, nil
}

67
swarm/storage/common_test.go
View file

@ -1,6 +1,7 @@
package storage
import (
"bytes"
"crypto/rand"
"io"
"sync"
@ -10,61 +11,39 @@ import (
"github.com/ethereum/go-ethereum/logger/glog"
)
func testDataReader(l int) (r *ChunkReader, slice []byte) {
func testDataReader(l int) (r io.Reader) {
return io.LimitReader(rand.Reader, int64(l))
}
func testDataReaderAndSlice(l int) (r io.Reader, slice []byte) {
slice = make([]byte, l)
if _, err := rand.Read(slice); err != nil {
panic("rand error")
}
r = NewChunkReaderFromBytes(slice)
return
}
func randomChunks(l int64, branches int64, chunkC chan *Chunk) (key Key, errC chan error) {
chunker := NewTreeChunker(&ChunkerParams{
Branches: branches,
Hash: defaultHash,
SplitTimeout: splitTimeout,
})
key = make([]byte, 32)
b := make([]byte, l)
_, err := rand.Read(b)
if err != nil {
panic("no rand")
}
wg := &sync.WaitGroup{}
errC = chunker.Split(key, NewChunkReaderFromBytes(b), chunkC, wg)
wg.Wait()
r = bytes.NewReader(slice)
return
}
func testStore(m ChunkStore, l int64, branches int64, t *testing.T) {
chunkC := make(chan *Chunk)
key, errC := randomChunks(l, branches, chunkC)
SPLIT:
for {
select {
case chunk := <-chunkC:
go func() {
for chunk := range chunkC {
m.Put(chunk)
case err, ok := <-errC:
if err != nil {
t.Errorf("Chunker error: %v", err)
return
}
if !ok {
break SPLIT
}
if chunk.wg != nil {
chunk.wg.Done()
}
}
}()
chunker := NewTreeChunker(&ChunkerParams{
Branches: branches,
Hash: defaultHash,
SplitTimeout: splitTimeout,
})
swg := &sync.WaitGroup{}
key, err := chunker.Split(rand.Reader, l, chunkC, swg, nil)
swg.Wait()
close(chunkC)
chunkC = make(chan *Chunk)
var r SectionReader
r = chunker.Join(key, chunkC)
quit := make(chan bool)
@ -73,22 +52,26 @@ SPLIT:
go func(chunk *Chunk) {
storedChunk, err := m.Get(chunk.Key)
if err == notFound {
glog.V(logger.Detail).Infof("[BZZ] chunk '%x' not found", chunk.Key)
glog.V(logger.Detail).Infof("[BZZ] chunk '%v' not found", chunk.Key.Log())
} else if err != nil {
glog.V(logger.Detail).Infof("[BZZ] error retrieving chunk %x: %v", chunk.Key, err)
glog.V(logger.Detail).Infof("[BZZ] error retrieving chunk %v: %v", chunk.Key.Log(), err)
} else {
chunk.SData = storedChunk.SData
chunk.Size = storedChunk.Size
}
glog.V(logger.Detail).Infof("[BZZ] chunk '%x' not found", chunk.Key[:4])
glog.V(logger.Detail).Infof("[BZZ] chunk '%v' not found", chunk.Key.Log())
close(chunk.C)
}(ch)
}
close(quit)
}()
r := chunker.Join(key, chunkC)
b := make([]byte, l)
n, err := r.ReadAt(b, 0)
if err != io.EOF {
t.Errorf("read error (%v/%v) %v", n, l, err)
close(quit)
t.Fatalf("read error (%v/%v) %v", n, l, err)
}
close(chunkC)
<-quit
}

28
swarm/storage/dpa.go
View file

@ -2,6 +2,7 @@ package storage
import (
"errors"
"io"
"sync"
"time"
@ -72,33 +73,14 @@ func NewDPA(store ChunkStore, params *ChunkerParams) *DPA {
// FS-aware API and httpaccess
// Chunk retrieval blocks on netStore requests with a timeout so reader will
// report error if retrieval of chunks within requested range time out.
func (self *DPA) Retrieve(key Key) SectionReader {
func (self *DPA) Retrieve(key Key) LazySectionReader {
return self.Chunker.Join(key, self.retrieveC)
}
// Public API. Main entry point for document storage directly. Used by the
// FS-aware API and httpaccess
func (self *DPA) Store(data SectionReader, wg *sync.WaitGroup) (key Key, err error) {
key = make([]byte, self.Chunker.KeySize())
errC := self.Chunker.Split(key, data, self.storeC, wg)
SPLIT:
for {
select {
case err, ok := <-errC:
if err != nil {
glog.V(logger.Error).Infof("[BZZ] chunker split error: %v", err)
}
if !ok {
break SPLIT
}
case <-self.quitC:
break SPLIT
}
}
return
func (self *DPA) Store(data io.Reader, size int64, wg *sync.WaitGroup) (key Key, err error) {
return self.Chunker.Split(data, size, self.storeC, nil, wg)
}
func (self *DPA) Start() {
@ -164,7 +146,7 @@ func (self *DPA) storeLoop() {
go func(chunk *Chunk) {
self.Put(chunk)
if chunk.wg != nil {
glog.V(logger.Detail).Infof("[BZZ] DPA.storeLoop %v", chunk.Key.Log())
glog.V(logger.Detail).Infof("[BZZ] dpa: store loop %v", chunk.Key.Log())
chunk.wg.Done()
}
}(ch)

8
swarm/storage/dpa_test.go
View file

@ -29,9 +29,9 @@ func TestDPArandom(t *testing.T) {
ChunkStore: localStore,
}
dpa.Start()
reader, slice := testDataReader(testDataSize)
reader, slice := testDataReaderAndSlice(testDataSize)
wg := &sync.WaitGroup{}
key, err := dpa.Store(reader, wg)
key, err := dpa.Store(reader, testDataSize, wg)
if err != nil {
t.Errorf("Store error: %v", err)
}
@ -85,9 +85,9 @@ func TestDPA_capacity(t *testing.T) {
ChunkStore: localStore,
}
dpa.Start()
reader, slice := testDataReader(testDataSize)
reader, slice := testDataReaderAndSlice(testDataSize)
wg := &sync.WaitGroup{}
key, err := dpa.Store(reader, wg)
key, err := dpa.Store(reader, testDataSize, wg)
if err != nil {
t.Errorf("Store error: %v", err)
}

5
swarm/storage/localstore.go
View file

@ -1,5 +1,9 @@
package storage
import (
"encoding/binary"
)
// LocalStore is a combination of inmemory db over a disk persisted db
// implements a Get/Put with fallback (caching) logic using any 2 ChunkStores
type LocalStore struct {
@ -48,6 +52,7 @@ func (self *LocalStore) Get(key Key) (chunk *Chunk, err error) {
if err != nil {
return
}
chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8]))
self.memStore.Put(chunk)
return
}

36
swarm/storage/memstore.go
View file

@ -3,7 +3,6 @@
package storage
import (
"bytes"
"sync"
)
@ -44,41 +43,6 @@ func NewMemStore(d *DbStore, capacity uint) (m *MemStore) {
return
}
func (x Key) Size() uint {
return uint(len(x))
}
func (x Key) isEqual(y Key) bool {
return bytes.Compare(x, y) == 0
}
func (h Key) bits(i, j uint) uint {
ii := i >> 3
jj := i & 7
if ii >= h.Size() {
return 0
}
if jj+j <= 8 {
return uint((h[ii] >> jj) & ((1 << j) - 1))
}
res := uint(h[ii] >> jj)
jj = 8 - jj
j -= jj
for j != 0 {
ii++
if j < 8 {
res += uint(h[ii]&((1<<j)-1)) << jj
return res
}
res += uint(h[ii]) << jj
jj += 8
j -= 8
}
return res
}
type memTree struct {
subtree []*memTree
parent *memTree

3
swarm/storage/netstore.go
View file

@ -91,7 +91,8 @@ func (self *NetStore) Put(entry *Chunk) {
} else {
glog.V(logger.Detail).Infof("[BZZ] NetStore.Put: localStore.Put %v stored locally", entry.Key.Log())
// handle propagating store requests
go self.cloud.Store(entry)
// go self.cloud.Store(entry)
self.cloud.Store(entry)
}
}

165
swarm/storage/pyramid.go Normal file
View file

@ -0,0 +1,165 @@
package storage
import (
"io"
"math"
"sync"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/logger"
"github.com/ethereum/go-ethereum/logger/glog"
)
const (
processors = 8
)
type Tree struct {
Chunks int64
Levels []map[int64]*Node
Lock sync.RWMutex
}
type Node struct {
Pending int64
Children []common.Hash
Last bool
}
type Task struct {
Index int64 // Index of the chunk being processed
Data []byte // Binary blob of the chunk
Last bool
}
type PyramidChunker struct {
hashFunc Hasher
chunkSize int64
hashSize int64
branches int64
workerCount int
}
func NewPyramidChunker(params *ChunkerParams) (self *PyramidChunker) {
self = &PyramidChunker{}
self.hashFunc = MakeHashFunc(params.Hash)
self.branches = params.Branches
self.hashSize = int64(self.hashFunc().Size())
self.chunkSize = self.hashSize * self.branches
self.workerCount = 1
return
}
func (self *PyramidChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, swg, wwg *sync.WaitGroup) (Key, error) {
chunks := (size + self.chunkSize - 1) / self.chunkSize
depth := int(math.Ceil(math.Log(float64(chunks))/math.Log(float64(self.branches)))) + 1
glog.V(logger.Detail).Infof("chunks: %v, depth: %v", chunks, depth)
results := Tree{
Chunks: chunks,
Levels: make([]map[int64]*Node, depth),
}
for i := 0; i < depth; i++ {
results.Levels[i] = make(map[int64]*Node)
}
// Create a pool of workers to crunch through the file
tasks := make(chan *Task, 2*processors)
pend := new(sync.WaitGroup)
abortC := make(chan bool)
for i := 0; i < processors; i++ {
pend.Add(1)
go self.processor(pend, tasks, &results)
}
// Feed the chunks into the task pool
for index := 0; ; index++ {
buffer := make([]byte, self.chunkSize+8)
n, err := io.ReadFull(data, buffer)
last := err == io.ErrUnexpectedEOF
if err != nil && !last {
glog.V(logger.Info).Infof("error: %v", err)
close(abortC)
}
pend.Add(1)
// glog.V(logger.Info).Infof("-> task %v (%v)", index, n)
select {
case tasks <- &Task{Index: int64(index), Data: buffer[:n+8], Last: last}:
case <-abortC:
return nil, err
}
if last {
// glog.V(logger.Info).Infof("last task %v (%v)", index, n)
break
}
}
// Wait for the workers and return
close(tasks)
pend.Wait()
// glog.V(logger.Info).Infof("len: %v", results.Levels[0][0])
key := results.Levels[0][0].Children[0][:]
return key, nil
}
func (self *PyramidChunker) processor(pend *sync.WaitGroup, tasks chan *Task, results *Tree) {
defer pend.Done()
// glog.V(logger.Info).Infof("processor started")
// Start processing leaf chunks ad infinitum
hasher := self.hashFunc()
for task := range tasks {
depth, pow := len(results.Levels)-1, self.branches
// glog.V(logger.Info).Infof("task: %v, last: %v", task.Index, task.Last)
var node *Node
for depth >= 0 {
// New chunk received, reset the hasher and start processing
hasher.Reset()
if node == nil { // Leaf node, hash the data chunk
hasher.Write(task.Data)
} else { // Internal node, hash the children
for _, hash := range node.Children {
hasher.Write(hash[:])
}
}
hash := hasher.Sum(nil)
last := task.Last || (node != nil) && node.Last
// Insert the subresult into the memoization tree
results.Lock.Lock()
if node = results.Levels[depth][task.Index/pow]; node == nil {
// Figure out the pending tasks
pending := self.branches
if task.Index/pow == results.Chunks/pow {
pending = (results.Chunks + pow/self.branches - 1) / (pow / self.branches) % self.branches
}
node = &Node{pending, make([]common.Hash, pending), last}
results.Levels[depth][task.Index/pow] = node
}
node.Pending--
i := task.Index / (pow / self.branches) % self.branches
if last {
node.Pending -= self.branches - i
node.Children = node.Children[:i+1]
node.Last = true
}
copy(node.Children[i][:], hash)
left := node.Pending
if depth+1 < len(results.Levels) {
delete(results.Levels[depth+1], task.Index/(pow/self.branches))
}
results.Lock.Unlock()
// If there's more work to be done, leave for others
// glog.V(logger.Info).Infof("left %v", left)
if left > 0 {
break
}
// We're the last ones in this batch, merge the children together
depth--
pow *= self.branches
}
pend.Done()
}
}

73
swarm/storage/types.go
View file

@ -5,6 +5,7 @@ import (
"crypto"
"fmt"
"hash"
"io"
"sync"
"github.com/ethereum/go-ethereum/common"
@ -13,10 +14,47 @@ import (
type Hasher func() hash.Hash
// Peer is the recorded as Source on the chunk
// should probably not be here? but network should wrap chunk object
type Peer interface{}
type Key []byte
func (x Key) Size() uint {
return uint(len(x))
}
func (x Key) isEqual(y Key) bool {
return bytes.Compare(x, y) == 0
}
func (h Key) bits(i, j uint) uint {
ii := i >> 3
jj := i & 7
if ii >= h.Size() {
return 0
}
if jj+j <= 8 {
return uint((h[ii] >> jj) & ((1 << j) - 1))
}
res := uint(h[ii] >> jj)
jj = 8 - jj
j -= jj
for j != 0 {
ii++
if j < 8 {
res += uint(h[ii]&((1<<j)-1)) << jj
return res
}
res += uint(h[ii]) << jj
jj += 8
j -= 8
}
return res
}
func IsZeroKey(key Key) bool {
return len(key) == 0 || bytes.Equal(key, ZeroKey)
}
@ -127,7 +165,7 @@ After getting notified that all the data has been split (the error channel is cl
When calling Join with a root key, the caller gets returned a seekable lazy reader. The caller again provides a channel on which the caller receives placeholder chunks with missing data. The DPA is supposed to forward this to the chunk stores and notify the chunker if the data has been delivered (i.e. retrieved from memory cache, disk-persisted db or cloud based swarm delivery). As the seekable reader is used, the chunker then puts these together the relevant parts on demand.
*/
type Chunker interface {
type Splitter interface {
/*
When splitting, data is given as a SectionReader, and the key is a hashSize long byte slice (Key), the root hash of the entire content will fill this once processing finishes.
New chunks to store are coming to caller via the chunk storage channel, which the caller provides.
@ -135,7 +173,10 @@ type Chunker interface {
The caller gets returned an error channel, if an error is encountered during splitting, it is fed to errC error channel.
A closed error signals process completion at which point the key can be considered final if there were no errors.
*/
Split(key Key, data SectionReader, chunkC chan *Chunk, wg *sync.WaitGroup) chan error
Split(io.Reader, int64, chan *Chunk, *sync.WaitGroup, *sync.WaitGroup) (Key, error)
}
type Joiner interface {
/*
Join reconstructs original content based on a root key.
When joining, the caller gets returned a Lazy SectionReader, which is
@ -148,8 +189,28 @@ type Chunker interface {
The chunks are not meant to be validated by the chunker when joining. This
is because it is left to the DPA to decide which sources are trusted.
*/
Join(key Key, chunkC chan *Chunk) SectionReader
// returns the key length
KeySize() int64
Join(key Key, chunkC chan *Chunk) LazySectionReader
}
type Chunker interface {
Joiner
Splitter
// returns the key length
// KeySize() int64
}
// Size, Seek, Read, ReadAt
type LazySectionReader interface {
Size(chan bool) (int64, error)
io.Seeker
io.Reader
io.ReaderAt
}
type LazyTestSectionReader struct {
*io.SectionReader
}
func (self *LazyTestSectionReader) Size(chan bool) (int64, error) {
return self.SectionReader.Size(), nil
}

30
swarm/test/connections/00.sh
View file

@ -1,33 +1,25 @@
#!/bin/bash
dir=`dirname $0`
source $dir/../../cmd/swarm/test.sh
swarm init 4
echo "expect each node to have 3 peers"
cmd="'net.peerCount'"
cmd="net.peerCount"
sleep 5
swarm attach 00 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
swarm attach 01 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
swarm attach 02 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
swarm attach 03 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
swarm execute 00 "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
swarm execute 01 "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
swarm execute 02 "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
swarm execute 03 "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
swarm stop all
echo "after static nodes is deleted, connections are recovered from kaddb in bzz-peers.json"
# echo rm -rf $DATA_ROOT/enodes\*
# echo rm -rf $DATA_ROOT/data/\*/static-nodes.json
rm -rf $DATA_ROOT/enodes*
rm -rf $DATA_ROOT/data/*/static-nodes.json
echo "connections are recovered from kaddb in bzz-peers.json"
swarm cluster 4
echo "expect each node to have 3 peers"
cmd="'net.peerCount'"
sleep 10
swarm attach 00 --exec "$cmd" |tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
swarm attach 01 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
swarm attach 02 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
swarm attach 03 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
sleep 5
swarm execute 00 "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
swarm execute 01 "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
swarm execute 02 "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
swarm execute 03 "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
swarm stop all

2
swarm/test/swap/00.sh
View file

@ -4,7 +4,7 @@ echo " two nodes that do not sync and do not have any funds"
echo " cannot retrieve content from each other"
dir=`dirname $0`
source $dir/../../cmd/swarm/test.sh
source $dir/../test.sh
FILE_00=/tmp/1K.0
randomfile 1 > $FILE_00

13
swarm/test/swap/01.sh
View file

@ -3,19 +3,20 @@ echo " two nodes that do not sync but have enough funds"
echo " can retrieve content from each other"
dir=`dirname $0`
source $dir/../../cmd/swarm/test.sh
source $dir/..s/test.sh
file=/tmp/test.file
mininginterval=120
key=/tmp/key
logargs="--verbosity=0 --vmodule='swarm/*=6'"
# logargs='--verbosity=6'
# logargs="--verbosity=0 --vmodule='swarm/*=6'"
logargs='--verbosity=6'
# swarm init 2 --mine --bzznosync --bzznoswap=false $logargsc
# echo "Mining some ether..."
# sleep $mininginterval
swarm init 2 --mine --bzznosync $logargs
swarm cluster 2 --mine --bzznosync --bzznoswap=false $logargsc
echo "Mining some ether..."
sleep $mininginterval
randomfile 10 > $file
swarm up 00 $file|tail -n1 > $key

6
swarm/test/syncing/00.sh
View file

@ -4,7 +4,7 @@ echo " two nodes that sync (no swap and do not have any funds)"
echo " can be in sync content with each other"
dir=`dirname $0`
source $dir/../../cmd/swarm/test.sh
source $dir/../test.sh
mkdir -p /tmp/swarm-test-files
FILE_00=/tmp/swarm-test-files/00
@ -28,7 +28,6 @@ swarm needs 00 $key $FILE_00
swarm needs 01 $key $FILE_00
swarm stop 01
# exit 1;
swarm up 00 $FILE_01|tail -n1 > $key
swarm needs 00 $key $FILE_01
@ -46,7 +45,8 @@ swarm needs 00 $key $FILE_03
swarm stop 00
swarm up 01 $FILE_04|tail -n1 > $key
swarm needs 01 $key $FILE_04
swarm start 00 #--bzznoswap
swarm start 00
sleep $wait
swarm needs 00 $key $FILE_04
swarm stop all

2
swarm/test/syncing/01.sh
View file

@ -4,7 +4,7 @@ echo " two nodes that do not have any funds"
echo " can still sync content with each other"
dir=`dirname $0`
source $dir/../../cmd/swarm/test.sh
source $dir/../test.sh
key=/tmp/key
long=/tmp/10M

28
swarm/test/syncing/02.sh
View file

@ -5,26 +5,26 @@ echo " two nodes that sync (no swap and do not have any funds)"
echo " can sync content with each other even with intermittent network connection"
dir=`dirname $0`
source $dir/../../cmd/swarm/test.sh
source $dir/../test.sh
long=/tmp/10M
key=/tmp/key
randomfile 10000 > $long
randomfile 100000 > $long
ls -l $long
swarm init 2
sleep $wait
swarm init 2 --vmodule='swarm/*=5'
swarm up 00 $long |tail -n1 > $key &
sleep $wait
swarm attach 01 -exec "'bzz.blockNetworkRead(true)'"
sleep $wait
swarm attach 01 -exec "'bzz.blockNetworkRead(false)'"
sleep $wait
swarm attach 01 -exec "'bzz.blockNetworkRead(true)'"
sleep $wait
sleep 1
swarm execute 01 'bzz.blockNetworkRead(true)'
sleep 3
swarm execute 01 'bzz.blockNetworkRead(false)'
# sleep $wait
# swarm attach 01 -exec "'bzz.blockNetworkRead(true)'"
# sleep $wait
swarm stop 01
swarm start 01
swarm needs 01 $key $long
# swarm start 01
# sleep $wait
# swarm needs 01 $key $long
# sleep 3
swarm stop all

20
swarm/test/test.sh Normal file
View file

@ -0,0 +1,20 @@
#!/bin/bash
TEST_DIR=`dirname $0`
TEST_NAME=`basename $0 .sh`
TEST_TYPE=`basename $TEST_DIR`
export IP_ADDR="[::]"
export SWARM_NETWORK_ID=322$TEST_NAME
export SWARM_DIR=~/bzz/test/$TEST_TYPE
rm -rf $SWARM_DIR/$SWARM_NETWORK_ID
wait=1
function randomfile {
dd if=/dev/urandom of=/dev/stdout bs=1024 count=$1 2>/dev/null
}