forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-19 02:12:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

swarm: don't use me, this, self as receiver names

Browse source
This commit is contained in:
Delweng Zheng 2018-06-03 16:01:04 +08:00
parent 0644ebeb7a
commit 0ad4915ff7
35 changed files with 1212 additions and 1212 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

66
swarm/api/api.go
View file

@ -147,8 +147,8 @@ type Api struct {
}
//the api constructor initialises
func NewApi(dpa *storage.DPA, dns Resolver) (self *Api) {
self = &Api{
func NewApi(dpa *storage.DPA, dns Resolver) (api *Api) {
api = &Api{
dpa: dpa,
dns: dns,
}
@ -156,25 +156,25 @@ func NewApi(dpa *storage.DPA, dns Resolver) (self *Api) {
}
// to be used only in TEST
func (self *Api) Upload(uploadDir, index string) (hash string, err error) {
fs := NewFileSystem(self)
func (api *Api) Upload(uploadDir, index string) (hash string, err error) {
fs := NewFileSystem(api)
hash, err = fs.Upload(uploadDir, index)
return hash, err
}
// DPA reader API
func (self *Api) Retrieve(key storage.Key) storage.LazySectionReader {
return self.dpa.Retrieve(key)
func (api *Api) Retrieve(key storage.Key) storage.LazySectionReader {
return api.dpa.Retrieve(key)
}
func (self *Api) Store(data io.Reader, size int64, wg *sync.WaitGroup) (key storage.Key, err error) {
return self.dpa.Store(data, size, wg, nil)
func (api *Api) Store(data io.Reader, size int64, wg *sync.WaitGroup) (key storage.Key, err error) {
return api.dpa.Store(data, size, wg, nil)
}
type ErrResolve error
// DNS Resolver
func (self *Api) Resolve(uri *URI) (storage.Key, error) {
func (api *Api) Resolve(uri *URI) (storage.Key, error) {
apiResolveCount.Inc(1)
log.Trace(fmt.Sprintf("Resolving : %v", uri.Addr))
@ -188,7 +188,7 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) {
}
// if DNS is not configured, check if the address is a hash
if self.dns == nil {
if api.dns == nil {
if !isHash {
apiResolveFail.Inc(1)
return nil, fmt.Errorf("no DNS to resolve name: %q", uri.Addr)
@ -197,7 +197,7 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) {
}
// try and resolve the address
resolved, err := self.dns.Resolve(uri.Addr)
resolved, err := api.dns.Resolve(uri.Addr)
if err == nil {
return resolved[:], nil
} else if !isHash {
@ -208,18 +208,18 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) {
}
// Put provides singleton manifest creation on top of dpa store
func (self *Api) Put(content, contentType string) (storage.Key, error) {
func (api *Api) Put(content, contentType string) (storage.Key, error) {
apiPutCount.Inc(1)
r := strings.NewReader(content)
wg := &sync.WaitGroup{}
key, err := self.dpa.Store(r, int64(len(content)), wg, nil)
key, err := api.dpa.Store(r, int64(len(content)), wg, nil)
if err != nil {
apiPutFail.Inc(1)
return nil, err
}
manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType)
r = strings.NewReader(manifest)
key, err = self.dpa.Store(r, int64(len(manifest)), wg, nil)
key, err = api.dpa.Store(r, int64(len(manifest)), wg, nil)
if err != nil {
apiPutFail.Inc(1)
return nil, err
@ -231,9 +231,9 @@ func (self *Api) Put(content, contentType string) (storage.Key, error) {
// Get uses iterative manifest retrieval and prefix matching
// to resolve basePath to content using dpa retrieve
// it returns a section reader, mimeType, status and an error
func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionReader, mimeType string, status int, err error) {
func (api *Api) Get(key storage.Key, path string) (reader storage.LazySectionReader, mimeType string, status int, err error) {
apiGetCount.Inc(1)
trie, err := loadManifest(self.dpa, key, nil)
trie, err := loadManifest(api.dpa, key, nil)
if err != nil {
apiGetNotFound.Inc(1)
status = http.StatusNotFound
@ -254,7 +254,7 @@ func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionRe
} else {
mimeType = entry.ContentType
log.Trace(fmt.Sprintf("content lookup key: '%v' (%v)", key, mimeType))
reader = self.dpa.Retrieve(key)
reader = api.dpa.Retrieve(key)
}
} else {
status = http.StatusNotFound
@ -265,10 +265,10 @@ func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionRe
return
}
func (self *Api) Modify(key storage.Key, path, contentHash, contentType string) (storage.Key, error) {
func (api *Api) Modify(key storage.Key, path, contentHash, contentType string) (storage.Key, error) {
apiModifyCount.Inc(1)
quitC := make(chan bool)
trie, err := loadManifest(self.dpa, key, quitC)
trie, err := loadManifest(api.dpa, key, quitC)
if err != nil {
apiModifyFail.Inc(1)
return nil, err
@ -291,7 +291,7 @@ func (self *Api) Modify(key storage.Key, path, contentHash, contentType string)
return trie.hash, nil
}
func (self *Api) AddFile(mhash, path, fname string, content []byte, nameresolver bool) (storage.Key, string, error) {
func (api *Api) AddFile(mhash, path, fname string, content []byte, nameresolver bool) (storage.Key, string, error) {
apiAddFileCount.Inc(1)
uri, err := Parse("bzz:/" + mhash)
@ -299,7 +299,7 @@ func (self *Api) AddFile(mhash, path, fname string, content []byte, nameresolver
apiAddFileFail.Inc(1)
return nil, "", err
}
mkey, err := self.Resolve(uri)
mkey, err := api.Resolve(uri)
if err != nil {
apiAddFileFail.Inc(1)
return nil, "", err
@ -318,7 +318,7 @@ func (self *Api) AddFile(mhash, path, fname string, content []byte, nameresolver
ModTime: time.Now(),
}
mw, err := self.NewManifestWriter(mkey, nil)
mw, err := api.NewManifestWriter(mkey, nil)
if err != nil {
apiAddFileFail.Inc(1)
return nil, "", err
@ -341,7 +341,7 @@ func (self *Api) AddFile(mhash, path, fname string, content []byte, nameresolver
}
func (self *Api) RemoveFile(mhash, path, fname string, nameresolver bool) (string, error) {
func (api *Api) RemoveFile(mhash, path, fname string, nameresolver bool) (string, error) {
apiRmFileCount.Inc(1)
uri, err := Parse("bzz:/" + mhash)
@ -349,7 +349,7 @@ func (self *Api) RemoveFile(mhash, path, fname string, nameresolver bool) (strin
apiRmFileFail.Inc(1)
return "", err
}
mkey, err := self.Resolve(uri)
mkey, err := api.Resolve(uri)
if err != nil {
apiRmFileFail.Inc(1)
return "", err
@ -360,7 +360,7 @@ func (self *Api) RemoveFile(mhash, path, fname string, nameresolver bool) (strin
path = path[1:]
}
mw, err := self.NewManifestWriter(mkey, nil)
mw, err := api.NewManifestWriter(mkey, nil)
if err != nil {
apiRmFileFail.Inc(1)
return "", err
@ -382,7 +382,7 @@ func (self *Api) RemoveFile(mhash, path, fname string, nameresolver bool) (strin
return newMkey.String(), nil
}
func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, content []byte, oldKey storage.Key, offset int64, addSize int64, nameresolver bool) (storage.Key, string, error) {
func (api *Api) AppendFile(mhash, path, fname string, existingSize int64, content []byte, oldKey storage.Key, offset int64, addSize int64, nameresolver bool) (storage.Key, string, error) {
apiAppendFileCount.Inc(1)
buffSize := offset + addSize
@ -392,7 +392,7 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte
buf := make([]byte, buffSize)
oldReader := self.Retrieve(oldKey)
oldReader := api.Retrieve(oldKey)
io.ReadAtLeast(oldReader, buf, int(offset))
newReader := bytes.NewReader(content)
@ -406,7 +406,7 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte
totalSize := int64(len(buf))
// TODO(jmozah): to append using pyramid chunker when it is ready
//oldReader := self.Retrieve(oldKey)
//oldReader := api.Retrieve(oldKey)
//newReader := bytes.NewReader(content)
//combinedReader := io.MultiReader(oldReader, newReader)
@ -415,7 +415,7 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte
apiAppendFileFail.Inc(1)
return nil, "", err
}
mkey, err := self.Resolve(uri)
mkey, err := api.Resolve(uri)
if err != nil {
apiAppendFileFail.Inc(1)
return nil, "", err
@ -426,7 +426,7 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte
path = path[1:]
}
mw, err := self.NewManifestWriter(mkey, nil)
mw, err := api.NewManifestWriter(mkey, nil)
if err != nil {
apiAppendFileFail.Inc(1)
return nil, "", err
@ -463,19 +463,19 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte
}
func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storage.Key, manifestEntryMap map[string]*manifestTrieEntry, err error) {
func (api *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storage.Key, manifestEntryMap map[string]*manifestTrieEntry, err error) {
uri, err := Parse("bzz:/" + mhash)
if err != nil {
return nil, nil, err
}
key, err = self.Resolve(uri)
key, err = api.Resolve(uri)
if err != nil {
return nil, nil, err
}
quitC := make(chan bool)
rootTrie, err := loadManifest(self.dpa, key, quitC)
rootTrie, err := loadManifest(api.dpa, key, quitC)
if err != nil {
return nil, nil, fmt.Errorf("can't load manifest %v: %v", key.String(), err)
}

22
swarm/api/config.go
View file

@ -64,9 +64,9 @@ type Config struct {
}
//create a default config with all parameters to set to defaults
func NewDefaultConfig() (self *Config) {
func NewDefaultConfig() (cfg *Config) {
self = &Config{
cfg = &Config{
StoreParams: storage.NewDefaultStoreParams(),
ChunkerParams: storage.NewChunkerParams(),
HiveParams: network.NewDefaultHiveParams(),
@ -89,11 +89,11 @@ func NewDefaultConfig() (self *Config) {
//some config params need to be initialized after the complete
//config building phase is completed (e.g. due to overriding flags)
func (self *Config) Init(prvKey *ecdsa.PrivateKey) {
func (cfg *Config) Init(prvKey *ecdsa.PrivateKey) {
address := crypto.PubkeyToAddress(prvKey.PublicKey)
self.Path = filepath.Join(self.Path, "bzz-"+common.Bytes2Hex(address.Bytes()))
err := os.MkdirAll(self.Path, os.ModePerm)
cfg.Path = filepath.Join(cfg.Path, "bzz-"+common.Bytes2Hex(address.Bytes()))
err := os.MkdirAll(cfg.Path, os.ModePerm)
if err != nil {
log.Error(fmt.Sprintf("Error creating root swarm data directory: %v", err))
return
@ -103,11 +103,11 @@ func (self *Config) Init(prvKey *ecdsa.PrivateKey) {
pubkeyhex := common.ToHex(pubkey)
keyhex := crypto.Keccak256Hash(pubkey).Hex()
self.PublicKey = pubkeyhex
self.BzzKey = keyhex
cfg.PublicKey = pubkeyhex
cfg.BzzKey = keyhex
self.Swap.Init(self.Contract, prvKey)
self.SyncParams.Init(self.Path)
self.HiveParams.Init(self.Path)
self.StoreParams.Init(self.Path)
cfg.Swap.Init(cfg.Contract, prvKey)
cfg.SyncParams.Init(cfg.Path)
cfg.HiveParams.Init(cfg.Path)
cfg.StoreParams.Init(cfg.Path)
}

14
swarm/api/filesystem.go
View file

@ -46,7 +46,7 @@ func NewFileSystem(api *Api) *FileSystem {
// TODO: localpath should point to a manifest
//
// DEPRECATED: Use the HTTP API instead
func (self *FileSystem) Upload(lpath, index string) (string, error) {
func (fs *FileSystem) Upload(lpath, index string) (string, error) {
var list []*manifestTrieEntry
localpath, err := filepath.Abs(filepath.Clean(lpath))
if err != nil {
@ -113,7 +113,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
stat, _ := f.Stat()
var hash storage.Key
wg := &sync.WaitGroup{}
hash, err = self.api.dpa.Store(f, stat.Size(), wg, nil)
hash, err = fs.api.dpa.Store(f, stat.Size(), wg, nil)
if hash != nil {
list[i].Hash = hash.String()
}
@ -142,7 +142,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
}
trie := &manifestTrie{
dpa: self.api.dpa,
dpa: fs.api.dpa,
}
quitC := make(chan bool)
for i, entry := range list {
@ -173,7 +173,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
// under localpath
//
// DEPRECATED: Use the HTTP API instead
func (self *FileSystem) Download(bzzpath, localpath string) error {
func (fs *FileSystem) Download(bzzpath, localpath string) error {
lpath, err := filepath.Abs(filepath.Clean(localpath))
if err != nil {
return err
@ -188,7 +188,7 @@ func (self *FileSystem) Download(bzzpath, localpath string) error {
if err != nil {
return err
}
key, err := self.api.Resolve(uri)
key, err := fs.api.Resolve(uri)
if err != nil {
return err
}
@ -199,7 +199,7 @@ func (self *FileSystem) Download(bzzpath, localpath string) error {
}
quitC := make(chan bool)
trie, err := loadManifest(self.api.dpa, key, quitC)
trie, err := loadManifest(fs.api.dpa, key, quitC)
if err != nil {
log.Warn(fmt.Sprintf("fs.Download: loadManifestTrie error: %v", err))
return err
@ -244,7 +244,7 @@ func (self *FileSystem) Download(bzzpath, localpath string) error {
}
go func(i int, entry *downloadListEntry) {
defer wg.Done()
err := retrieveToFile(quitC, self.api.dpa, entry.key, entry.path)
err := retrieveToFile(quitC, fs.api.dpa, entry.key, entry.path)
if err != nil {
select {
case errC <- err:

6
swarm/api/http/roundtripper.go
View file

@ -51,12 +51,12 @@ type RoundTripper struct {
Port string
}
func (self *RoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) {
host := self.Host
func (tripper *RoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) {
host := tripper.Host
if len(host) == 0 {
host = "localhost"
}
url := fmt.Sprintf("http://%s:%s/%s:/%s/%s", host, self.Port, req.Proto, req.URL.Host, req.URL.Path)
url := fmt.Sprintf("http://%s:%s/%s:/%s/%s", host, tripper.Port, req.Proto, req.URL.Host, req.URL.Path)
log.Info(fmt.Sprintf("roundtripper: proxying request '%s' to '%s'", req.RequestURI, url))
reqProxy, err := http.NewRequest(req.Method, url, req.Body)
if err != nil {

74
swarm/api/manifest.go
View file

@ -230,18 +230,18 @@ func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dp
return
}
func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
self.hash = nil // trie modified, hash needs to be re-calculated on demand
func (trie *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
trie.hash = nil // trie modified, hash needs to be re-calculated on demand
if len(entry.Path) == 0 {
self.entries[256] = entry
trie.entries[256] = entry
return
}
b := entry.Path[0]
oldentry := self.entries[b]
oldentry := trie.entries[b]
if (oldentry == nil) || (oldentry.Path == entry.Path && oldentry.ContentType != ManifestType) {
self.entries[b] = entry
trie.entries[b] = entry
return
}
@ -251,7 +251,7 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
}
if (oldentry.ContentType == ManifestType) && (cpl == len(oldentry.Path)) {
if self.loadSubTrie(oldentry, quitC) != nil {
if trie.loadSubTrie(oldentry, quitC) != nil {
return
}
entry.Path = entry.Path[cpl:]
@ -263,21 +263,21 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
commonPrefix := entry.Path[:cpl]
subtrie := &manifestTrie{
dpa: self.dpa,
dpa: trie.dpa,
}
entry.Path = entry.Path[cpl:]
oldentry.Path = oldentry.Path[cpl:]
subtrie.addEntry(entry, quitC)
subtrie.addEntry(oldentry, quitC)
self.entries[b] = newManifestTrieEntry(&ManifestEntry{
trie.entries[b] = newManifestTrieEntry(&ManifestEntry{
Path: commonPrefix,
ContentType: ManifestType,
}, subtrie)
}
func (self *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) {
for _, e := range self.entries {
func (trie *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) {
for _, e := range trie.entries {
if e != nil {
cnt++
entry = e
@ -286,27 +286,27 @@ func (self *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) {
return
}
func (self *manifestTrie) deleteEntry(path string, quitC chan bool) {
self.hash = nil // trie modified, hash needs to be re-calculated on demand
func (trie *manifestTrie) deleteEntry(path string, quitC chan bool) {
trie.hash = nil // trie modified, hash needs to be re-calculated on demand
if len(path) == 0 {
self.entries[256] = nil
trie.entries[256] = nil
return
}
b := path[0]
entry := self.entries[b]
entry := trie.entries[b]
if entry == nil {
return
}
if entry.Path == path {
self.entries[b] = nil
trie.entries[b] = nil
return
}
epl := len(entry.Path)
if (entry.ContentType == ManifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) {
if self.loadSubTrie(entry, quitC) != nil {
if trie.loadSubTrie(entry, quitC) != nil {
return
}
entry.subtrie.deleteEntry(path[epl:], quitC)
@ -317,13 +317,13 @@ func (self *manifestTrie) deleteEntry(path string, quitC chan bool) {
if lastentry != nil {
lastentry.Path = entry.Path + lastentry.Path
}
self.entries[b] = lastentry
trie.entries[b] = lastentry
}
}
}
func (self *manifestTrie) recalcAndStore() error {
if self.hash != nil {
func (trie *manifestTrie) recalcAndStore() error {
if trie.hash != nil {
return nil
}
@ -331,7 +331,7 @@ func (self *manifestTrie) recalcAndStore() error {
buffer.WriteString(`{"entries":[`)
list := &Manifest{}
for _, entry := range self.entries {
for _, entry := range trie.entries {
if entry != nil {
if entry.Hash == "" { // TODO: paralellize
err := entry.subtrie.recalcAndStore()
@ -352,22 +352,22 @@ func (self *manifestTrie) recalcAndStore() error {
sr := bytes.NewReader(manifest)
wg := &sync.WaitGroup{}
key, err2 := self.dpa.Store(sr, int64(len(manifest)), wg, nil)
key, err2 := trie.dpa.Store(sr, int64(len(manifest)), wg, nil)
wg.Wait()
self.hash = key
trie.hash = key
return err2
}
func (self *manifestTrie) loadSubTrie(entry *manifestTrieEntry, quitC chan bool) (err error) {
func (trie *manifestTrie) loadSubTrie(entry *manifestTrieEntry, quitC chan bool) (err error) {
if entry.subtrie == nil {
hash := common.Hex2Bytes(entry.Hash)
entry.subtrie, err = loadManifest(self.dpa, hash, quitC)
entry.subtrie, err = loadManifest(trie.dpa, hash, quitC)
entry.Hash = "" // might not match, should be recalculated
}
return
}
func (self *manifestTrie) listWithPrefixInt(prefix, rp string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) error {
func (trie *manifestTrie) listWithPrefixInt(prefix, rp string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) error {
plen := len(prefix)
var start, stop int
if plen == 0 {
@ -384,7 +384,7 @@ func (self *manifestTrie) listWithPrefixInt(prefix, rp string, quitC chan bool,
return fmt.Errorf("aborted")
default:
}
entry := self.entries[i]
entry := trie.entries[i]
if entry != nil {
epl := len(entry.Path)
if entry.ContentType == ManifestType {
@ -393,7 +393,7 @@ func (self *manifestTrie) listWithPrefixInt(prefix, rp string, quitC chan bool,
l = epl
}
if prefix[:l] == entry.Path[:l] {
err := self.loadSubTrie(entry, quitC)
err := trie.loadSubTrie(entry, quitC)
if err != nil {
return err
}
@ -412,23 +412,23 @@ func (self *manifestTrie) listWithPrefixInt(prefix, rp string, quitC chan bool,
return nil
}
func (self *manifestTrie) listWithPrefix(prefix string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) (err error) {
return self.listWithPrefixInt(prefix, "", quitC, cb)
func (trie *manifestTrie) listWithPrefix(prefix string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) (err error) {
return trie.listWithPrefixInt(prefix, "", quitC, cb)
}
func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *manifestTrieEntry, pos int) {
func (trie *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *manifestTrieEntry, pos int) {
log.Trace(fmt.Sprintf("findPrefixOf(%s)", path))
if len(path) == 0 {
return self.entries[256], 0
return trie.entries[256], 0
}
//see if first char is in manifest entries
b := path[0]
entry = self.entries[b]
entry = trie.entries[b]
if entry == nil {
return self.entries[256], 0
return trie.entries[256], 0
}
epl := len(entry.Path)
@ -436,7 +436,7 @@ func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *man
if len(path) <= epl {
if entry.Path[:len(path)] == path {
if entry.ContentType == ManifestType {
err := self.loadSubTrie(entry, quitC)
err := trie.loadSubTrie(entry, quitC)
if err == nil && entry.subtrie != nil {
subentries := entry.subtrie.entries
for i := 0; i < len(subentries); i++ {
@ -457,7 +457,7 @@ func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *man
log.Trace(fmt.Sprintf("entry.ContentType = %v", entry.ContentType))
//the subentry is a manifest, load subtrie
if entry.ContentType == ManifestType && (strings.Contains(entry.Path, path) || strings.Contains(path, entry.Path)) {
err := self.loadSubTrie(entry, quitC)
err := trie.loadSubTrie(entry, quitC)
if err != nil {
return nil, 0
}
@ -495,10 +495,10 @@ func RegularSlashes(path string) (res string) {
return
}
func (self *manifestTrie) getEntry(spath string) (entry *manifestTrieEntry, fullpath string) {
func (trie *manifestTrie) getEntry(spath string) (entry *manifestTrieEntry, fullpath string) {
path := RegularSlashes(spath)
var pos int
quitC := make(chan bool)
entry, pos = self.findPrefixOf(path, quitC)
entry, pos = trie.findPrefixOf(path, quitC)
return entry, path[:pos]
}

16
swarm/api/storage.go
View file

@ -41,8 +41,8 @@ func NewStorage(api *Api) *Storage {
// its content type
//
// DEPRECATED: Use the HTTP API instead
func (self *Storage) Put(content, contentType string) (string, error) {
key, err := self.api.Put(content, contentType)
func (s *Storage) Put(content, contentType string) (string, error) {
key, err := s.api.Put(content, contentType)
if err != nil {
return "", err
}
@ -57,16 +57,16 @@ func (self *Storage) Put(content, contentType string) (string, error) {
// size is resp.Size
//
// DEPRECATED: Use the HTTP API instead
func (self *Storage) Get(bzzpath string) (*Response, error) {
func (s *Storage) Get(bzzpath string) (*Response, error) {
uri, err := Parse(path.Join("bzz:/", bzzpath))
if err != nil {
return nil, err
}
key, err := self.api.Resolve(uri)
key, err := s.api.Resolve(uri)
if err != nil {
return nil, err
}
reader, mimeType, status, err := self.api.Get(key, uri.Path)
reader, mimeType, status, err := s.api.Get(key, uri.Path)
if err != nil {
return nil, err
}
@ -87,16 +87,16 @@ func (self *Storage) Get(bzzpath string) (*Response, error) {
// and merge on to it. creating an entry w conentType (mime)
//
// DEPRECATED: Use the HTTP API instead
func (self *Storage) Modify(rootHash, path, contentHash, contentType string) (newRootHash string, err error) {
func (s *Storage) Modify(rootHash, path, contentHash, contentType string) (newRootHash string, err error) {
uri, err := Parse("bzz:/" + rootHash)
if err != nil {
return "", err
}
key, err := self.api.Resolve(uri)
key, err := s.api.Resolve(uri)
if err != nil {
return "", err
}
key, err = self.api.Modify(key, path, contentHash, contentType)
key, err = s.api.Modify(key, path, contentHash, contentType)
if err != nil {
return "", err
}

16
swarm/api/testapi.go
View file

@ -29,18 +29,18 @@ func NewControl(api *Api, hive *network.Hive) *Control {
return &Control{api, hive}
}
func (self *Control) BlockNetworkRead(on bool) {
self.hive.BlockNetworkRead(on)
func (c *Control) BlockNetworkRead(on bool) {
c.hive.BlockNetworkRead(on)
}
func (self *Control) SyncEnabled(on bool) {
self.hive.SyncEnabled(on)
func (c *Control) SyncEnabled(on bool) {
c.hive.SyncEnabled(on)
}
func (self *Control) SwapEnabled(on bool) {
self.hive.SwapEnabled(on)
func (c *Control) SwapEnabled(on bool) {
c.hive.SwapEnabled(on)
}
func (self *Control) Hive() string {
return self.hive.String()
func (c *Control) Hive() string {
return c.hive.String()
}

8
swarm/fuse/swarmfs_fallback.go
View file

@ -34,18 +34,18 @@ type MountInfo struct {
LatestManifest string
}
func (self *SwarmFS) Mount(mhash, mountpoint string) (*MountInfo, error) {
func (fs *SwarmFS) Mount(mhash, mountpoint string) (*MountInfo, error) {
return nil, errNoFUSE
}
func (self *SwarmFS) Unmount(mountpoint string) (bool, error) {
func (fs *SwarmFS) Unmount(mountpoint string) (bool, error) {
return false, errNoFUSE
}
func (self *SwarmFS) Listmounts() ([]*MountInfo, error) {
func (fs *SwarmFS) Listmounts() ([]*MountInfo, error) {
return nil, errNoFUSE
}
func (self *SwarmFS) Stop() error {
func (fs *SwarmFS) Stop() error {
return nil
}

44
swarm/fuse/swarmfs_unix.go
View file

@ -72,7 +72,7 @@ func NewMountInfo(mhash, mpoint string, sapi *api.Api) *MountInfo {
return newMountInfo
}
func (self *SwarmFS) Mount(mhash, mountpoint string) (*MountInfo, error) {
func (s *SwarmFS) Mount(mhash, mountpoint string) (*MountInfo, error) {
if mountpoint == "" {
return nil, errEmptyMountPoint
@ -82,25 +82,25 @@ func (self *SwarmFS) Mount(mhash, mountpoint string) (*MountInfo, error) {
return nil, err
}
self.swarmFsLock.Lock()
defer self.swarmFsLock.Unlock()
s.swarmFsLock.Lock()
defer s.swarmFsLock.Unlock()
noOfActiveMounts := len(self.activeMounts)
noOfActiveMounts := len(s.activeMounts)
if noOfActiveMounts >= maxFuseMounts {
return nil, errMaxMountCount
}
if _, ok := self.activeMounts[cleanedMountPoint]; ok {
if _, ok := s.activeMounts[cleanedMountPoint]; ok {
return nil, errAlreadyMounted
}
log.Info(fmt.Sprintf("Attempting to mount %s ", cleanedMountPoint))
_, manifestEntryMap, err := self.swarmApi.BuildDirectoryTree(mhash, true)
_, manifestEntryMap, err := s.swarmApi.BuildDirectoryTree(mhash, true)
if err != nil {
return nil, err
}
mi := NewMountInfo(mhash, cleanedMountPoint, self.swarmApi)
mi := NewMountInfo(mhash, cleanedMountPoint, s.swarmApi)
dirTree := map[string]*SwarmDir{}
rootDir := NewSwarmDir("/", mi)
@ -174,21 +174,21 @@ func (self *SwarmFS) Mount(mhash, mountpoint string) (*MountInfo, error) {
log.Info("Now serving swarm FUSE FS", "manifest", mhash, "mountpoint", cleanedMountPoint)
}
self.activeMounts[cleanedMountPoint] = mi
s.activeMounts[cleanedMountPoint] = mi
return mi, nil
}
func (self *SwarmFS) Unmount(mountpoint string) (*MountInfo, error) {
func (s *SwarmFS) Unmount(mountpoint string) (*MountInfo, error) {
self.swarmFsLock.Lock()
defer self.swarmFsLock.Unlock()
s.swarmFsLock.Lock()
defer s.swarmFsLock.Unlock()
cleanedMountPoint, err := filepath.Abs(filepath.Clean(mountpoint))
if err != nil {
return nil, err
}
mountInfo := self.activeMounts[cleanedMountPoint]
mountInfo := s.activeMounts[cleanedMountPoint]
if mountInfo == nil || mountInfo.MountPoint != cleanedMountPoint {
return nil, fmt.Errorf("%s is not mounted", cleanedMountPoint)
@ -204,7 +204,7 @@ func (self *SwarmFS) Unmount(mountpoint string) (*MountInfo, error) {
}
mountInfo.fuseConnection.Close()
delete(self.activeMounts, cleanedMountPoint)
delete(s.activeMounts, cleanedMountPoint)
succString := fmt.Sprintf("UnMounting %v succeeded", cleanedMountPoint)
log.Info(succString)
@ -212,21 +212,21 @@ func (self *SwarmFS) Unmount(mountpoint string) (*MountInfo, error) {
return mountInfo, nil
}
func (self *SwarmFS) Listmounts() []*MountInfo {
self.swarmFsLock.RLock()
defer self.swarmFsLock.RUnlock()
func (s *SwarmFS) Listmounts() []*MountInfo {
s.swarmFsLock.RLock()
defer s.swarmFsLock.RUnlock()
rows := make([]*MountInfo, 0, len(self.activeMounts))
for _, mi := range self.activeMounts {
rows := make([]*MountInfo, 0, len(s.activeMounts))
for _, mi := range s.activeMounts {
rows = append(rows, mi)
}
return rows
}
func (self *SwarmFS) Stop() bool {
for mp := range self.activeMounts {
mountInfo := self.activeMounts[mp]
self.Unmount(mountInfo.MountPoint)
func (s *SwarmFS) Stop() bool {
for mp := range s.activeMounts {
mountInfo := s.activeMounts[mp]
s.Unmount(mountInfo.MountPoint)
}
return true
}

32
swarm/network/depo.go
View file

@ -58,14 +58,14 @@ func NewDepo(hash storage.SwarmHasher, localStore, remoteStore storage.ChunkStor
// * back immediately as a deliveryRequest message
// * empty message just pings back for more (is this needed?)
// * strict signed sync states may be needed.
func (self *Depo) HandleUnsyncedKeysMsg(req *unsyncedKeysMsgData, p *peer) error {
func (d *Depo) HandleUnsyncedKeysMsg(req *unsyncedKeysMsgData, p *peer) error {
unsynced := req.Unsynced
var missing []*syncRequest
var chunk *storage.Chunk
var err error
for _, req := range unsynced {
// skip keys that are found,
chunk, err = self.localStore.Get(req.Key[:])
chunk, err = d.localStore.Get(req.Key[:])
if err != nil || chunk.SData == nil {
missing = append(missing, req)
}
@ -88,7 +88,7 @@ func (self *Depo) HandleUnsyncedKeysMsg(req *unsyncedKeysMsgData, p *peer) error
// (remote peer is free to reprioritize)
// * the message implies remote peer wants more, so trigger for
// * new outgoing unsynced keys message is fired
func (self *Depo) HandleDeliveryRequestMsg(req *deliveryRequestMsgData, p *peer) error {
func (d *Depo) HandleDeliveryRequestMsg(req *deliveryRequestMsgData, p *peer) error {
deliver := req.Deliver
// queue the actual delivery of a chunk ()
log.Trace(fmt.Sprintf("Depo.HandleDeliveryRequestMsg: received %v delivery requests: %v", len(deliver), deliver))
@ -96,7 +96,7 @@ func (self *Depo) HandleDeliveryRequestMsg(req *deliveryRequestMsgData, p *peer)
// TODO: look up in cache here or in deliveries
// priorities are taken from the message so the remote party can
// reprioritise to at their leisure
// r = self.pullCached(sreq.Key) // pulls and deletes from cache
// r = d.pullCached(sreq.Key) // pulls and deletes from cache
Push(p, sreq.Key, sreq.Priority)
}
@ -108,10 +108,10 @@ func (self *Depo) HandleDeliveryRequestMsg(req *deliveryRequestMsgData, p *peer)
// the entrypoint for store requests coming from the bzz wire protocol
// if key found locally, return. otherwise
// remote is untrusted, so hash is verified and chunk passed on to NetStore
func (self *Depo) HandleStoreRequestMsg(req *storeRequestMsgData, p *peer) {
func (d *Depo) HandleStoreRequestMsg(req *storeRequestMsgData, p *peer) {
var islocal bool
req.from = p
chunk, err := self.localStore.Get(req.Key)
chunk, err := d.localStore.Get(req.Key)
switch {
case err != nil:
log.Trace(fmt.Sprintf("Depo.handleStoreRequest: %v not found locally. create new chunk/request", req.Key))
@ -133,7 +133,7 @@ func (self *Depo) HandleStoreRequestMsg(req *storeRequestMsgData, p *peer) {
//return
}
hasher := self.hashfunc()
hasher := d.hashfunc()
hasher.Write(req.SData)
if !bytes.Equal(hasher.Sum(nil), req.Key) {
// data does not validate, ignore
@ -150,12 +150,12 @@ func (self *Depo) HandleStoreRequestMsg(req *storeRequestMsgData, p *peer) {
chunk.Size = int64(binary.LittleEndian.Uint64(req.SData[0:8]))
log.Trace(fmt.Sprintf("delivery of %v from %v", chunk, p))
chunk.Source = p
self.netStore.Put(chunk)
d.netStore.Put(chunk)
}
// entrypoint for retrieve requests coming from the bzz wire protocol
// checks swap balance - return if peer has no credit
func (self *Depo) HandleRetrieveRequestMsg(req *retrieveRequestMsgData, p *peer) {
func (d *Depo) HandleRetrieveRequestMsg(req *retrieveRequestMsgData, p *peer) {
req.from = p
// swap - record credit for 1 request
// note that only charge actual reqsearches
@ -171,8 +171,8 @@ func (self *Depo) HandleRetrieveRequestMsg(req *retrieveRequestMsgData, p *peer)
// call storage.NetStore#Get which
// blocks until local retrieval finished
// launches cloud retrieval
chunk, _ := self.netStore.Get(req.Key)
req = self.strategyUpdateRequest(chunk.Req, req)
chunk, _ := d.netStore.Get(req.Key)
req = d.strategyUpdateRequest(chunk.Req, req)
// check if we can immediately deliver
if chunk.SData != nil {
log.Trace(fmt.Sprintf("Depo.HandleRetrieveRequest: %v - content found, delivering...", req.Key.Log()))
@ -197,20 +197,20 @@ func (self *Depo) HandleRetrieveRequestMsg(req *retrieveRequestMsgData, p *peer)
}
// add peer request the chunk and decides the timeout for the response if still searching
func (self *Depo) strategyUpdateRequest(rs *storage.RequestStatus, origReq *retrieveRequestMsgData) (req *retrieveRequestMsgData) {
func (d *Depo) strategyUpdateRequest(rs *storage.RequestStatus, origReq *retrieveRequestMsgData) (req *retrieveRequestMsgData) {
log.Trace(fmt.Sprintf("Depo.strategyUpdateRequest: key %v", origReq.Key.Log()))
// we do not create an alternative one
req = origReq
if rs != nil {
self.addRequester(rs, req)
req.setTimeout(self.searchTimeout(rs, req))
d.addRequester(rs, req)
req.setTimeout(d.searchTimeout(rs, req))
}
return
}
// decides the timeout promise sent with the immediate peers response to a retrieve request
// if timeout is explicitly set and expired
func (self *Depo) searchTimeout(rs *storage.RequestStatus, req *retrieveRequestMsgData) (timeout *time.Time) {
func (d *Depo) searchTimeout(rs *storage.RequestStatus, req *retrieveRequestMsgData) (timeout *time.Time) {
reqt := req.getTimeout()
t := time.Now().Add(searchTimeout)
if reqt != nil && reqt.Before(t) {
@ -225,7 +225,7 @@ adds a new peer to an existing open request
only add if less than requesterCount peers forwarded the same request id so far
note this is done irrespective of status (searching or found)
*/
func (self *Depo) addRequester(rs *storage.RequestStatus, req *retrieveRequestMsgData) {
func (d *Depo) addRequester(rs *storage.RequestStatus, req *retrieveRequestMsgData) {
log.Trace(fmt.Sprintf("Depo.addRequester: key %v - add peer to req.Id %v", req.Key.Log(), req.Id))
list := rs.Requesters[req.Id]
rs.Requesters[req.Id] = append(list, req)

10
swarm/network/forwarding.go
View file

@ -54,8 +54,8 @@ var searchTimeout = 3 * time.Second
// forwarding logic
// logic propagating retrieve requests to peers given by the kademlia hive
func (self *forwarder) Retrieve(chunk *storage.Chunk) {
peers := self.hive.getPeers(chunk.Key, 0)
func (f *forwarder) Retrieve(chunk *storage.Chunk) {
peers := f.hive.getPeers(chunk.Key, 0)
log.Trace(fmt.Sprintf("forwarder.Retrieve: %v - received %d peers from KΛÐΞMLIΛ...", chunk.Key.Log(), len(peers)))
OUT:
for _, p := range peers {
@ -87,7 +87,7 @@ OUT:
// requests to specific peers given by the kademlia hive
// except for peers that the store request came from (if any)
// delivery queueing taken care of by syncer
func (self *forwarder) Store(chunk *storage.Chunk) {
func (f *forwarder) Store(chunk *storage.Chunk) {
var n int
msg := &storeRequestMsgData{
Key: chunk.Key,
@ -97,7 +97,7 @@ func (self *forwarder) Store(chunk *storage.Chunk) {
if chunk.Source != nil {
source = chunk.Source.(*peer)
}
for _, p := range self.hive.getPeers(chunk.Key, 0) {
for _, p := range f.hive.getPeers(chunk.Key, 0) {
log.Trace(fmt.Sprintf("forwarder.Store: %v %v", p, chunk))
if p.syncer != nil && (source == nil || p.Addr() != source.Addr()) {
@ -109,7 +109,7 @@ func (self *forwarder) Store(chunk *storage.Chunk) {
}
// once a chunk is found deliver it to its requesters unless timed out
func (self *forwarder) Deliver(chunk *storage.Chunk) {
func (f *forwarder) Deliver(chunk *storage.Chunk) {
// iterate over request entries
for id, requesters := range chunk.Req.Requesters {
counter := requesterCount

134
swarm/network/hive.go
View file

@ -92,8 +92,8 @@ func NewDefaultHiveParams() *HiveParams {
//this can only finally be set after all config options (file, cmd line, env vars)
//have been evaluated
func (self *HiveParams) Init(path string) {
self.KadDbPath = filepath.Join(path, "bzz-peers.json")
func (hp *HiveParams) Init(path string) {
hp.KadDbPath = filepath.Join(path, "bzz-peers.json")
}
func NewHive(addr common.Hash, params *HiveParams, swapEnabled, syncEnabled bool) *Hive {
@ -108,53 +108,53 @@ func NewHive(addr common.Hash, params *HiveParams, swapEnabled, syncEnabled bool
}
}
func (self *Hive) SyncEnabled(on bool) {
self.syncEnabled = on
func (hive *Hive) SyncEnabled(on bool) {
hive.syncEnabled = on
}
func (self *Hive) SwapEnabled(on bool) {
self.swapEnabled = on
func (hive *Hive) SwapEnabled(on bool) {
hive.swapEnabled = on
}
func (self *Hive) BlockNetworkRead(on bool) {
self.blockRead = on
func (hive *Hive) BlockNetworkRead(on bool) {
hive.blockRead = on
}
func (self *Hive) BlockNetworkWrite(on bool) {
self.blockWrite = on
func (hive *Hive) BlockNetworkWrite(on bool) {
hive.blockWrite = on
}
// public accessor to the hive base address
func (self *Hive) Addr() kademlia.Address {
return self.addr
func (hive *Hive) Addr() kademlia.Address {
return hive.addr
}
// Start receives network info only at startup
// listedAddr is a function to retrieve listening address to advertise to peers
// connectPeer is a function to connect to a peer based on its NodeID or enode URL
// there are called on the p2p.Server which runs on the node
func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPeer func(string) error) (err error) {
self.toggle = make(chan bool)
self.more = make(chan bool)
self.quit = make(chan bool)
self.id = id
self.listenAddr = listenAddr
err = self.kad.Load(self.path, nil)
func (hive *Hive) Start(id discover.NodeID, listenAddr func() string, connectPeer func(string) error) (err error) {
hive.toggle = make(chan bool)
hive.more = make(chan bool)
hive.quit = make(chan bool)
hive.id = id
hive.listenAddr = listenAddr
err = hive.kad.Load(hive.path, nil)
if err != nil {
log.Warn(fmt.Sprintf("Warning: error reading kaddb '%s' (skipping): %v", self.path, err))
log.Warn(fmt.Sprintf("Warning: error reading kaddb '%s' (skipping): %v", hive.path, err))
err = nil
}
// this loop is doing bootstrapping and maintains a healthy table
go self.keepAlive()
go hive.keepAlive()
go func() {
// whenever toggled ask kademlia about most preferred peer
for alive := range self.more {
for alive := range hive.more {
if !alive {
// receiving false closes the loop while allowing parallel routines
// to attempt to write to more (remove Peer when shutting down)
return
}
node, need, proxLimit := self.kad.Suggest()
node, need, proxLimit := hive.kad.Suggest()
if node != nil && len(node.Url) > 0 {
log.Trace(fmt.Sprintf("call known bee %v", node.Url))
@ -164,10 +164,10 @@ func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPee
}
if need {
// a random peer is taken from the table
peers := self.kad.FindClosest(kademlia.RandomAddressAt(self.addr, rand.Intn(self.kad.MaxProx)), 1)
peers := hive.kad.FindClosest(kademlia.RandomAddressAt(hive.addr, rand.Intn(hive.kad.MaxProx)), 1)
if len(peers) > 0 {
// a random address at prox bin 0 is sent for lookup
randAddr := kademlia.RandomAddressAt(self.addr, proxLimit)
randAddr := kademlia.RandomAddressAt(hive.addr, proxLimit)
req := &retrieveRequestMsgData{
Key: storage.Key(randAddr[:]),
}
@ -181,11 +181,11 @@ func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPee
log.Info(fmt.Sprintf("no need for more bees"))
}
select {
case self.toggle <- need:
case <-self.quit:
case hive.toggle <- need:
case <-hive.quit:
return
}
log.Debug(fmt.Sprintf("queen's address: %v, population: %d (%d)", self.addr, self.kad.Count(), self.kad.DBCount()))
log.Debug(fmt.Sprintf("queen's address: %v, population: %d (%d)", hive.addr, hive.kad.Count(), hive.kad.DBCount()))
}
}()
return
@ -193,60 +193,60 @@ func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPee
// keepAlive is a forever loop
// in its awake state it periodically triggers connection attempts
// by writing to self.more until Kademlia Table is saturated
// wake state is toggled by writing to self.toggle
// by writing to hive.more until Kademlia Table is saturated
// wake state is toggled by writing to hive.toggle
// it restarts if the table becomes non-full again due to disconnections
func (self *Hive) keepAlive() {
alarm := time.NewTicker(time.Duration(self.callInterval)).C
func (hive *Hive) keepAlive() {
alarm := time.NewTicker(time.Duration(hive.callInterval)).C
for {
peersNumGauge.Update(int64(self.kad.Count()))
peersNumGauge.Update(int64(hive.kad.Count()))
select {
case <-alarm:
if self.kad.DBCount() > 0 {
if hive.kad.DBCount() > 0 {
select {
case self.more <- true:
case hive.more <- true:
log.Debug(fmt.Sprintf("buzz wakeup"))
default:
}
}
case need := <-self.toggle:
case need := <-hive.toggle:
if alarm == nil && need {
alarm = time.NewTicker(time.Duration(self.callInterval)).C
alarm = time.NewTicker(time.Duration(hive.callInterval)).C
}
if alarm != nil && !need {
alarm = nil
}
case <-self.quit:
case <-hive.quit:
return
}
}
}
func (self *Hive) Stop() error {
func (hive *Hive) Stop() error {
// closing toggle channel quits the updateloop
close(self.quit)
return self.kad.Save(self.path, saveSync)
close(hive.quit)
return hive.kad.Save(hive.path, saveSync)
}
// called at the end of a successful protocol handshake
func (self *Hive) addPeer(p *peer) error {
func (hive *Hive) addPeer(p *peer) error {
addPeerCounter.Inc(1)
defer func() {
select {
case self.more <- true:
case hive.more <- true:
default:
}
}()
log.Trace(fmt.Sprintf("hi new bee %v", p))
err := self.kad.On(p, loadSync)
err := hive.kad.On(p, loadSync)
if err != nil {
return err
}
// self lookup (can be encoded as nil/zero key since peers addr known) + no id ()
// hive lookup (can be encoded as nil/zero key since peers addr known) + no id ()
// the most common way of saying hi in bzz is initiation of gossip
// let me know about anyone new from my hood , here is the storageradius
// to send the 6 byte self lookup
// to send the 6 byte hive lookup
// we do not record as request or forward it, just reply with peers
p.retrieve(&retrieveRequestMsgData{})
log.Trace(fmt.Sprintf("'whatsup wheresdaparty' sent to %v", p))
@ -255,33 +255,33 @@ func (self *Hive) addPeer(p *peer) error {
}
// called after peer disconnected
func (self *Hive) removePeer(p *peer) {
func (hive *Hive) removePeer(p *peer) {
removePeerCounter.Inc(1)
log.Debug(fmt.Sprintf("bee %v removed", p))
self.kad.Off(p, saveSync)
hive.kad.Off(p, saveSync)
select {
case self.more <- true:
case hive.more <- true:
default:
}
if self.kad.Count() == 0 {
if hive.kad.Count() == 0 {
log.Debug(fmt.Sprintf("empty, all bees gone"))
}
}
// Retrieve a list of live peers that are closer to target than us
func (self *Hive) getPeers(target storage.Key, max int) (peers []*peer) {
func (hive *Hive) getPeers(target storage.Key, max int) (peers []*peer) {
var addr kademlia.Address
copy(addr[:], target[:])
for _, node := range self.kad.FindClosest(addr, max) {
for _, node := range hive.kad.FindClosest(addr, max) {
peers = append(peers, node.(*peer))
}
return
}
// disconnects all the peers
func (self *Hive) DropAll() {
func (hive *Hive) DropAll() {
log.Info(fmt.Sprintf("dropping all bees"))
for _, node := range self.kad.FindClosest(kademlia.Address{}, 0) {
for _, node := range hive.kad.FindClosest(kademlia.Address{}, 0) {
node.Drop()
}
}
@ -301,7 +301,7 @@ func newNodeRecord(addr *peerAddr) *kademlia.NodeRecord {
// called by the protocol when receiving peerset (for target address)
// peersMsgData is converted to a slice of NodeRecords for Kademlia
// this is to store all thats needed
func (self *Hive) HandlePeersMsg(req *peersMsgData, from *peer) {
func (hive *Hive) HandlePeersMsg(req *peersMsgData, from *peer) {
var nrs []*kademlia.NodeRecord
for _, p := range req.Peers {
if err := netutil.CheckRelayIP(from.remoteAddr.IP, p.IP); err != nil {
@ -310,7 +310,7 @@ func (self *Hive) HandlePeersMsg(req *peersMsgData, from *peer) {
}
nrs = append(nrs, newNodeRecord(p))
}
self.kad.Add(nrs)
hive.kad.Add(nrs)
}
// peer wraps the protocol instance to represent a connected peer
@ -320,17 +320,17 @@ type peer struct {
}
// protocol instance implements kademlia.Node interface (embedded peer)
func (self *peer) Addr() kademlia.Address {
return self.remoteAddr.Addr
func (p *peer) Addr() kademlia.Address {
return p.remoteAddr.Addr
}
func (self *peer) Url() string {
return self.remoteAddr.String()
func (p *peer) Url() string {
return p.remoteAddr.String()
}
// TODO take into account traffic
func (self *peer) LastActive() time.Time {
return self.lastActive
func (p *peer) LastActive() time.Time {
return p.lastActive
}
// reads the serialised form of sync state persisted as the 'Meta' attribute
@ -370,19 +370,19 @@ func saveSync(record *kademlia.NodeRecord, node kademlia.Node) {
// the immediate response to a retrieve request,
// sends relevant peer data given by the kademlia hive to the requester
// TODO: remember peers sent for duration of the session, only new peers sent
func (self *Hive) peers(req *retrieveRequestMsgData) {
func (hive *Hive) peers(req *retrieveRequestMsgData) {
if req != nil {
var addrs []*peerAddr
if req.timeout == nil || time.Now().Before(*(req.timeout)) {
key := req.Key
// self lookup from remote peer
// hive lookup from remote peer
if storage.IsZeroKey(key) {
addr := req.from.Addr()
key = storage.Key(addr[:])
req.Key = nil
}
// get peer addresses from hive
for _, peer := range self.getPeers(key, int(req.MaxPeers)) {
for _, peer := range hive.getPeers(key, int(req.MaxPeers)) {
addrs = append(addrs, peer.remoteAddr)
}
log.Debug(fmt.Sprintf("Hive sending %d peer addresses to %v. req.Id: %v, req.Key: %v", len(addrs), req.from, req.Id, req.Key.Log()))
@ -398,6 +398,6 @@ func (self *Hive) peers(req *retrieveRequestMsgData) {
}
}
func (self *Hive) String() string {
return self.kad.String()
func (hive *Hive) String() string {
return hive.kad.String()
}

106
swarm/network/kademlia/kaddb.go
View file

@ -43,14 +43,14 @@ type NodeRecord struct {
node Node
}
func (self *NodeRecord) setSeen() {
func (record *NodeRecord) setSeen() {
t := time.Now()
self.Seen = t
self.After = t
record.Seen = t
record.After = t
}
func (self *NodeRecord) String() string {
return fmt.Sprintf("<%v>", self.Addr)
func (record *NodeRecord) String() string {
return fmt.Sprintf("<%v>", record.Addr)
}
// persisted node record database ()
@ -77,11 +77,11 @@ func newKadDb(addr Address, params *KadParams) *KadDb {
}
}
func (self *KadDb) findOrCreate(index int, a Address, url string) *NodeRecord {
defer self.lock.Unlock()
self.lock.Lock()
func (db *KadDb) findOrCreate(index int, a Address, url string) *NodeRecord {
defer db.lock.Unlock()
db.lock.Lock()
record, found := self.index[a]
record, found := db.index[a]
if !found {
record = &NodeRecord{
Addr: a,
@ -89,8 +89,8 @@ func (self *KadDb) findOrCreate(index int, a Address, url string) *NodeRecord {
}
log.Info(fmt.Sprintf("add new record %v to kaddb", record))
// insert in kaddb
self.index[a] = record
self.Nodes[index] = append(self.Nodes[index], record)
db.index[a] = record
db.Nodes[index] = append(db.Nodes[index], record)
} else {
log.Info(fmt.Sprintf("found record %v in kaddb", record))
}
@ -102,26 +102,26 @@ func (self *KadDb) findOrCreate(index int, a Address, url string) *NodeRecord {
}
// add adds node records to kaddb (persisted node record db)
func (self *KadDb) add(nrs []*NodeRecord, proximityBin func(Address) int) {
defer self.lock.Unlock()
self.lock.Lock()
func (db *KadDb) add(nrs []*NodeRecord, proximityBin func(Address) int) {
defer db.lock.Unlock()
db.lock.Lock()
var n int
var nodes []*NodeRecord
for _, node := range nrs {
_, found := self.index[node.Addr]
if !found && node.Addr != self.Address {
_, found := db.index[node.Addr]
if !found && node.Addr != db.Address {
node.setSeen()
self.index[node.Addr] = node
db.index[node.Addr] = node
index := proximityBin(node.Addr)
dbcursor := self.cursors[index]
nodes = self.Nodes[index]
dbcursor := db.cursors[index]
nodes = db.Nodes[index]
// this is inefficient for allocation, need to just append then shift
newnodes := make([]*NodeRecord, len(nodes)+1)
copy(newnodes[:], nodes[:dbcursor])
newnodes[dbcursor] = node
copy(newnodes[dbcursor+1:], nodes[dbcursor:])
log.Trace(fmt.Sprintf("new nodes: %v, nodes: %v", newnodes, nodes))
self.Nodes[index] = newnodes
db.Nodes[index] = newnodes
n++
}
}
@ -168,10 +168,10 @@ offline past peer)
The second argument returned names the first missing slot found
*/
func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRecord, need bool, proxLimit int) {
func (db *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRecord, need bool, proxLimit int) {
// return nil, proxLimit indicates that all buckets are filled
defer self.lock.Unlock()
self.lock.Lock()
defer db.lock.Unlock()
db.lock.Lock()
var interval time.Duration
var found bool
@ -185,7 +185,7 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe
for rounds := 1; rounds <= maxBinSize; rounds++ {
ROUND:
// iterate over rows from PO 0 upto MaxProx
for po, dbrow := range self.Nodes {
for po, dbrow := range db.Nodes {
// if row has rounds connected peers, then take the next
if binSize(po) >= rounds {
continue ROUND
@ -200,7 +200,7 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe
// there is a missing slot - finding a node to connect to
// select a node record from the relavant kaddb row (of identical prox order)
ROW:
for cursor = self.cursors[po]; !found && count < len(dbrow); cursor = (cursor + 1) % len(dbrow) {
for cursor = db.cursors[po]; !found && count < len(dbrow); cursor = (cursor + 1) % len(dbrow) {
count++
node = dbrow[cursor]
@ -217,10 +217,10 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe
}
delta = time.Since(node.Seen)
if delta < self.initialRetryInterval {
delta = self.initialRetryInterval
if delta < db.initialRetryInterval {
delta = db.initialRetryInterval
}
if delta > self.purgeInterval {
if delta > db.purgeInterval {
// remove node
purge[cursor] = true
log.Debug(fmt.Sprintf("kaddb record %v (PO%03d:%d) unreachable since %v. Removed", node.Addr, po, cursor, node.Seen))
@ -230,15 +230,15 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe
log.Debug(fmt.Sprintf("kaddb record %v (PO%03d:%d) ready to be tried. seen at %v (%v ago), scheduled at %v", node.Addr, po, cursor, node.Seen, delta, node.After))
// scheduling next check
interval = delta * time.Duration(self.connRetryExp)
interval = delta * time.Duration(db.connRetryExp)
after = time.Now().Add(interval)
log.Debug(fmt.Sprintf("kaddb record %v (PO%03d:%d) selected as candidate connection %v. seen at %v (%v ago), selectable since %v, retry after %v (in %v)", node.Addr, po, cursor, rounds, node.Seen, delta, node.After, after, interval))
node.After = after
found = true
} // ROW
self.cursors[po] = cursor
self.delete(po, purge)
db.cursors[po] = cursor
db.delete(po, purge)
if found {
return node, need, proxLimit
}
@ -251,33 +251,33 @@ func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRe
// deletes the noderecords of a kaddb row corresponding to the indexes
// caller must hold the dblock
// the call is unsafe, no index checks
func (self *KadDb) delete(row int, purge []bool) {
func (db *KadDb) delete(row int, purge []bool) {
var nodes []*NodeRecord
dbrow := self.Nodes[row]
dbrow := db.Nodes[row]
for i, del := range purge {
if i == self.cursors[row] {
if i == db.cursors[row] {
//reset cursor
self.cursors[row] = len(nodes)
db.cursors[row] = len(nodes)
}
// delete the entry to be purged
if del {
delete(self.index, dbrow[i].Addr)
delete(db.index, dbrow[i].Addr)
continue
}
// otherwise append to new list
nodes = append(nodes, dbrow[i])
}
self.Nodes[row] = nodes
db.Nodes[row] = nodes
}
// save persists kaddb on disk (written to file on path in json format.
func (self *KadDb) save(path string, cb func(*NodeRecord, Node)) error {
defer self.lock.Unlock()
self.lock.Lock()
func (db *KadDb) save(path string, cb func(*NodeRecord, Node)) error {
defer db.lock.Unlock()
db.lock.Lock()
var n int
for _, b := range self.Nodes {
for _, b := range db.Nodes {
for _, node := range b {
n++
node.After = time.Now()
@ -288,7 +288,7 @@ func (self *KadDb) save(path string, cb func(*NodeRecord, Node)) error {
}
}
data, err := json.MarshalIndent(self, "", " ")
data, err := json.MarshalIndent(db, "", " ")
if err != nil {
return err
}
@ -302,9 +302,9 @@ func (self *KadDb) save(path string, cb func(*NodeRecord, Node)) error {
}
// Load(path) loads the node record database (kaddb) from file on path.
func (self *KadDb) load(path string, cb func(*NodeRecord, Node) error) (err error) {
defer self.lock.Unlock()
self.lock.Lock()
func (db *KadDb) load(path string, cb func(*NodeRecord, Node) error) (err error) {
defer db.lock.Unlock()
db.lock.Lock()
var data []byte
data, err = ioutil.ReadFile(path)
@ -312,13 +312,13 @@ func (self *KadDb) load(path string, cb func(*NodeRecord, Node) error) (err erro
return
}
err = json.Unmarshal(data, self)
err = json.Unmarshal(data, db)
if err != nil {
return
}
var n int
var purge []bool
for po, b := range self.Nodes {
for po, b := range db.Nodes {
purge = make([]bool, len(b))
ROW:
for i, node := range b {
@ -333,9 +333,9 @@ func (self *KadDb) load(path string, cb func(*NodeRecord, Node) error) (err erro
if node.After.IsZero() {
node.After = time.Now()
}
self.index[node.Addr] = node
db.index[node.Addr] = node
}
self.delete(po, purge)
db.delete(po, purge)
}
log.Info(fmt.Sprintf("loaded kaddb with %v nodes from %v", n, path))
@ -343,8 +343,8 @@ func (self *KadDb) load(path string, cb func(*NodeRecord, Node) error) (err erro
}
// accessor for KAD offline db count
func (self *KadDb) count() int {
defer self.lock.Unlock()
self.lock.Lock()
return len(self.index)
func (db *KadDb) count() int {
defer db.lock.Unlock()
db.lock.Lock()
return len(db.index)
}

182
swarm/network/kademlia/kademlia.go
View file

@ -109,31 +109,31 @@ func New(addr Address, params *KadParams) *Kademlia {
}
// accessor for KAD base address
func (self *Kademlia) Addr() Address {
return self.addr
func (kad *Kademlia) Addr() Address {
return kad.addr
}
// accessor for KAD active node count
func (self *Kademlia) Count() int {
defer self.lock.Unlock()
self.lock.Lock()
return self.count
func (kad *Kademlia) Count() int {
defer kad.lock.Unlock()
kad.lock.Lock()
return kad.count
}
// accessor for KAD active node count
func (self *Kademlia) DBCount() int {
return self.db.count()
func (kad *Kademlia) DBCount() int {
return kad.db.count()
}
// On is the entry point called when a new nodes is added
// unsafe in that node is not checked to be already active node (to be called once)
func (self *Kademlia) On(node Node, cb func(*NodeRecord, Node) error) (err error) {
log.Debug(fmt.Sprintf("%v", self))
defer self.lock.Unlock()
self.lock.Lock()
func (kad *Kademlia) On(node Node, cb func(*NodeRecord, Node) error) (err error) {
log.Debug(fmt.Sprintf("%v", kad))
defer kad.lock.Unlock()
kad.lock.Lock()
index := self.proximityBin(node.Addr())
record := self.db.findOrCreate(index, node.Addr(), node.Url())
index := kad.proximityBin(node.Addr())
record := kad.db.findOrCreate(index, node.Addr(), node.Url())
if cb != nil {
err = cb(record, node)
@ -145,21 +145,21 @@ func (self *Kademlia) On(node Node, cb func(*NodeRecord, Node) error) (err error
}
// insert in kademlia table of active nodes
bucket := self.buckets[index]
bucket := kad.buckets[index]
// if bucket is full insertion replaces the worst node
// TODO: give priority to peers with active traffic
if len(bucket) < self.BucketSize { // >= allows us to add peers beyond the bucketsize limitation
self.buckets[index] = append(bucket, node)
if len(bucket) < kad.BucketSize { // >= allows us to add peers beyond the bucketsize limitation
kad.buckets[index] = append(bucket, node)
bucketAddIndexCount[index].Inc(1)
log.Debug(fmt.Sprintf("add node %v to table", node))
self.setProxLimit(index, true)
kad.setProxLimit(index, true)
record.node = node
self.count++
kad.count++
return nil
}
// always rotate peers
idle := self.MaxIdleInterval
idle := kad.MaxIdleInterval
var pos int
var replaced Node
for i, p := range bucket {
@ -174,41 +174,41 @@ func (self *Kademlia) On(node Node, cb func(*NodeRecord, Node) error) (err error
log.Debug(fmt.Sprintf("all peers wanted, PO%03d bucket full", index))
return fmt.Errorf("bucket full")
}
log.Debug(fmt.Sprintf("node %v replaced by %v (idle for %v > %v)", replaced, node, idle, self.MaxIdleInterval))
log.Debug(fmt.Sprintf("node %v replaced by %v (idle for %v > %v)", replaced, node, idle, kad.MaxIdleInterval))
replaced.Drop()
// actually replace in the row. When off(node) is called, the peer is no longer in the row
bucket[pos] = node
// there is no change in bucket cardinalities so no prox limit adjustment is needed
record.node = node
self.count++
kad.count++
return nil
}
// Off is the called when a node is taken offline (from the protocol main loop exit)
func (self *Kademlia) Off(node Node, cb func(*NodeRecord, Node)) (err error) {
self.lock.Lock()
defer self.lock.Unlock()
func (kad *Kademlia) Off(node Node, cb func(*NodeRecord, Node)) (err error) {
kad.lock.Lock()
defer kad.lock.Unlock()
index := self.proximityBin(node.Addr())
index := kad.proximityBin(node.Addr())
bucketRmIndexCount[index].Inc(1)
bucket := self.buckets[index]
bucket := kad.buckets[index]
for i := 0; i < len(bucket); i++ {
if node.Addr() == bucket[i].Addr() {
self.buckets[index] = append(bucket[:i], bucket[(i+1):]...)
self.setProxLimit(index, false)
kad.buckets[index] = append(bucket[:i], bucket[(i+1):]...)
kad.setProxLimit(index, false)
break
}
}
record := self.db.index[node.Addr()]
record := kad.db.index[node.Addr()]
// callback on remove
if cb != nil {
cb(record, record.node)
}
record.node = nil
self.count--
log.Debug(fmt.Sprintf("remove node %v from table, population now is %v", node, self.count))
kad.count--
log.Debug(fmt.Sprintf("remove node %v from table, population now is %v", node, kad.count))
return
}
@ -218,39 +218,39 @@ func (self *Kademlia) Off(node Node, cb func(*NodeRecord, Node)) (err error) {
// 2) the sum of all items are the minimum possible but higher than ProxBinSize
// adjust Prox (proxLimit and proxSize after an insertion/removal of nodes)
// caller holds the lock
func (self *Kademlia) setProxLimit(r int, on bool) {
func (kad *Kademlia) setProxLimit(r int, on bool) {
// if the change is outside the core (PO lower)
// and the change does not leave a bucket empty then
// no adjustment needed
if r < self.proxLimit && len(self.buckets[r]) > 0 {
if r < kad.proxLimit && len(kad.buckets[r]) > 0 {
return
}
// if on=a node was added, then r must be within prox limit so increment cardinality
if on {
self.proxSize++
curr := len(self.buckets[self.proxLimit])
kad.proxSize++
curr := len(kad.buckets[kad.proxLimit])
// if now core is big enough without the furthest bucket, then contract
// this can result in more than one bucket change
for self.proxSize >= self.ProxBinSize+curr && curr > 0 {
self.proxSize -= curr
self.proxLimit++
curr = len(self.buckets[self.proxLimit])
for kad.proxSize >= kad.ProxBinSize+curr && curr > 0 {
kad.proxSize -= curr
kad.proxLimit++
curr = len(kad.buckets[kad.proxLimit])
log.Trace(fmt.Sprintf("proxbin contraction (size: %v, limit: %v, bin: %v)", self.proxSize, self.proxLimit, r))
log.Trace(fmt.Sprintf("proxbin contraction (size: %v, limit: %v, bin: %v)", kad.proxSize, kad.proxLimit, r))
}
return
}
// otherwise
if r >= self.proxLimit {
self.proxSize--
if r >= kad.proxLimit {
kad.proxSize--
}
// expand core by lowering prox limit until hit zero or cover the empty bucket or reached target cardinality
for (self.proxSize < self.ProxBinSize || r < self.proxLimit) &&
self.proxLimit > 0 {
for (kad.proxSize < kad.ProxBinSize || r < kad.proxLimit) &&
kad.proxLimit > 0 {
//
self.proxLimit--
self.proxSize += len(self.buckets[self.proxLimit])
log.Trace(fmt.Sprintf("proxbin expansion (size: %v, limit: %v, bin: %v)", self.proxSize, self.proxLimit, r))
kad.proxLimit--
kad.proxSize += len(kad.buckets[kad.proxLimit])
log.Trace(fmt.Sprintf("proxbin expansion (size: %v, limit: %v, bin: %v)", kad.proxSize, kad.proxLimit, r))
}
}
@ -259,15 +259,15 @@ returns the list of nodes belonging to the same proximity bin
as the target. The most proximate bin will be the union of the bins between
proxLimit and MaxProx.
*/
func (self *Kademlia) FindClosest(target Address, max int) []Node {
self.lock.Lock()
defer self.lock.Unlock()
func (kad *Kademlia) FindClosest(target Address, max int) []Node {
kad.lock.Lock()
defer kad.lock.Unlock()
r := nodesByDistance{
target: target,
}
po := self.proximityBin(target)
po := kad.proximityBin(target)
index := po
step := 1
log.Trace(fmt.Sprintf("serving %v nodes at %v (PO%02d)", max, index, po))
@ -284,17 +284,17 @@ func (self *Kademlia) FindClosest(target Address, max int) []Node {
var n int
for index >= 0 {
// add entire bucket
for _, p := range self.buckets[index] {
for _, p := range kad.buckets[index] {
r.push(p, limit)
n++
}
// terminate if index reached the bottom or enough peers > min
log.Trace(fmt.Sprintf("add %v -> %v (PO%02d, PO%03d)", len(self.buckets[index]), n, index, po))
log.Trace(fmt.Sprintf("add %v -> %v (PO%02d, PO%03d)", len(kad.buckets[index]), n, index, po))
if n >= min && (step < 0 || max == 0) {
break
}
// reach top most non-empty PO bucket, turn around
if index == self.MaxProx {
if index == kad.MaxProx {
index = po
step = -1
}
@ -304,15 +304,15 @@ func (self *Kademlia) FindClosest(target Address, max int) []Node {
return r.nodes
}
func (self *Kademlia) Suggest() (*NodeRecord, bool, int) {
defer self.lock.RUnlock()
self.lock.RLock()
return self.db.findBest(self.BucketSize, func(i int) int { return len(self.buckets[i]) })
func (kad *Kademlia) Suggest() (*NodeRecord, bool, int) {
defer kad.lock.RUnlock()
kad.lock.RLock()
return kad.db.findBest(kad.BucketSize, func(i int) int { return len(kad.buckets[i]) })
}
// adds node records to kaddb (persisted node record db)
func (self *Kademlia) Add(nrs []*NodeRecord) {
self.db.add(nrs, self.proximityBin)
func (kad *Kademlia) Add(nrs []*NodeRecord) {
kad.db.add(nrs, kad.proximityBin)
}
// nodesByDistance is a list of nodes, ordered by distance to target.
@ -369,52 +369,52 @@ a guaranteed constant maximum limit on the number of hops needed to reach one
node from the other.
*/
func (self *Kademlia) proximityBin(other Address) (ret int) {
ret = proximity(self.addr, other)
if ret > self.MaxProx {
ret = self.MaxProx
func (kad *Kademlia) proximityBin(other Address) (ret int) {
ret = proximity(kad.addr, other)
if ret > kad.MaxProx {
ret = kad.MaxProx
}
return
}
// provides keyrange for chunk db iteration
func (self *Kademlia) KeyRange(other Address) (start, stop Address) {
defer self.lock.RUnlock()
self.lock.RLock()
return KeyRange(self.addr, other, self.proxLimit)
func (kad *Kademlia) KeyRange(other Address) (start, stop Address) {
defer kad.lock.RUnlock()
kad.lock.RLock()
return KeyRange(kad.addr, other, kad.proxLimit)
}
// save persists kaddb on disk (written to file on path in json format.
func (self *Kademlia) Save(path string, cb func(*NodeRecord, Node)) error {
return self.db.save(path, cb)
func (kad *Kademlia) Save(path string, cb func(*NodeRecord, Node)) error {
return kad.db.save(path, cb)
}
// Load(path) loads the node record database (kaddb) from file on path.
func (self *Kademlia) Load(path string, cb func(*NodeRecord, Node) error) (err error) {
return self.db.load(path, cb)
func (kad *Kademlia) Load(path string, cb func(*NodeRecord, Node) error) (err error) {
return kad.db.load(path, cb)
}
// kademlia table + kaddb table displayed with ascii
func (self *Kademlia) String() string {
defer self.lock.RUnlock()
self.lock.RLock()
defer self.db.lock.RUnlock()
self.db.lock.RLock()
func (kad *Kademlia) String() string {
defer kad.lock.RUnlock()
kad.lock.RLock()
defer kad.db.lock.RUnlock()
kad.db.lock.RLock()
var rows []string
rows = append(rows, "=========================================================================")
rows = append(rows, fmt.Sprintf("%v KΛÐΞMLIΛ hive: queen's address: %v", time.Now().UTC().Format(time.UnixDate), self.addr.String()[:6]))
rows = append(rows, fmt.Sprintf("population: %d (%d), proxLimit: %d, proxSize: %d", self.count, len(self.db.index), self.proxLimit, self.proxSize))
rows = append(rows, fmt.Sprintf("MaxProx: %d, ProxBinSize: %d, BucketSize: %d", self.MaxProx, self.ProxBinSize, self.BucketSize))
rows = append(rows, fmt.Sprintf("%v KΛÐΞMLIΛ hive: queen's address: %v", time.Now().UTC().Format(time.UnixDate), kad.addr.String()[:6]))
rows = append(rows, fmt.Sprintf("population: %d (%d), proxLimit: %d, proxSize: %d", kad.count, len(kad.db.index), kad.proxLimit, kad.proxSize))
rows = append(rows, fmt.Sprintf("MaxProx: %d, ProxBinSize: %d, BucketSize: %d", kad.MaxProx, kad.ProxBinSize, kad.BucketSize))
for i, bucket := range self.buckets {
for i, bucket := range kad.buckets {
if i == self.proxLimit {
if i == kad.proxLimit {
rows = append(rows, fmt.Sprintf("============ PROX LIMIT: %d ==========================================", i))
}
row := []string{fmt.Sprintf("%03d", i), fmt.Sprintf("%2d", len(bucket))}
var k int
c := self.db.cursors[i]
c := kad.db.cursors[i]
for ; k < len(bucket); k++ {
p := bucket[(c+k)%len(bucket)]
row = append(row, p.Addr().String()[:6])
@ -425,16 +425,16 @@ func (self *Kademlia) String() string {
for ; k < 4; k++ {
row = append(row, " ")
}
row = append(row, fmt.Sprintf("| %2d %2d", len(self.db.Nodes[i]), self.db.cursors[i]))
row = append(row, fmt.Sprintf("| %2d %2d", len(kad.db.Nodes[i]), kad.db.cursors[i]))
for j, p := range self.db.Nodes[i] {
for j, p := range kad.db.Nodes[i] {
row = append(row, p.Addr.String()[:6])
if j == 3 {
break
}
}
rows = append(rows, strings.Join(row, " "))
if i == self.MaxProx {
if i == kad.MaxProx {
}
}
rows = append(rows, "=========================================================================")
@ -442,12 +442,12 @@ func (self *Kademlia) String() string {
}
//We have to build up the array of counters for each index
func (self *Kademlia) initMetricsVariables() {
func (kad *Kademlia) initMetricsVariables() {
//create the arrays
bucketAddIndexCount = make([]metrics.Counter, self.MaxProx+1)
bucketRmIndexCount = make([]metrics.Counter, self.MaxProx+1)
bucketAddIndexCount = make([]metrics.Counter, kad.MaxProx+1)
bucketRmIndexCount = make([]metrics.Counter, kad.MaxProx+1)
//at each index create a metrics counter
for i := 0; i < (self.KadParams.MaxProx + 1); i++ {
for i := 0; i < (kad.KadParams.MaxProx + 1); i++ {
bucketAddIndexCount[i] = metrics.NewRegisteredCounter(fmt.Sprintf("network.kademlia.bucket.add.%d.index", i), nil)
bucketRmIndexCount[i] = metrics.NewRegisteredCounter(fmt.Sprintf("network.kademlia.bucket.rm.%d.index", i), nil)
}

22
swarm/network/kademlia/kademlia_test.go
View file

@ -272,31 +272,31 @@ func TestSaveLoad(t *testing.T) {
}
}
func (self *Kademlia) proxCheck(t *testing.T) bool {
func (kad *Kademlia) proxCheck(t *testing.T) bool {
var sum int
for i, b := range self.buckets {
for i, b := range kad.buckets {
l := len(b)
// if we are in the high prox multibucket
if i >= self.proxLimit {
if i >= kad.proxLimit {
sum += l
} else if l == 0 {
t.Errorf("bucket %d empty, yet proxLimit is %d\n%v", len(b), self.proxLimit, self)
t.Errorf("bucket %d empty, yet proxLimit is %d\n%v", len(b), kad.proxLimit, kad)
return false
}
}
// check if merged high prox bucket does not exceed size
if sum > 0 {
if sum != self.proxSize {
t.Errorf("proxSize incorrect, expected %v, got %v", sum, self.proxSize)
if sum != kad.proxSize {
t.Errorf("proxSize incorrect, expected %v, got %v", sum, kad.proxSize)
return false
}
last := len(self.buckets[self.proxLimit])
if last > 0 && sum >= self.ProxBinSize+last {
t.Errorf("proxLimit %v incorrect, redundant non-empty bucket %d added to proxBin with %v (target %v)\n%v", self.proxLimit, last, sum-last, self.ProxBinSize, self)
last := len(kad.buckets[kad.proxLimit])
if last > 0 && sum >= kad.ProxBinSize+last {
t.Errorf("proxLimit %v incorrect, redundant non-empty bucket %d added to proxBin with %v (target %v)\n%v", kad.proxLimit, last, sum-last, kad.ProxBinSize, kad)
return false
}
if self.proxLimit > 0 && sum < self.ProxBinSize {
t.Errorf("proxLimit %v incorrect. proxSize %v is less than target %v, yet there is more peers", self.proxLimit, sum, self.ProxBinSize)
if kad.proxLimit > 0 && sum < kad.ProxBinSize {
t.Errorf("proxLimit %v incorrect. proxSize %v is less than target %v, yet there is more peers", kad.proxLimit, sum, kad.ProxBinSize)
return false
}
}

94
swarm/network/messages.go
View file

@ -64,8 +64,8 @@ type statusMsgData struct {
NetworkId uint64
}
func (self *statusMsgData) String() string {
return fmt.Sprintf("Status: Version: %v, ID: %v, Addr: %v, Swap: %v, NetworkId: %v", self.Version, self.ID, self.Addr, self.Swap, self.NetworkId)
func (data *statusMsgData) String() string {
return fmt.Sprintf("Status: Version: %v, ID: %v, Addr: %v, Swap: %v, NetworkId: %v", data.Version, data.ID, data.Addr, data.Swap, data.NetworkId)
}
/*
@ -86,18 +86,18 @@ type storeRequestMsgData struct {
from *peer // [not serialised] protocol registers the requester
}
func (self storeRequestMsgData) String() string {
func (data storeRequestMsgData) String() string {
var from string
if self.from == nil {
from = "self"
if data.from == nil {
from = "data"
} else {
from = self.from.Addr().String()
from = data.from.Addr().String()
}
end := len(self.SData)
if len(self.SData) > 10 {
end := len(data.SData)
if len(data.SData) > 10 {
end = 10
}
return fmt.Sprintf("from: %v, Key: %v; ID: %v, requestTimeout: %v, storageTimeout: %v, SData %x", from, self.Key, self.Id, self.requestTimeout, self.storageTimeout, self.SData[:end])
return fmt.Sprintf("from: %v, Key: %v; ID: %v, requestTimeout: %v, storageTimeout: %v, SData %x", from, data.Key, data.Id, data.requestTimeout, data.storageTimeout, data.SData[:end])
}
/*
@ -133,40 +133,40 @@ type retrieveRequestMsgData struct {
from *peer //
}
func (self *retrieveRequestMsgData) String() string {
func (data *retrieveRequestMsgData) String() string {
var from string
if self.from == nil {
if data.from == nil {
from = "ourselves"
} else {
from = self.from.Addr().String()
from = data.from.Addr().String()
}
var target []byte
if len(self.Key) > 3 {
target = self.Key[:4]
if len(data.Key) > 3 {
target = data.Key[:4]
}
return fmt.Sprintf("from: %v, Key: %x; ID: %v, MaxSize: %v, MaxPeers: %d", from, target, self.Id, self.MaxSize, self.MaxPeers)
return fmt.Sprintf("from: %v, Key: %x; ID: %v, MaxSize: %v, MaxPeers: %d", from, target, data.Id, data.MaxSize, data.MaxPeers)
}
// lookups are encoded by missing request ID
func (self *retrieveRequestMsgData) isLookup() bool {
return self.Id == 0
func (data *retrieveRequestMsgData) isLookup() bool {
return data.Id == 0
}
// sets timeout fields
func (self *retrieveRequestMsgData) setTimeout(t *time.Time) {
self.timeout = t
func (data *retrieveRequestMsgData) setTimeout(t *time.Time) {
data.timeout = t
if t != nil {
self.Timeout = uint64(t.UnixNano())
data.Timeout = uint64(t.UnixNano())
} else {
self.Timeout = 0
data.Timeout = 0
}
}
func (self *retrieveRequestMsgData) getTimeout() (t *time.Time) {
if self.Timeout > 0 && self.timeout == nil {
timeout := time.Unix(int64(self.Timeout), 0)
func (data *retrieveRequestMsgData) getTimeout() (t *time.Time) {
if data.Timeout > 0 && data.timeout == nil {
timeout := time.Unix(int64(data.Timeout), 0)
t = &timeout
self.timeout = t
data.timeout = t
}
return
}
@ -180,10 +180,10 @@ type peerAddr struct {
}
// peerAddr pretty prints as enode
func (self *peerAddr) String() string {
func (data *peerAddr) String() string {
var nodeid discover.NodeID
copy(nodeid[:], self.ID)
return discover.NewNode(nodeid, self.IP, 0, self.Port).String()
copy(nodeid[:], data.ID)
return discover.NewNode(nodeid, data.IP, 0, data.Port).String()
}
/*
@ -199,7 +199,7 @@ the timeout or not.
NodeID serves as the owner of payment contracts and signer of proofs of transfer.
The Key is the target (if response to a retrieval request) or missing (zero value)
peers address (hash of NodeID) if retrieval request was a self lookup.
peers address (hash of NodeID) if retrieval request was a data lookup.
Peers message is requested by retrieval requests with a missing or zero value request ID
*/
@ -213,26 +213,26 @@ type peersMsgData struct {
}
// peers msg pretty printer
func (self *peersMsgData) String() string {
func (data *peersMsgData) String() string {
var from string
if self.from == nil {
if data.from == nil {
from = "ourselves"
} else {
from = self.from.Addr().String()
from = data.from.Addr().String()
}
var target []byte
if len(self.Key) > 3 {
target = self.Key[:4]
if len(data.Key) > 3 {
target = data.Key[:4]
}
return fmt.Sprintf("from: %v, Key: %x; ID: %v, Peers: %v", from, target, self.Id, self.Peers)
return fmt.Sprintf("from: %v, Key: %x; ID: %v, Peers: %v", from, target, data.Id, data.Peers)
}
func (self *peersMsgData) setTimeout(t *time.Time) {
self.timeout = t
func (data *peersMsgData) setTimeout(t *time.Time) {
data.timeout = t
if t != nil {
self.Timeout = uint64(t.UnixNano())
data.Timeout = uint64(t.UnixNano())
} else {
self.Timeout = 0
data.Timeout = 0
}
}
@ -248,8 +248,8 @@ type syncRequestMsgData struct {
SyncState *syncState `rlp:"nil"`
}
func (self *syncRequestMsgData) String() string {
return fmt.Sprintf("%v", self.SyncState)
func (data *syncRequestMsgData) String() string {
return fmt.Sprintf("%v", data.SyncState)
}
/*
@ -265,8 +265,8 @@ type deliveryRequestMsgData struct {
Deliver []*syncRequest
}
func (self *deliveryRequestMsgData) String() string {
return fmt.Sprintf("sync request for new chunks\ndelivery request for %v chunks", len(self.Deliver))
func (data *deliveryRequestMsgData) String() string {
return fmt.Sprintf("sync request for new chunks\ndelivery request for %v chunks", len(data.Deliver))
}
/*
@ -287,8 +287,8 @@ type unsyncedKeysMsgData struct {
State *syncState
}
func (self *unsyncedKeysMsgData) String() string {
return fmt.Sprintf("sync: keys of %d new chunks (state %v) => synced: %v", len(self.Unsynced), self.State, self.State.Synced)
func (data *unsyncedKeysMsgData) String() string {
return fmt.Sprintf("sync: keys of %d new chunks (state %v) => synced: %v", len(data.Unsynced), data.State, data.State.Synced)
}
/*
@ -303,6 +303,6 @@ type paymentMsgData struct {
Promise *chequebook.Cheque // payment with cheque
}
func (self *paymentMsgData) String() string {
return fmt.Sprintf("payment for %d units: %v", self.Units, self.Promise)
func (data *paymentMsgData) String() string {
return fmt.Sprintf("payment for %d units: %v", data.Units, data.Promise)
}

190
swarm/network/protocol.go
View file

@ -145,7 +145,7 @@ the main protocol loop that
*/
func run(requestDb *storage.LDBDatabase, depo StorageHandler, backend chequebook.Backend, hive *Hive, dbaccess *DbAccess, sp *bzzswap.SwapParams, sy *SyncParams, networkId uint64, p *p2p.Peer, rw p2p.MsgReadWriter) (err error) {
self := &bzz{
b := &bzz{
storage: depo,
backend: backend,
hive: hive,
@ -161,30 +161,30 @@ func run(requestDb *storage.LDBDatabase, depo StorageHandler, backend chequebook
}
// handle handshake
err = self.handleStatus()
err = b.handleStatus()
if err != nil {
return err
}
defer func() {
// if the handler loop exits, the peer is disconnecting
// deregister the peer in the hive
self.hive.removePeer(&peer{bzz: self})
if self.syncer != nil {
self.syncer.stop() // quits request db and delivery loops, save requests
b.hive.removePeer(&peer{bzz: b})
if b.syncer != nil {
b.syncer.stop() // quits request db and delivery loops, save requests
}
if self.swap != nil {
self.swap.Stop() // quits chequebox autocash etc
if b.swap != nil {
b.swap.Stop() // quits chequebox autocash etc
}
}()
// the main forever loop that handles incoming requests
for {
if self.hive.blockRead {
if b.hive.blockRead {
log.Warn(fmt.Sprintf("Cannot read network"))
time.Sleep(100 * time.Millisecond)
continue
}
err = self.handle()
err = b.handle()
if err != nil {
return
}
@ -193,13 +193,13 @@ func run(requestDb *storage.LDBDatabase, depo StorageHandler, backend chequebook
// TODO: may need to implement protocol drop only? don't want to kick off the peer
// if they are useful for other protocols
func (self *bzz) Drop() {
self.peer.Disconnect(p2p.DiscSubprotocolError)
func (b *bzz) Drop() {
b.peer.Disconnect(p2p.DiscSubprotocolError)
}
// one cycle of the main forever loop that handles and dispatches incoming messages
func (self *bzz) handle() error {
msg, err := self.rw.ReadMsg()
func (b *bzz) handle() error {
msg, err := b.rw.ReadMsg()
log.Debug(fmt.Sprintf("<- %v", msg))
if err != nil {
return err
@ -229,10 +229,10 @@ func (self *bzz) handle() error {
return fmt.Errorf("<- %v: Data too short (%v)", msg, n)
}
// last Active time is set only when receiving chunks
self.lastActive = time.Now()
b.lastActive = time.Now()
log.Trace(fmt.Sprintf("incoming store request: %s", req.String()))
// swap accounting is done within forwarding
self.storage.HandleStoreRequestMsg(&req, &peer{bzz: self})
b.storage.HandleStoreRequestMsg(&req, &peer{bzz: b})
case retrieveRequestMsg:
// retrieve Requests are dispatched to netStore
@ -241,18 +241,18 @@ func (self *bzz) handle() error {
if err := msg.Decode(&req); err != nil {
return fmt.Errorf("<- %v: %v", msg, err)
}
req.from = &peer{bzz: self}
req.from = &peer{bzz: b}
// if request is lookup and not to be delivered
if req.isLookup() {
log.Trace(fmt.Sprintf("self lookup for %v: responding with peers only...", req.from))
log.Trace(fmt.Sprintf("b lookup for %v: responding with peers only...", req.from))
} else if req.Key == nil {
return fmt.Errorf("protocol handler: req.Key == nil || req.Timeout == nil")
} else {
// swap accounting is done within netStore
self.storage.HandleRetrieveRequestMsg(&req, &peer{bzz: self})
b.storage.HandleRetrieveRequestMsg(&req, &peer{bzz: b})
}
// direct response with peers, TODO: sort this out
self.hive.peers(&req)
b.hive.peers(&req)
case peersMsg:
// response to lookups and immediate response to retrieve requests
@ -262,9 +262,9 @@ func (self *bzz) handle() error {
if err := msg.Decode(&req); err != nil {
return fmt.Errorf("<- %v: %v", msg, err)
}
req.from = &peer{bzz: self}
req.from = &peer{bzz: b}
log.Trace(fmt.Sprintf("<- peer addresses: %v", req))
self.hive.HandlePeersMsg(&req, &peer{bzz: self})
b.hive.HandlePeersMsg(&req, &peer{bzz: b})
case syncRequestMsg:
syncRequestMsgCounter.Inc(1)
@ -273,8 +273,8 @@ func (self *bzz) handle() error {
return fmt.Errorf("<- %v: %v", msg, err)
}
log.Debug(fmt.Sprintf("<- sync request: %v", req))
self.lastActive = time.Now()
self.sync(req.SyncState)
b.lastActive = time.Now()
b.sync(req.SyncState)
case unsyncedKeysMsg:
// coming from parent node offering
@ -284,8 +284,8 @@ func (self *bzz) handle() error {
return fmt.Errorf("<- %v: %v", msg, err)
}
log.Debug(fmt.Sprintf("<- unsynced keys : %s", req.String()))
err := self.storage.HandleUnsyncedKeysMsg(&req, &peer{bzz: self})
self.lastActive = time.Now()
err := b.storage.HandleUnsyncedKeysMsg(&req, &peer{bzz: b})
b.lastActive = time.Now()
if err != nil {
return fmt.Errorf("<- %v: %v", msg, err)
}
@ -299,8 +299,8 @@ func (self *bzz) handle() error {
return fmt.Errorf("<-msg %v: %v", msg, err)
}
log.Debug(fmt.Sprintf("<- delivery request: %s", req.String()))
err := self.storage.HandleDeliveryRequestMsg(&req, &peer{bzz: self})
self.lastActive = time.Now()
err := b.storage.HandleDeliveryRequestMsg(&req, &peer{bzz: b})
b.lastActive = time.Now()
if err != nil {
return fmt.Errorf("<- %v: %v", msg, err)
}
@ -308,13 +308,13 @@ func (self *bzz) handle() error {
case paymentMsg:
// swap protocol message for payment, Units paid for, Cheque paid with
paymentMsgCounter.Inc(1)
if self.swapEnabled {
if b.swapEnabled {
var req paymentMsgData
if err := msg.Decode(&req); err != nil {
return fmt.Errorf("<- %v: %v", msg, err)
}
log.Debug(fmt.Sprintf("<- payment: %s", req.String()))
self.swap.Receive(int(req.Units), req.Promise)
b.swap.Receive(int(req.Units), req.Promise)
}
default:
@ -325,27 +325,27 @@ func (self *bzz) handle() error {
return nil
}
func (self *bzz) handleStatus() (err error) {
func (b *bzz) handleStatus() (err error) {
handshake := &statusMsgData{
Version: uint64(Version),
ID: "honey",
Addr: self.selfAddr(),
NetworkId: self.NetworkId,
Addr: b.bAddr(),
NetworkId: b.NetworkId,
Swap: &bzzswap.SwapProfile{
Profile: self.swapParams.Profile,
PayProfile: self.swapParams.PayProfile,
Profile: b.swapParams.Profile,
PayProfile: b.swapParams.PayProfile,
},
}
err = p2p.Send(self.rw, statusMsg, handshake)
err = p2p.Send(b.rw, statusMsg, handshake)
if err != nil {
return err
}
// read and handle remote status
var msg p2p.Msg
msg, err = self.rw.ReadMsg()
msg, err = b.rw.ReadMsg()
if err != nil {
return err
}
@ -365,52 +365,52 @@ func (self *bzz) handleStatus() (err error) {
return fmt.Errorf("<- %v: %v", msg, err)
}
if status.NetworkId != self.NetworkId {
return fmt.Errorf("network id mismatch: %d (!= %d)", status.NetworkId, self.NetworkId)
if status.NetworkId != b.NetworkId {
return fmt.Errorf("network id mismatch: %d (!= %d)", status.NetworkId, b.NetworkId)
}
if Version != status.Version {
return fmt.Errorf("protocol version mismatch: %d (!= %d)", status.Version, Version)
}
self.remoteAddr = self.peerAddr(status.Addr)
log.Trace(fmt.Sprintf("self: advertised IP: %v, peer advertised: %v, local address: %v\npeer: advertised IP: %v, remote address: %v\n", self.selfAddr(), self.remoteAddr, self.peer.LocalAddr(), status.Addr.IP, self.peer.RemoteAddr()))
b.remoteAddr = b.peerAddr(status.Addr)
log.Trace(fmt.Sprintf("b: advertised IP: %v, peer advertised: %v, local address: %v\npeer: advertised IP: %v, remote address: %v\n", b.bAddr(), b.remoteAddr, b.peer.LocalAddr(), status.Addr.IP, b.peer.RemoteAddr()))
if self.swapEnabled {
if b.swapEnabled {
// set remote profile for accounting
self.swap, err = bzzswap.NewSwap(self.swapParams, status.Swap, self.backend, self)
b.swap, err = bzzswap.NewSwap(b.swapParams, status.Swap, b.backend, b)
if err != nil {
return err
}
}
log.Info(fmt.Sprintf("Peer %08x is capable (%d/%d)", self.remoteAddr.Addr[:4], status.Version, status.NetworkId))
err = self.hive.addPeer(&peer{bzz: self})
log.Info(fmt.Sprintf("Peer %08x is capable (%d/%d)", b.remoteAddr.Addr[:4], status.Version, status.NetworkId))
err = b.hive.addPeer(&peer{bzz: b})
if err != nil {
return err
}
// hive sets syncstate so sync should start after node added
log.Info(fmt.Sprintf("syncronisation request sent with %v", self.syncState))
self.syncRequest()
log.Info(fmt.Sprintf("syncronisation request sent with %v", b.syncState))
b.syncRequest()
return nil
}
func (self *bzz) sync(state *syncState) error {
func (b *bzz) sync(state *syncState) error {
// syncer setup
if self.syncer != nil {
if b.syncer != nil {
return errors.New("sync request can only be sent once")
}
cnt := self.dbAccess.counter()
remoteaddr := self.remoteAddr.Addr
start, stop := self.hive.kad.KeyRange(remoteaddr)
cnt := b.dbAccess.counter()
remoteaddr := b.remoteAddr.Addr
start, stop := b.hive.kad.KeyRange(remoteaddr)
// an explicitly received nil syncstate disables syncronisation
if state == nil {
self.syncEnabled = false
log.Warn(fmt.Sprintf("syncronisation disabled for peer %v", self))
b.syncEnabled = false
log.Warn(fmt.Sprintf("syncronisation disabled for peer %v", b))
state = &syncState{DbSyncState: &storage.DbSyncState{}, Synced: true}
} else {
state.synced = make(chan bool)
@ -419,45 +419,45 @@ func (self *bzz) sync(state *syncState) error {
state.Start = storage.Key(start[:])
state.Stop = storage.Key(stop[:])
}
log.Debug(fmt.Sprintf("syncronisation requested by peer %v at state %v", self, state))
log.Debug(fmt.Sprintf("syncronisation requested by peer %v at state %v", b, state))
}
var err error
self.syncer, err = newSyncer(
self.requestDb,
b.syncer, err = newSyncer(
b.requestDb,
storage.Key(remoteaddr[:]),
self.dbAccess,
self.unsyncedKeys, self.store,
self.syncParams, state, func() bool { return self.syncEnabled },
b.dbAccess,
b.unsyncedKeys, b.store,
b.syncParams, state, func() bool { return b.syncEnabled },
)
if err != nil {
return nil
}
log.Trace(fmt.Sprintf("syncer set for peer %v", self))
log.Trace(fmt.Sprintf("syncer set for peer %v", b))
return nil
}
func (self *bzz) String() string {
return self.remoteAddr.String()
func (b *bzz) String() string {
return b.remoteAddr.String()
}
// repair reported address if IP missing
func (self *bzz) peerAddr(base *peerAddr) *peerAddr {
func (b *bzz) peerAddr(base *peerAddr) *peerAddr {
if base.IP.IsUnspecified() {
host, _, _ := net.SplitHostPort(self.peer.RemoteAddr().String())
host, _, _ := net.SplitHostPort(b.peer.RemoteAddr().String())
base.IP = net.ParseIP(host)
}
return base
}
// returns self advertised node connection info (listening address w enodes)
// returns b advertised node connection info (listening address w enodes)
// IP will get repaired on the other end if missing
// or resolved via ID by discovery at dialout
func (self *bzz) selfAddr() *peerAddr {
id := self.hive.id
host, port, _ := net.SplitHostPort(self.hive.listenAddr())
func (b *bzz) bAddr() *peerAddr {
id := b.hive.id
host, port, _ := net.SplitHostPort(b.hive.listenAddr())
intport, _ := strconv.Atoi(port)
addr := &peerAddr{
Addr: self.hive.addr,
Addr: b.hive.addr,
ID: id[:],
IP: net.ParseIP(host),
Port: uint16(intport),
@ -467,68 +467,68 @@ func (self *bzz) selfAddr() *peerAddr {
// outgoing messages
// send retrieveRequestMsg
func (self *bzz) retrieve(req *retrieveRequestMsgData) error {
return self.send(retrieveRequestMsg, req)
func (b *bzz) retrieve(req *retrieveRequestMsgData) error {
return b.send(retrieveRequestMsg, req)
}
// send storeRequestMsg
func (self *bzz) store(req *storeRequestMsgData) error {
return self.send(storeRequestMsg, req)
func (b *bzz) store(req *storeRequestMsgData) error {
return b.send(storeRequestMsg, req)
}
func (self *bzz) syncRequest() error {
func (b *bzz) syncRequest() error {
req := &syncRequestMsgData{}
if self.hive.syncEnabled {
log.Debug(fmt.Sprintf("syncronisation request to peer %v at state %v", self, self.syncState))
req.SyncState = self.syncState
if b.hive.syncEnabled {
log.Debug(fmt.Sprintf("syncronisation request to peer %v at state %v", b, b.syncState))
req.SyncState = b.syncState
}
if self.syncState == nil {
log.Warn(fmt.Sprintf("syncronisation disabled for peer %v at state %v", self, self.syncState))
if b.syncState == nil {
log.Warn(fmt.Sprintf("syncronisation disabled for peer %v at state %v", b, b.syncState))
}
return self.send(syncRequestMsg, req)
return b.send(syncRequestMsg, req)
}
// queue storeRequestMsg in request db
func (self *bzz) deliveryRequest(reqs []*syncRequest) error {
func (b *bzz) deliveryRequest(reqs []*syncRequest) error {
req := &deliveryRequestMsgData{
Deliver: reqs,
}
return self.send(deliveryRequestMsg, req)
return b.send(deliveryRequestMsg, req)
}
// batch of syncRequests to send off
func (self *bzz) unsyncedKeys(reqs []*syncRequest, state *syncState) error {
func (b *bzz) unsyncedKeys(reqs []*syncRequest, state *syncState) error {
req := &unsyncedKeysMsgData{
Unsynced: reqs,
State: state,
}
return self.send(unsyncedKeysMsg, req)
return b.send(unsyncedKeysMsg, req)
}
// send paymentMsg
func (self *bzz) Pay(units int, promise swap.Promise) {
func (b *bzz) Pay(units int, promise swap.Promise) {
req := &paymentMsgData{uint(units), promise.(*chequebook.Cheque)}
self.payment(req)
b.payment(req)
}
// send paymentMsg
func (self *bzz) payment(req *paymentMsgData) error {
return self.send(paymentMsg, req)
func (b *bzz) payment(req *paymentMsgData) error {
return b.send(paymentMsg, req)
}
// sends peersMsg
func (self *bzz) peers(req *peersMsgData) error {
return self.send(peersMsg, req)
func (b *bzz) peers(req *peersMsgData) error {
return b.send(peersMsg, req)
}
func (self *bzz) send(msg uint64, data interface{}) error {
if self.hive.blockWrite {
func (b *bzz) send(msg uint64, data interface{}) error {
if b.hive.blockWrite {
return fmt.Errorf("network write blocked")
}
log.Trace(fmt.Sprintf("-> %v: %v (%T) to %v", msg, data, data, self))
err := p2p.Send(self.rw, msg, data)
log.Trace(fmt.Sprintf("-> %v: %v (%T) to %v", msg, data, data, b))
err := p2p.Send(b.rw, msg, data)
if err != nil {
self.Drop()
b.Drop()
}
return err
}

124
swarm/network/syncdb.go
View file

@ -108,26 +108,26 @@ It is automatically started when syncdb is initialised.
It saves the buffer to db upon receiving quit signal. syncDb#stop()
*/
func (self *syncDb) bufferRead(deliver func(interface{}, chan bool) bool) {
var buffer, db chan interface{} // channels representing the two read modes
func (db *syncDb) bufferRead(deliver func(interface{}, chan bool) bool) {
var buffer, dbChan chan interface{} // channels representing the two read modes
var more bool
var req interface{}
var entry *syncDbEntry
var inBatch, inDb int
batch := new(leveldb.Batch)
var dbSize chan int
quit := self.quit
quit := db.quit
counterValue := make([]byte, 8)
// counter is used for keeping the items in order, persisted to db
// start counter where db was at, 0 if not found
data, err := self.db.Get(self.counterKey)
data, err := db.db.Get(db.counterKey)
var counter uint64
if err == nil {
counter = binary.BigEndian.Uint64(data)
log.Trace(fmt.Sprintf("syncDb[%v/%v] - counter read from db at %v", self.key.Log(), self.priority, counter))
log.Trace(fmt.Sprintf("syncDb[%v/%v] - counter read from db at %v", db.key.Log(), db.priority, counter))
} else {
log.Trace(fmt.Sprintf("syncDb[%v/%v] - counter starts at %v", self.key.Log(), self.priority, counter))
log.Trace(fmt.Sprintf("syncDb[%v/%v] - counter starts at %v", db.key.Log(), db.priority, counter))
}
LOOP:
@ -139,58 +139,58 @@ LOOP:
// deliver request : this is blocking on network write so
// it is passed the quit channel as argument, so that it returns
// if syncdb is stopped. In this case we need to save the item to the db
more = deliver(req, self.quit)
more = deliver(req, db.quit)
if !more {
log.Debug(fmt.Sprintf("syncDb[%v/%v] quit: switching to db. session tally (db/total): %v/%v", self.key.Log(), self.priority, self.dbTotal, self.total))
log.Debug(fmt.Sprintf("syncDb[%v/%v] quit: switching to db. session tally (db/total): %v/%v", db.key.Log(), db.priority, db.dbTotal, db.total))
// received quit signal, save request currently waiting delivery
// by switching to db mode and closing the buffer
buffer = nil
db = self.buffer
close(db)
dbChan = db.buffer
close(dbChan)
quit = nil // needs to block the quit case in select
break // break from select, this item will be written to the db
}
self.total++
log.Trace(fmt.Sprintf("syncDb[%v/%v] deliver (db/total): %v/%v", self.key.Log(), self.priority, self.dbTotal, self.total))
db.total++
log.Trace(fmt.Sprintf("syncDb[%v/%v] deliver (db/total): %v/%v", db.key.Log(), db.priority, db.dbTotal, db.total))
// by the time deliver returns, there were new writes to the buffer
// if buffer contention is detected, switch to db mode which drains
// the buffer so no process will block on pushing store requests
if len(buffer) == cap(buffer) {
log.Debug(fmt.Sprintf("syncDb[%v/%v] buffer full %v: switching to db. session tally (db/total): %v/%v", self.key.Log(), self.priority, cap(buffer), self.dbTotal, self.total))
log.Debug(fmt.Sprintf("syncDb[%v/%v] buffer full %v: switching to db. session tally (db/total): %v/%v", db.key.Log(), db.priority, cap(buffer), db.dbTotal, db.total))
buffer = nil
db = self.buffer
dbChan = db.buffer
}
continue LOOP
// incoming entry to put into db
case req, more = <-db:
case req, more = <-dbChan:
if !more {
// only if quit is called, saved all the buffer
binary.BigEndian.PutUint64(counterValue, counter)
batch.Put(self.counterKey, counterValue) // persist counter in batch
self.writeSyncBatch(batch) // save batch
log.Trace(fmt.Sprintf("syncDb[%v/%v] quitting: save current batch to db", self.key.Log(), self.priority))
batch.Put(db.counterKey, counterValue) // persist counter in batch
db.writeSyncBatch(batch) // save batch
log.Trace(fmt.Sprintf("syncDb[%v/%v] quitting: save current batch to db", db.key.Log(), db.priority))
break LOOP
}
self.dbTotal++
self.total++
db.dbTotal++
db.total++
// otherwise break after select
case dbSize = <-self.batch:
case dbSize = <-db.batch:
// explicit request for batch
if inBatch == 0 && quit != nil {
// there was no writes since the last batch so db depleted
// switch to buffer mode
log.Debug(fmt.Sprintf("syncDb[%v/%v] empty db: switching to buffer", self.key.Log(), self.priority))
db = nil
buffer = self.buffer
log.Debug(fmt.Sprintf("syncDb[%v/%v] empty db: switching to buffer", db.key.Log(), db.priority))
dbChan = nil
buffer = db.buffer
dbSize <- 0 // indicates to 'caller' that batch has been written
inDb = 0
continue LOOP
}
binary.BigEndian.PutUint64(counterValue, counter)
batch.Put(self.counterKey, counterValue)
log.Debug(fmt.Sprintf("syncDb[%v/%v] write batch %v/%v - %x - %x", self.key.Log(), self.priority, inBatch, counter, self.counterKey, counterValue))
batch = self.writeSyncBatch(batch)
batch.Put(db.counterKey, counterValue)
log.Debug(fmt.Sprintf("syncDb[%v/%v] write batch %v/%v - %x - %x", db.key.Log(), db.priority, inBatch, counter, db.counterKey, counterValue))
batch = db.writeSyncBatch(batch)
dbSize <- inBatch // indicates to 'caller' that batch has been written
inBatch = 0
continue LOOP
@ -198,45 +198,45 @@ LOOP:
// closing syncDb#quit channel is used to signal to all goroutines to quit
case <-quit:
// need to save backlog, so switch to db mode
db = self.buffer
dbChan = db.buffer
buffer = nil
quit = nil
log.Trace(fmt.Sprintf("syncDb[%v/%v] quitting: save buffer to db", self.key.Log(), self.priority))
close(db)
log.Trace(fmt.Sprintf("syncDb[%v/%v] quitting: save buffer to db", db.key.Log(), db.priority))
close(dbChan)
continue LOOP
}
// only get here if we put req into db
entry, err = self.newSyncDbEntry(req, counter)
entry, err = db.newSyncDbEntry(req, counter)
if err != nil {
log.Warn(fmt.Sprintf("syncDb[%v/%v] saving request %v (#%v/%v) failed: %v", self.key.Log(), self.priority, req, inBatch, inDb, err))
log.Warn(fmt.Sprintf("syncDb[%v/%v] saving request %v (#%v/%v) failed: %v", db.key.Log(), db.priority, req, inBatch, inDb, err))
continue LOOP
}
batch.Put(entry.key, entry.val)
log.Trace(fmt.Sprintf("syncDb[%v/%v] to batch %v '%v' (#%v/%v/%v)", self.key.Log(), self.priority, req, entry, inBatch, inDb, counter))
log.Trace(fmt.Sprintf("syncDb[%v/%v] to batch %v '%v' (#%v/%v/%v)", db.key.Log(), db.priority, req, entry, inBatch, inDb, counter))
// if just switched to db mode and not quitting, then launch dbRead
// in a parallel go routine to send deliveries from db
if inDb == 0 && quit != nil {
log.Trace(fmt.Sprintf("syncDb[%v/%v] start dbRead", self.key.Log(), self.priority))
go self.dbRead(true, counter, deliver)
log.Trace(fmt.Sprintf("syncDb[%v/%v] start dbRead", db.key.Log(), db.priority))
go db.dbRead(true, counter, deliver)
}
inDb++
inBatch++
counter++
// need to save the batch if it gets too large (== dbBatchSize)
if inBatch%int(self.dbBatchSize) == 0 {
batch = self.writeSyncBatch(batch)
if inBatch%int(db.dbBatchSize) == 0 {
batch = db.writeSyncBatch(batch)
}
}
log.Info(fmt.Sprintf("syncDb[%v:%v]: saved %v keys (saved counter at %v)", self.key.Log(), self.priority, inBatch, counter))
close(self.done)
log.Info(fmt.Sprintf("syncDb[%v:%v]: saved %v keys (saved counter at %v)", db.key.Log(), db.priority, inBatch, counter))
close(db.done)
}
// writes the batch to the db and returns a new batch object
func (self *syncDb) writeSyncBatch(batch *leveldb.Batch) *leveldb.Batch {
err := self.db.Write(batch)
func (db *syncDb) writeSyncBatch(batch *leveldb.Batch) *leveldb.Batch {
err := db.db.Write(batch)
if err != nil {
log.Warn(fmt.Sprintf("syncDb[%v/%v] saving batch to db failed: %v", self.key.Log(), self.priority, err))
log.Warn(fmt.Sprintf("syncDb[%v/%v] saving batch to db failed: %v", db.key.Log(), db.priority, err))
return batch
}
return new(leveldb.Batch)
@ -247,8 +247,8 @@ type syncDbEntry struct {
key, val []byte
}
func (self syncDbEntry) String() string {
return fmt.Sprintf("key: %x, value: %x", self.key, self.val)
func (entry syncDbEntry) String() string {
return fmt.Sprintf("key: %x, value: %x", entry.key, entry.val)
}
/*
@ -272,9 +272,9 @@ dbRead needs a boolean to indicate if on first round all the historical
record is synced. Second argument to indicate current db counter
The third is the function to apply
*/
func (self *syncDb) dbRead(useBatches bool, counter uint64, fun func(interface{}, chan bool) bool) {
func (db *syncDb) dbRead(useBatches bool, counter uint64, fun func(interface{}, chan bool) bool) {
key := make([]byte, 42)
copy(key, self.start)
copy(key, db.start)
binary.BigEndian.PutUint64(key[34:], counter)
var batches, n, cnt, total int
var more bool
@ -290,8 +290,8 @@ func (self *syncDb) dbRead(useBatches bool, counter uint64, fun func(interface{}
// so that loop is not blocking while delivering
// only relevant if cnt is large
select {
case self.batch <- batchSizes:
case <-self.quit:
case db.batch <- batchSizes:
case <-db.quit:
return
}
// wait for the write to finish and get the item count in the next batch
@ -302,31 +302,31 @@ func (self *syncDb) dbRead(useBatches bool, counter uint64, fun func(interface{}
return
}
}
it = self.db.NewIterator()
it = db.db.NewIterator()
it.Seek(key)
if !it.Valid() {
copy(key, self.start)
copy(key, db.start)
useBatches = true
continue
}
del = new(leveldb.Batch)
log.Trace(fmt.Sprintf("syncDb[%v/%v]: new iterator: %x (batch %v, count %v)", self.key.Log(), self.priority, key, batches, cnt))
log.Trace(fmt.Sprintf("syncDb[%v/%v]: new iterator: %x (batch %v, count %v)", db.key.Log(), db.priority, key, batches, cnt))
for n = 0; !useBatches || n < cnt; it.Next() {
copy(key, it.Key())
if len(key) == 0 || key[0] != 0 {
copy(key, self.start)
copy(key, db.start)
useBatches = true
break
}
val := make([]byte, 40)
copy(val, it.Value())
entry = &syncDbEntry{key, val}
// log.Trace(fmt.Sprintf("syncDb[%v/%v] - %v, batches: %v, total: %v, session total from db: %v/%v", self.key.Log(), self.priority, self.key.Log(), batches, total, self.dbTotal, self.total))
more = fun(entry, self.quit)
// log.Trace(fmt.Sprintf("syncDb[%v/%v] - %v, batches: %v, total: %v, session total from db: %v/%v", db.key.Log(), db.priority, db.key.Log(), batches, total, db.dbTotal, db.total))
more = fun(entry, db.quit)
if !more {
// quit received when waiting to deliver entry, the entry will not be deleted
log.Trace(fmt.Sprintf("syncDb[%v/%v] batch %v quit after %v/%v items", self.key.Log(), self.priority, batches, n, cnt))
log.Trace(fmt.Sprintf("syncDb[%v/%v] batch %v quit after %v/%v items", db.key.Log(), db.priority, batches, n, cnt))
break
}
// since subsequent batches of the same db session are indexed incrementally
@ -336,22 +336,22 @@ func (self *syncDb) dbRead(useBatches bool, counter uint64, fun func(interface{}
n++
total++
}
log.Debug(fmt.Sprintf("syncDb[%v/%v] - db session closed, batches: %v, total: %v, session total from db: %v/%v", self.key.Log(), self.priority, batches, total, self.dbTotal, self.total))
self.db.Write(del) // this could be async called only when db is idle
log.Debug(fmt.Sprintf("syncDb[%v/%v] - db session closed, batches: %v, total: %v, session total from db: %v/%v", db.key.Log(), db.priority, batches, total, db.dbTotal, db.total))
db.db.Write(del) // this could be async called only when db is idle
it.Release()
}
}
//
func (self *syncDb) stop() {
close(self.quit)
<-self.done
func (db *syncDb) stop() {
close(db.quit)
<-db.done
}
// calculate a dbkey for the request, for the db to work
// see syncdb for db key structure
// polimorphic: accepted types, see syncer#addRequest
func (self *syncDb) newSyncDbEntry(req interface{}, counter uint64) (entry *syncDbEntry, err error) {
func (db *syncDb) newSyncDbEntry(req interface{}, counter uint64) (entry *syncDbEntry, err error) {
var key storage.Key
var chunk *storage.Chunk
var id uint64
@ -377,7 +377,7 @@ func (self *syncDb) newSyncDbEntry(req interface{}, counter uint64) (entry *sync
dbval := make([]byte, 40)
// encode key
copy(dbkey[:], self.start[:34]) // db peer
copy(dbkey[:], db.start[:34]) // db peer
binary.BigEndian.PutUint64(dbkey[34:], counter)
// encode value
copy(dbval, key[:])

64
swarm/network/syncdb_test.go
View file

@ -53,43 +53,43 @@ func newTestSyncDb(priority, bufferSize, batchSize int, dbdir string, t *testing
}
dbdir = tmp
}
db, err := storage.NewLDBDatabase(filepath.Join(dbdir, "requestdb"))
database, err := storage.NewLDBDatabase(filepath.Join(dbdir, "requestdb"))
if err != nil {
t.Fatalf("unable to create db: %v", err)
}
self := &testSyncDb{
db := &testSyncDb{
fromDb: make(chan bool),
dbdir: dbdir,
t: t,
}
h := crypto.Keccak256Hash([]byte{0})
key := storage.Key(h[:])
self.syncDb = newSyncDb(db, key, uint(priority), uint(bufferSize), uint(batchSize), self.deliver)
db.syncDb = newSyncDb(database, key, uint(priority), uint(bufferSize), uint(batchSize), db.deliver)
// kick off db iterator right away, if no items on db this will allow
// reading from the buffer
return self
return db
}
func (self *testSyncDb) close() {
self.db.Close()
os.RemoveAll(self.dbdir)
func (db *testSyncDb) close() {
db.db.Close()
os.RemoveAll(db.dbdir)
}
func (self *testSyncDb) push(n int) {
func (db *testSyncDb) push(n int) {
for i := 0; i < n; i++ {
self.buffer <- storage.Key(crypto.Keccak256([]byte{byte(self.c)}))
self.sent = append(self.sent, self.c)
self.c++
db.buffer <- storage.Key(crypto.Keccak256([]byte{byte(db.c)}))
db.sent = append(db.sent, db.c)
db.c++
}
log.Debug(fmt.Sprintf("pushed %v requests", n))
}
func (self *testSyncDb) draindb() {
it := self.db.NewIterator()
func (db *testSyncDb) draindb() {
it := db.db.NewIterator()
defer it.Release()
for {
it.Seek(self.start)
it.Seek(db.start)
if !it.Valid() {
return
}
@ -98,44 +98,44 @@ func (self *testSyncDb) draindb() {
return
}
it.Release()
it = self.db.NewIterator()
it = db.db.NewIterator()
}
}
func (self *testSyncDb) deliver(req interface{}, quit chan bool) bool {
_, db := req.(*syncDbEntry)
func (db *testSyncDb) deliver(req interface{}, quit chan bool) bool {
_, entry := req.(*syncDbEntry)
key, _, _, _, err := parseRequest(req)
if err != nil {
self.t.Fatalf("unexpected error of key %v: %v", key, err)
db.t.Fatalf("unexpected error of key %v: %v", key, err)
}
self.delivered = append(self.delivered, key)
db.delivered = append(db.delivered, key)
select {
case self.fromDb <- db:
case db.fromDb <- entry:
return true
case <-quit:
return false
}
}
func (self *testSyncDb) expect(n int, db bool) {
func (db *testSyncDb) expect(n int, b bool) {
var ok bool
// for n items
for i := 0; i < n; i++ {
ok = <-self.fromDb
if self.at+1 > len(self.delivered) {
self.t.Fatalf("expected %v, got %v", self.at+1, len(self.delivered))
ok = <-db.fromDb
if db.at+1 > len(db.delivered) {
db.t.Fatalf("expected %v, got %v", db.at+1, len(db.delivered))
}
if len(self.sent) > self.at && !bytes.Equal(crypto.Keccak256([]byte{byte(self.sent[self.at])}), self.delivered[self.at]) {
self.t.Fatalf("expected delivery %v/%v/%v to be hash of %v, from db: %v = %v", i, n, self.at, self.sent[self.at], ok, db)
log.Debug(fmt.Sprintf("%v/%v/%v to be hash of %v, from db: %v = %v", i, n, self.at, self.sent[self.at], ok, db))
if len(db.sent) > db.at && !bytes.Equal(crypto.Keccak256([]byte{byte(db.sent[db.at])}), db.delivered[db.at]) {
db.t.Fatalf("expected delivery %v/%v/%v to be hash of %v, from db: %v = %v", i, n, db.at, db.sent[db.at], ok, db)
log.Debug(fmt.Sprintf("%v/%v/%v to be hash of %v, from db: %v = %v", i, n, db.at, db.sent[db.at], ok, db))
}
if !ok && db {
self.t.Fatalf("expected delivery %v/%v/%v from db", i, n, self.at)
if !ok && b {
db.t.Fatalf("expected delivery %v/%v/%v from db", i, n, db.at)
}
if ok && !db {
self.t.Fatalf("expected delivery %v/%v/%v from cache", i, n, self.at)
if ok && !b {
db.t.Fatalf("expected delivery %v/%v/%v from cache", i, n, db.at)
}
self.at++
db.at++
}
}

258
swarm/network/syncer.go
View file

@ -77,13 +77,13 @@ func NewDbAccess(loc *storage.LocalStore) *DbAccess {
}
// to obtain the chunks from key or request db entry only
func (self *DbAccess) get(key storage.Key) (*storage.Chunk, error) {
return self.loc.Get(key)
func (access *DbAccess) get(key storage.Key) (*storage.Chunk, error) {
return access.loc.Get(key)
}
// current storage counter of chunk db
func (self *DbAccess) counter() uint64 {
return self.db.Counter()
func (access *DbAccess) counter() uint64 {
return access.db.Counter()
}
// implemented by dbStoreSyncIterator
@ -92,28 +92,28 @@ type keyIterator interface {
}
// generator function for iteration by address range and storage counter
func (self *DbAccess) iterator(s *syncState) keyIterator {
it, err := self.db.NewSyncIterator(*(s.DbSyncState))
func (access *DbAccess) iterator(s *syncState) keyIterator {
it, err := access.db.NewSyncIterator(*(s.DbSyncState))
if err != nil {
return nil
}
return keyIterator(it)
}
func (self syncState) String() string {
if self.Synced {
func (state syncState) String() string {
if state.Synced {
return fmt.Sprintf(
"session started at: %v, last seen at: %v, latest key: %v",
self.SessionAt, self.LastSeenAt,
self.Latest.Log(),
state.SessionAt, state.LastSeenAt,
state.Latest.Log(),
)
} else {
return fmt.Sprintf(
"address: %v-%v, index: %v-%v, session started at: %v, last seen at: %v, latest key: %v",
self.Start.Log(), self.Stop.Log(),
self.First, self.Last,
self.SessionAt, self.LastSeenAt,
self.Latest.Log(),
state.Start.Log(), state.Stop.Log(),
state.First, state.Last,
state.SessionAt, state.LastSeenAt,
state.Latest.Log(),
)
}
}
@ -145,8 +145,8 @@ func NewDefaultSyncParams() *SyncParams {
//this can only finally be set after all config options (file, cmd line, env vars)
//have been evaluated
func (self *SyncParams) Init(path string) {
self.RequestDbPath = filepath.Join(path, "requests")
func (params *SyncParams) Init(path string) {
params.RequestDbPath = filepath.Join(path, "requests")
}
// syncer is the agent that manages content distribution/storage replication/chunk storeRequest forwarding
@ -191,7 +191,7 @@ func newSyncer(
keyBufferSize := params.KeyBufferSize
dbBatchSize := params.RequestDbBatchSize
self := &syncer{
syncer := &syncer{
syncF: syncF,
key: remotekey,
dbAccess: dbAccess,
@ -207,22 +207,22 @@ func newSyncer(
// initialising
for i := 0; i < priorities; i++ {
self.keys[i] = make(chan interface{}, keyBufferSize)
self.deliveries[i] = make(chan *storeRequestMsgData)
syncer.keys[i] = make(chan interface{}, keyBufferSize)
syncer.deliveries[i] = make(chan *storeRequestMsgData)
// initialise a syncdb instance for each priority queue
self.queues[i] = newSyncDb(db, remotekey, uint(i), syncBufferSize, dbBatchSize, self.deliver(uint(i)))
syncer.queues[i] = newSyncDb(db, remotekey, uint(i), syncBufferSize, dbBatchSize, syncer.deliver(uint(i)))
}
log.Info(fmt.Sprintf("syncer started: %v", state))
// launch chunk delivery service
go self.syncDeliveries()
go syncer.syncDeliveries()
// launch sync task manager
if self.syncF() {
go self.sync()
if syncer.syncF() {
go syncer.sync()
}
// process unsynced keys to broadcast
go self.syncUnsyncedKeys()
go syncer.syncUnsyncedKeys()
return self, nil
return syncer, nil
}
// metadata serialisation
@ -266,21 +266,21 @@ func decodeSync(meta *json.RawMessage) (*syncState, error) {
sync is called from the syncer constructor and is not supposed to be used externally
*/
func (self *syncer) sync() {
state := self.state
func (s *syncer) sync() {
state := s.state
// sync finished
defer close(self.syncStates)
defer close(s.syncStates)
// 0. first replay stale requests from request db
if state.SessionAt == 0 {
log.Debug(fmt.Sprintf("syncer[%v]: nothing to sync", self.key.Log()))
log.Debug(fmt.Sprintf("syncer[%v]: nothing to sync", s.key.Log()))
return
}
log.Debug(fmt.Sprintf("syncer[%v]: start replaying stale requests from request db", self.key.Log()))
log.Debug(fmt.Sprintf("syncer[%v]: start replaying stale requests from request db", s.key.Log()))
for p := priorities - 1; p >= 0; p-- {
self.queues[p].dbRead(false, 0, self.replay())
s.queues[p].dbRead(false, 0, s.replay())
}
log.Debug(fmt.Sprintf("syncer[%v]: done replaying stale requests from request db", self.key.Log()))
log.Debug(fmt.Sprintf("syncer[%v]: done replaying stale requests from request db", s.key.Log()))
// unless peer is synced sync unfinished history beginning on
if !state.Synced {
@ -289,9 +289,9 @@ func (self *syncer) sync() {
if !storage.IsZeroKey(state.Latest) {
// 1. there is unfinished earlier sync
state.Start = state.Latest
log.Debug(fmt.Sprintf("syncer[%v]: start syncronising backlog (unfinished sync: %v)", self.key.Log(), state))
log.Debug(fmt.Sprintf("syncer[%v]: start syncronising backlog (unfinished sync: %v)", s.key.Log(), state))
// blocks while the entire history upto state is synced
self.syncState(state)
s.syncState(state)
if state.Last < state.SessionAt {
state.First = state.Last + 1
}
@ -301,8 +301,8 @@ func (self *syncer) sync() {
// 2. sync up to last disconnect1
if state.First < state.LastSeenAt {
state.Last = state.LastSeenAt
log.Debug(fmt.Sprintf("syncer[%v]: start syncronising history upto last disconnect at %v: %v", self.key.Log(), state.LastSeenAt, state))
self.syncState(state)
log.Debug(fmt.Sprintf("syncer[%v]: start syncronising history upto last disconnect at %v: %v", s.key.Log(), state.LastSeenAt, state))
s.syncState(state)
state.First = state.LastSeenAt
}
state.Latest = storage.ZeroKey
@ -316,28 +316,28 @@ func (self *syncer) sync() {
// if there have been new chunks since last session
if state.LastSeenAt < state.SessionAt {
state.Last = state.SessionAt
log.Debug(fmt.Sprintf("syncer[%v]: start syncronising history since last disconnect at %v up until session start at %v: %v", self.key.Log(), state.LastSeenAt, state.SessionAt, state))
log.Debug(fmt.Sprintf("syncer[%v]: start syncronising history since last disconnect at %v up until session start at %v: %v", s.key.Log(), state.LastSeenAt, state.SessionAt, state))
// blocks until state syncing is finished
self.syncState(state)
s.syncState(state)
}
log.Info(fmt.Sprintf("syncer[%v]: syncing all history complete", self.key.Log()))
log.Info(fmt.Sprintf("syncer[%v]: syncing all history complete", s.key.Log()))
}
// wait till syncronised block uptil state is synced
func (self *syncer) syncState(state *syncState) {
self.syncStates <- state
func (s *syncer) syncState(state *syncState) {
s.syncStates <- state
select {
case <-state.synced:
case <-self.quit:
case <-s.quit:
}
}
// stop quits both request processor and saves the request cache to disk
func (self *syncer) stop() {
close(self.quit)
log.Trace(fmt.Sprintf("syncer[%v]: stop and save sync request db backlog", self.key.Log()))
for _, db := range self.queues {
func (s *syncer) stop() {
close(s.quit)
log.Trace(fmt.Sprintf("syncer[%v]: stop and save sync request db backlog", s.key.Log()))
for _, db := range s.queues {
db.stop()
}
}
@ -348,11 +348,11 @@ type syncRequest struct {
Priority uint
}
func (self *syncRequest) String() string {
return fmt.Sprintf("<Key: %v, Priority: %v>", self.Key.Log(), self.Priority)
func (req *syncRequest) String() string {
return fmt.Sprintf("<Key: %v, Priority: %v>", req.Key.Log(), req.Priority)
}
func (self *syncer) newSyncRequest(req interface{}, p int) (*syncRequest, error) {
func (s *syncer) newSyncRequest(req interface{}, p int) (*syncRequest, error) {
key, _, _, _, err := parseRequest(req)
// TODO: if req has chunk, it should be put in a cache
// create
@ -366,11 +366,11 @@ func (self *syncer) newSyncRequest(req interface{}, p int) (*syncRequest, error)
// * read is on demand, blocking unless history channel is read
// * accepts sync requests (syncStates) to create new db iterator
// * closes the channel one iteration finishes
func (self *syncer) syncHistory(state *syncState) chan interface{} {
func (s *syncer) syncHistory(state *syncState) chan interface{} {
var n uint
history := make(chan interface{})
log.Debug(fmt.Sprintf("syncer[%v]: syncing history between %v - %v for chunk addresses %v - %v", self.key.Log(), state.First, state.Last, state.Start, state.Stop))
it := self.dbAccess.iterator(state)
log.Debug(fmt.Sprintf("syncer[%v]: syncing history between %v - %v for chunk addresses %v - %v", s.key.Log(), state.First, state.Last, state.Start, state.Stop))
it := s.dbAccess.iterator(state)
if it != nil {
go func() {
// signal end of the iteration ended
@ -385,22 +385,22 @@ func (self *syncer) syncHistory(state *syncState) chan interface{} {
// blocking until history channel is read from
case history <- key:
n++
log.Trace(fmt.Sprintf("syncer[%v]: history: %v (%v keys)", self.key.Log(), key.Log(), n))
log.Trace(fmt.Sprintf("syncer[%v]: history: %v (%v keys)", s.key.Log(), key.Log(), n))
state.Latest = key
case <-self.quit:
case <-s.quit:
return
}
}
log.Debug(fmt.Sprintf("syncer[%v]: finished syncing history between %v - %v for chunk addresses %v - %v (at %v) (chunks = %v)", self.key.Log(), state.First, state.Last, state.Start, state.Stop, state.Latest, n))
log.Debug(fmt.Sprintf("syncer[%v]: finished syncing history between %v - %v for chunk addresses %v - %v (at %v) (chunks = %v)", s.key.Log(), state.First, state.Last, state.Start, state.Stop, state.Latest, n))
}()
}
return history
}
// triggers key syncronisation
func (self *syncer) sendUnsyncedKeys() {
func (s *syncer) sendUnsyncedKeys() {
select {
case self.deliveryRequest <- true:
case s.deliveryRequest <- true:
default:
}
}
@ -411,7 +411,7 @@ func (self *syncer) sendUnsyncedKeys() {
// historical data is used so historical items are lower priority within
// their priority group.
// * Order of historical data is unspecified
func (self *syncer) syncUnsyncedKeys() {
func (s *syncer) syncUnsyncedKeys() {
// send out new
var unsynced []*syncRequest
var more, justSynced bool
@ -419,12 +419,12 @@ func (self *syncer) syncUnsyncedKeys() {
var history chan interface{}
priority := High
keys := self.keys[priority]
keys := s.keys[priority]
var newUnsyncedKeys, deliveryRequest chan bool
keyCounts := make([]int, priorities)
histPrior := self.SyncPriorities[HistoryReq]
syncStates := self.syncStates
state := self.state
histPrior := s.SyncPriorities[HistoryReq]
syncStates := s.syncStates
state := s.state
LOOP:
for {
@ -440,15 +440,15 @@ LOOP:
PRIORITIES:
for priority = High; priority >= 0; priority-- {
// the first priority channel that is non-empty will be assigned to keys
if len(self.keys[priority]) > 0 {
log.Trace(fmt.Sprintf("syncer[%v]: reading request with priority %v", self.key.Log(), priority))
keys = self.keys[priority]
if len(s.keys[priority]) > 0 {
log.Trace(fmt.Sprintf("syncer[%v]: reading request with priority %v", s.key.Log(), priority))
keys = s.keys[priority]
break PRIORITIES
}
log.Trace(fmt.Sprintf("syncer[%v/%v]: queue: [%v, %v, %v]", self.key.Log(), priority, len(self.keys[High]), len(self.keys[Medium]), len(self.keys[Low])))
log.Trace(fmt.Sprintf("syncer[%v/%v]: queue: [%v, %v, %v]", s.key.Log(), priority, len(s.keys[High]), len(s.keys[Medium]), len(s.keys[Low])))
// if the input queue is empty on this level, resort to history if there is any
if uint(priority) == histPrior && history != nil {
log.Trace(fmt.Sprintf("syncer[%v]: reading history for %v", self.key.Log(), self.key))
log.Trace(fmt.Sprintf("syncer[%v]: reading history for %v", s.key.Log(), s.key))
keys = history
break PRIORITIES
}
@ -458,8 +458,8 @@ LOOP:
// if peer ready to receive but nothing to send
if keys == nil && deliveryRequest == nil {
// if no items left and switch to waiting mode
log.Trace(fmt.Sprintf("syncer[%v]: buffers consumed. Waiting", self.key.Log()))
newUnsyncedKeys = self.newUnsyncedKeys
log.Trace(fmt.Sprintf("syncer[%v]: buffers consumed. Waiting", s.key.Log()))
newUnsyncedKeys = s.newUnsyncedKeys
}
// send msg iff
@ -470,48 +470,48 @@ LOOP:
if deliveryRequest == nil &&
(justSynced ||
len(unsynced) > 0 && keys == nil ||
len(unsynced) == int(self.SyncBatchSize)) {
len(unsynced) == int(s.SyncBatchSize)) {
justSynced = false
// listen to requests
deliveryRequest = self.deliveryRequest
deliveryRequest = s.deliveryRequest
newUnsyncedKeys = nil // not care about data until next req comes in
// set sync to current counter
// (all nonhistorical outgoing traffic sheduled and persisted
state.LastSeenAt = self.dbAccess.counter()
state.LastSeenAt = s.dbAccess.counter()
state.Latest = storage.ZeroKey
log.Trace(fmt.Sprintf("syncer[%v]: sending %v", self.key.Log(), unsynced))
log.Trace(fmt.Sprintf("syncer[%v]: sending %v", s.key.Log(), unsynced))
// send the unsynced keys
stateCopy := *state
err := self.unsyncedKeys(unsynced, &stateCopy)
err := s.unsyncedKeys(unsynced, &stateCopy)
if err != nil {
log.Warn(fmt.Sprintf("syncer[%v]: unable to send unsynced keys: %v", self.key.Log(), err))
log.Warn(fmt.Sprintf("syncer[%v]: unable to send unsynced keys: %v", s.key.Log(), err))
}
self.state = state
log.Debug(fmt.Sprintf("syncer[%v]: --> %v keys sent: (total: %v (%v), history: %v), sent sync state: %v", self.key.Log(), len(unsynced), keyCounts, keyCount, historyCnt, stateCopy))
s.state = state
log.Debug(fmt.Sprintf("syncer[%v]: --> %v keys sent: (total: %v (%v), history: %v), sent sync state: %v", s.key.Log(), len(unsynced), keyCounts, keyCount, historyCnt, stateCopy))
unsynced = nil
keys = nil
}
// process item and add it to the batch
select {
case <-self.quit:
case <-s.quit:
break LOOP
case req, more = <-keys:
if keys == history && !more {
log.Trace(fmt.Sprintf("syncer[%v]: syncing history segment complete", self.key.Log()))
log.Trace(fmt.Sprintf("syncer[%v]: syncing history segment complete", s.key.Log()))
// history channel is closed, waiting for new state (called from sync())
syncStates = self.syncStates
syncStates = s.syncStates
state.Synced = true // this signals that the current segment is complete
select {
case state.synced <- false:
case <-self.quit:
case <-s.quit:
break LOOP
}
justSynced = true
history = nil
}
case <-deliveryRequest:
log.Trace(fmt.Sprintf("syncer[%v]: peer ready to receive", self.key.Log()))
log.Trace(fmt.Sprintf("syncer[%v]: peer ready to receive", s.key.Log()))
// this 1 cap channel can wake up the loop
// signaling that peer is ready to receive unsynced Keys
@ -519,23 +519,23 @@ LOOP:
deliveryRequest = nil
case <-newUnsyncedKeys:
log.Trace(fmt.Sprintf("syncer[%v]: new unsynced keys available", self.key.Log()))
log.Trace(fmt.Sprintf("syncer[%v]: new unsynced keys available", s.key.Log()))
// this 1 cap channel can wake up the loop
// signals that data is available to send if peer is ready to receive
newUnsyncedKeys = nil
keys = self.keys[High]
keys = s.keys[High]
case state, more = <-syncStates:
// this resets the state
if !more {
state = self.state
log.Trace(fmt.Sprintf("syncer[%v]: (priority %v) syncing complete upto %v)", self.key.Log(), priority, state))
state = s.state
log.Trace(fmt.Sprintf("syncer[%v]: (priority %v) syncing complete upto %v)", s.key.Log(), priority, state))
state.Synced = true
syncStates = nil
} else {
log.Trace(fmt.Sprintf("syncer[%v]: (priority %v) syncing history upto %v priority %v)", self.key.Log(), priority, state, histPrior))
log.Trace(fmt.Sprintf("syncer[%v]: (priority %v) syncing history upto %v priority %v)", s.key.Log(), priority, state, histPrior))
state.Synced = false
history = self.syncHistory(state)
history = s.syncHistory(state)
// only one history at a time, only allow another one once the
// history channel is closed
syncStates = nil
@ -545,19 +545,19 @@ LOOP:
continue LOOP
}
log.Trace(fmt.Sprintf("syncer[%v]: (priority %v) added to unsynced keys: %v", self.key.Log(), priority, req))
log.Trace(fmt.Sprintf("syncer[%v]: (priority %v) added to unsynced keys: %v", s.key.Log(), priority, req))
keyCounts[priority]++
keyCount++
if keys == history {
log.Trace(fmt.Sprintf("syncer[%v]: (priority %v) history item %v (synced = %v)", self.key.Log(), priority, req, state.Synced))
log.Trace(fmt.Sprintf("syncer[%v]: (priority %v) history item %v (synced = %v)", s.key.Log(), priority, req, state.Synced))
historyCnt++
}
if sreq, err := self.newSyncRequest(req, priority); err == nil {
if sreq, err := s.newSyncRequest(req, priority); err == nil {
// extract key from req
log.Trace(fmt.Sprintf("syncer[%v]: (priority %v): request %v (synced = %v)", self.key.Log(), priority, req, state.Synced))
log.Trace(fmt.Sprintf("syncer[%v]: (priority %v): request %v (synced = %v)", s.key.Log(), priority, req, state.Synced))
unsynced = append(unsynced, sreq)
} else {
log.Warn(fmt.Sprintf("syncer[%v]: (priority %v): error creating request for %v: %v)", self.key.Log(), priority, req, err))
log.Warn(fmt.Sprintf("syncer[%v]: (priority %v): error creating request for %v: %v)", s.key.Log(), priority, req, err))
}
}
@ -566,7 +566,7 @@ LOOP:
// delivery loop
// takes into account priority, send store Requests with chunk (delivery)
// idle blocking if no new deliveries in any of the queues
func (self *syncer) syncDeliveries() {
func (s *syncer) syncDeliveries() {
var req *storeRequestMsgData
p := High
var deliveries chan *storeRequestMsgData
@ -577,7 +577,7 @@ func (self *syncer) syncDeliveries() {
var total, success uint
for {
deliveries = self.deliveries[p]
deliveries = s.deliveries[p]
select {
case req = <-deliveries:
n[p]++
@ -586,13 +586,13 @@ func (self *syncer) syncDeliveries() {
if p == Low {
// blocking, depletion on all channels, no preference for priority
select {
case req = <-self.deliveries[High]:
case req = <-s.deliveries[High]:
n[High]++
case req = <-self.deliveries[Medium]:
case req = <-s.deliveries[Medium]:
n[Medium]++
case req = <-self.deliveries[Low]:
case req = <-s.deliveries[Low]:
n[Low]++
case <-self.quit:
case <-s.quit:
return
}
p = High
@ -602,20 +602,20 @@ func (self *syncer) syncDeliveries() {
}
}
total++
msg, err = self.newStoreRequestMsgData(req)
msg, err = s.newStoreRequestMsgData(req)
if err != nil {
log.Warn(fmt.Sprintf("syncer[%v]: failed to create store request for %v: %v", self.key.Log(), req, err))
log.Warn(fmt.Sprintf("syncer[%v]: failed to create store request for %v: %v", s.key.Log(), req, err))
} else {
err = self.store(msg)
err = s.store(msg)
if err != nil {
log.Warn(fmt.Sprintf("syncer[%v]: failed to deliver %v: %v", self.key.Log(), req, err))
log.Warn(fmt.Sprintf("syncer[%v]: failed to deliver %v: %v", s.key.Log(), req, err))
} else {
success++
log.Trace(fmt.Sprintf("syncer[%v]: %v successfully delivered", self.key.Log(), req))
log.Trace(fmt.Sprintf("syncer[%v]: %v successfully delivered", s.key.Log(), req))
}
}
if total%self.SyncBatchSize == 0 {
log.Debug(fmt.Sprintf("syncer[%v]: deliver Total: %v, Success: %v, High: %v/%v, Medium: %v/%v, Low %v/%v", self.key.Log(), total, success, c[High], n[High], c[Medium], n[Medium], c[Low], n[Low]))
if total%s.SyncBatchSize == 0 {
log.Debug(fmt.Sprintf("syncer[%v]: deliver Total: %v, Success: %v, High: %v/%v, Medium: %v/%v, Low %v/%v", s.key.Log(), total, success, c[High], n[High], c[Medium], n[Medium], c[Low], n[Low]))
}
}
}
@ -635,28 +635,28 @@ func (self *syncer) syncDeliveries() {
If sync mode is off then, requests are directly sent to deliveries
*/
func (self *syncer) addRequest(req interface{}, ty int) {
func (s *syncer) addRequest(req interface{}, ty int) {
// retrieve priority for request type name int8
priority := self.SyncPriorities[ty]
priority := s.SyncPriorities[ty]
// sync mode for this type ON
if self.syncF() || ty == DeliverReq {
if self.SyncModes[ty] {
self.addKey(req, priority, self.quit)
if s.syncF() || ty == DeliverReq {
if s.SyncModes[ty] {
s.addKey(req, priority, s.quit)
} else {
self.addDelivery(req, priority, self.quit)
s.addDelivery(req, priority, s.quit)
}
}
}
// addKey queues sync request for sync confirmation with given priority
// ie the key will go out in an unsyncedKeys message
func (self *syncer) addKey(req interface{}, priority uint, quit chan bool) bool {
func (s *syncer) addKey(req interface{}, priority uint, quit chan bool) bool {
select {
case self.keys[priority] <- req:
case s.keys[priority] <- req:
// this wakes up the unsynced keys loop if idle
select {
case self.newUnsyncedKeys <- true:
case s.newUnsyncedKeys <- true:
default:
}
return true
@ -668,9 +668,9 @@ func (self *syncer) addKey(req interface{}, priority uint, quit chan bool) bool
// addDelivery queues delivery request for with given priority
// ie the chunk will be delivered ASAP mod priority queueing handled by syncdb
// requests are persisted across sessions for correct sync
func (self *syncer) addDelivery(req interface{}, priority uint, quit chan bool) bool {
func (s *syncer) addDelivery(req interface{}, priority uint, quit chan bool) bool {
select {
case self.queues[priority].buffer <- req:
case s.queues[priority].buffer <- req:
return true
case <-quit:
return false
@ -679,14 +679,14 @@ func (self *syncer) addDelivery(req interface{}, priority uint, quit chan bool)
// doDelivery delivers the chunk for the request with given priority
// without queuing
func (self *syncer) doDelivery(req interface{}, priority uint, quit chan bool) bool {
msgdata, err := self.newStoreRequestMsgData(req)
func (s *syncer) doDelivery(req interface{}, priority uint, quit chan bool) bool {
msgdata, err := s.newStoreRequestMsgData(req)
if err != nil {
log.Warn(fmt.Sprintf("unable to deliver request %v: %v", msgdata, err))
return false
}
select {
case self.deliveries[priority] <- msgdata:
case s.deliveries[priority] <- msgdata:
return true
case <-quit:
return false
@ -695,9 +695,9 @@ func (self *syncer) doDelivery(req interface{}, priority uint, quit chan bool) b
// returns the delivery function for given priority
// passed on to syncDb
func (self *syncer) deliver(priority uint) func(req interface{}, quit chan bool) bool {
func (s *syncer) deliver(priority uint) func(req interface{}, quit chan bool) bool {
return func(req interface{}, quit chan bool) bool {
return self.doDelivery(req, priority, quit)
return s.doDelivery(req, priority, quit)
}
}
@ -705,17 +705,17 @@ func (self *syncer) deliver(priority uint) func(req interface{}, quit chan bool)
// depending on sync mode settings for BacklogReq,
// re play of request db backlog sends items via confirmation
// or directly delivers
func (self *syncer) replay() func(req interface{}, quit chan bool) bool {
sync := self.SyncModes[BacklogReq]
priority := self.SyncPriorities[BacklogReq]
func (s *syncer) replay() func(req interface{}, quit chan bool) bool {
sync := s.SyncModes[BacklogReq]
priority := s.SyncPriorities[BacklogReq]
// sync mode for this type ON
if sync {
return func(req interface{}, quit chan bool) bool {
return self.addKey(req, priority, quit)
return s.addKey(req, priority, quit)
}
} else {
return func(req interface{}, quit chan bool) bool {
return self.doDelivery(req, priority, quit)
return s.doDelivery(req, priority, quit)
}
}
@ -723,7 +723,7 @@ func (self *syncer) replay() func(req interface{}, quit chan bool) bool {
// given a request, extends it to a full storeRequestMsgData
// polimorphic: see addRequest for the types accepted
func (self *syncer) newStoreRequestMsgData(req interface{}) (*storeRequestMsgData, error) {
func (s *syncer) newStoreRequestMsgData(req interface{}) (*storeRequestMsgData, error) {
key, id, chunk, sreq, err := parseRequest(req)
if err != nil {
@ -733,7 +733,7 @@ func (self *syncer) newStoreRequestMsgData(req interface{}) (*storeRequestMsgDat
if sreq == nil {
if chunk == nil {
var err error
chunk, err = self.dbAccess.get(key)
chunk, err = s.dbAccess.get(key)
if err != nil {
return nil, err
}

78
swarm/services/swap/swap.go
View file

@ -104,10 +104,10 @@ func NewDefaultSwapParams() *SwapParams {
//this can only finally be set after all config options (file, cmd line, env vars)
//have been evaluated
func (self *SwapParams) Init(contract common.Address, prvkey *ecdsa.PrivateKey) {
func (params *SwapParams) Init(contract common.Address, prvkey *ecdsa.PrivateKey) {
pubkey := &prvkey.PublicKey
self.PayProfile = &PayProfile{
params.PayProfile = &PayProfile{
PublicKey: common.ToHex(crypto.FromECDSAPub(pubkey)),
Contract: contract,
Beneficiary: crypto.PubkeyToAddress(*pubkey),
@ -126,7 +126,7 @@ func (self *SwapParams) Init(contract common.Address, prvkey *ecdsa.PrivateKey)
// n < 0 called when receiving chunks = receiving delivery responses
// OR receiving cheques.
func NewSwap(local *SwapParams, remote *SwapProfile, backend chequebook.Backend, proto swap.Protocol) (self *swap.Swap, err error) {
func NewSwap(local *SwapParams, remote *SwapProfile, backend chequebook.Backend, proto swap.Protocol) (s *swap.Swap, err error) {
var (
ctx = context.TODO()
ok bool
@ -162,19 +162,19 @@ func NewSwap(local *SwapParams, remote *SwapProfile, backend chequebook.Backend,
Buys: out != nil,
Sells: in != nil,
}
self, err = swap.New(local.Params, pm, proto)
s, err = swap.New(local.Params, pm, proto)
if err != nil {
return
}
// remote profile given (first) in handshake
self.SetRemote(remote.Profile)
s.SetRemote(remote.Profile)
var buy, sell string
if self.Buys {
if s.Buys {
buy = "purchase from peer enabled at " + remote.SellAt.String() + " wei/chunk"
} else {
buy = "purchase from peer disabled"
}
if self.Sells {
if s.Sells {
sell = "selling to peer enabled at " + local.SellAt.String() + " wei/chunk"
} else {
sell = "selling to peer disabled"
@ -184,44 +184,44 @@ func NewSwap(local *SwapParams, remote *SwapProfile, backend chequebook.Backend,
return
}
func (self *SwapParams) Chequebook() *chequebook.Chequebook {
defer self.lock.Unlock()
self.lock.Lock()
return self.chbook
func (params *SwapParams) Chequebook() *chequebook.Chequebook {
defer params.lock.Unlock()
params.lock.Lock()
return params.chbook
}
func (self *SwapParams) PrivateKey() *ecdsa.PrivateKey {
return self.privateKey
func (params *SwapParams) PrivateKey() *ecdsa.PrivateKey {
return params.privateKey
}
// func (self *SwapParams) PublicKey() *ecdsa.PublicKey {
// return self.publicKey
// func (params *SwapParams) PublicKey() *ecdsa.PublicKey {
// return params.publicKey
// }
func (self *SwapParams) SetKey(prvkey *ecdsa.PrivateKey) {
self.privateKey = prvkey
self.publicKey = &prvkey.PublicKey
func (params *SwapParams) SetKey(prvkey *ecdsa.PrivateKey) {
params.privateKey = prvkey
params.publicKey = &prvkey.PublicKey
}
// setChequebook(path, backend) wraps the
// chequebook initialiser and sets up autoDeposit to cover spending.
func (self *SwapParams) SetChequebook(ctx context.Context, backend chequebook.Backend, path string) error {
self.lock.Lock()
contract := self.Contract
self.lock.Unlock()
func (params *SwapParams) SetChequebook(ctx context.Context, backend chequebook.Backend, path string) error {
params.lock.Lock()
contract := params.Contract
params.lock.Unlock()
valid, err := chequebook.ValidateCode(ctx, backend, contract)
if err != nil {
return err
} else if valid {
return self.newChequebookFromContract(path, backend)
return params.newChequebookFromContract(path, backend)
}
return self.deployChequebook(ctx, backend, path)
return params.deployChequebook(ctx, backend, path)
}
func (self *SwapParams) deployChequebook(ctx context.Context, backend chequebook.Backend, path string) error {
opts := bind.NewKeyedTransactor(self.privateKey)
opts.Value = self.AutoDepositBuffer
func (params *SwapParams) deployChequebook(ctx context.Context, backend chequebook.Backend, path string) error {
opts := bind.NewKeyedTransactor(params.privateKey)
opts.Value = params.AutoDepositBuffer
opts.Context = ctx
log.Info(fmt.Sprintf("Deploying new chequebook (owner: %v)", opts.From.Hex()))
@ -233,10 +233,10 @@ func (self *SwapParams) deployChequebook(ctx context.Context, backend chequebook
log.Info(fmt.Sprintf("new chequebook deployed at %v (owner: %v)", contract.Hex(), opts.From.Hex()))
// need to save config at this point
self.lock.Lock()
self.Contract = contract
err = self.newChequebookFromContract(path, backend)
self.lock.Unlock()
params.lock.Lock()
params.Contract = contract
err = params.newChequebookFromContract(path, backend)
params.lock.Unlock()
if err != nil {
log.Warn(fmt.Sprintf("error initialising cheque book (owner: %v): %v", opts.From.Hex(), err))
}
@ -265,26 +265,26 @@ func deployChequebookLoop(opts *bind.TransactOpts, backend chequebook.Backend) (
// initialise the chequebook from a persisted json file or create a new one
// caller holds the lock
func (self *SwapParams) newChequebookFromContract(path string, backend chequebook.Backend) error {
hexkey := common.Bytes2Hex(self.Contract.Bytes())
func (params *SwapParams) newChequebookFromContract(path string, backend chequebook.Backend) error {
hexkey := common.Bytes2Hex(params.Contract.Bytes())
err := os.MkdirAll(filepath.Join(path, "chequebooks"), os.ModePerm)
if err != nil {
return fmt.Errorf("unable to create directory for chequebooks: %v", err)
}
chbookpath := filepath.Join(path, "chequebooks", hexkey+".json")
self.chbook, err = chequebook.LoadChequebook(chbookpath, self.privateKey, backend, true)
params.chbook, err = chequebook.LoadChequebook(chbookpath, params.privateKey, backend, true)
if err != nil {
self.chbook, err = chequebook.NewChequebook(chbookpath, self.Contract, self.privateKey, backend)
params.chbook, err = chequebook.NewChequebook(chbookpath, params.Contract, params.privateKey, backend)
if err != nil {
log.Warn(fmt.Sprintf("unable to initialise chequebook (owner: %v): %v", self.owner.Hex(), err))
return fmt.Errorf("unable to initialise chequebook (owner: %v): %v", self.owner.Hex(), err)
log.Warn(fmt.Sprintf("unable to initialise chequebook (owner: %v): %v", params.owner.Hex(), err))
return fmt.Errorf("unable to initialise chequebook (owner: %v): %v", params.owner.Hex(), err)
}
}
self.chbook.AutoDeposit(self.AutoDepositInterval, self.AutoDepositThreshold, self.AutoDepositBuffer)
log.Info(fmt.Sprintf("auto deposit ON for %v -> %v: interval = %v, threshold = %v, buffer = %v)", crypto.PubkeyToAddress(*(self.publicKey)).Hex()[:8], self.Contract.Hex()[:8], self.AutoDepositInterval, self.AutoDepositThreshold, self.AutoDepositBuffer))
params.chbook.AutoDeposit(params.AutoDepositInterval, params.AutoDepositThreshold, params.AutoDepositBuffer)
log.Info(fmt.Sprintf("auto deposit ON for %v -> %v: interval = %v, threshold = %v, buffer = %v)", crypto.PubkeyToAddress(*(params.publicKey)).Hex()[:8], params.Contract.Hex()[:8], params.AutoDepositInterval, params.AutoDepositThreshold, params.AutoDepositBuffer))
return nil
}

144
swarm/services/swap/swap/swap.go
View file

@ -100,153 +100,153 @@ type Payment struct {
}
// swap constructor
func New(local *Params, pm Payment, proto Protocol) (self *Swap, err error) {
func New(local *Params, pm Payment, proto Protocol) (s *Swap, err error) {
self = &Swap{
s = &Swap{
local: local,
Payment: pm,
proto: proto,
}
self.SetParams(local)
s.SetParams(local)
return
}
// entry point for setting remote swap profile (e.g from handshake or other message)
func (self *Swap) SetRemote(remote *Profile) {
defer self.lock.Unlock()
self.lock.Lock()
func (s *Swap) SetRemote(remote *Profile) {
defer s.lock.Unlock()
s.lock.Lock()
self.remote = remote
if self.Sells && (remote.BuyAt.Sign() <= 0 || self.local.SellAt.Sign() <= 0 || remote.BuyAt.Cmp(self.local.SellAt) < 0) {
self.Out.Stop()
self.Sells = false
s.remote = remote
if s.Sells && (remote.BuyAt.Sign() <= 0 || s.local.SellAt.Sign() <= 0 || remote.BuyAt.Cmp(s.local.SellAt) < 0) {
s.Out.Stop()
s.Sells = false
}
if self.Buys && (remote.SellAt.Sign() <= 0 || self.local.BuyAt.Sign() <= 0 || self.local.BuyAt.Cmp(self.remote.SellAt) < 0) {
self.In.Stop()
self.Buys = false
if s.Buys && (remote.SellAt.Sign() <= 0 || s.local.BuyAt.Sign() <= 0 || s.local.BuyAt.Cmp(s.remote.SellAt) < 0) {
s.In.Stop()
s.Buys = false
}
log.Debug(fmt.Sprintf("<%v> remote profile set: pay at: %v, drop at: %v, buy at: %v, sell at: %v", self.proto, remote.PayAt, remote.DropAt, remote.BuyAt, remote.SellAt))
log.Debug(fmt.Sprintf("<%v> remote profile set: pay at: %v, drop at: %v, buy at: %v, sell at: %v", s.proto, remote.PayAt, remote.DropAt, remote.BuyAt, remote.SellAt))
}
// to set strategy dynamically
func (self *Swap) SetParams(local *Params) {
defer self.lock.Unlock()
self.lock.Lock()
self.local = local
self.setParams(local)
func (s *Swap) SetParams(local *Params) {
defer s.lock.Unlock()
s.lock.Lock()
s.local = local
s.setParams(local)
}
// caller holds the lock
func (self *Swap) setParams(local *Params) {
func (s *Swap) setParams(local *Params) {
if self.Sells {
self.In.AutoCash(local.AutoCashInterval, local.AutoCashThreshold)
log.Info(fmt.Sprintf("<%v> set autocash to every %v, max uncashed limit: %v", self.proto, local.AutoCashInterval, local.AutoCashThreshold))
if s.Sells {
s.In.AutoCash(local.AutoCashInterval, local.AutoCashThreshold)
log.Info(fmt.Sprintf("<%v> set autocash to every %v, max uncashed limit: %v", s.proto, local.AutoCashInterval, local.AutoCashThreshold))
} else {
log.Info(fmt.Sprintf("<%v> autocash off (not selling)", self.proto))
log.Info(fmt.Sprintf("<%v> autocash off (not selling)", s.proto))
}
if self.Buys {
self.Out.AutoDeposit(local.AutoDepositInterval, local.AutoDepositThreshold, local.AutoDepositBuffer)
log.Info(fmt.Sprintf("<%v> set autodeposit to every %v, pay at: %v, buffer: %v", self.proto, local.AutoDepositInterval, local.AutoDepositThreshold, local.AutoDepositBuffer))
if s.Buys {
s.Out.AutoDeposit(local.AutoDepositInterval, local.AutoDepositThreshold, local.AutoDepositBuffer)
log.Info(fmt.Sprintf("<%v> set autodeposit to every %v, pay at: %v, buffer: %v", s.proto, local.AutoDepositInterval, local.AutoDepositThreshold, local.AutoDepositBuffer))
} else {
log.Info(fmt.Sprintf("<%v> autodeposit off (not buying)", self.proto))
log.Info(fmt.Sprintf("<%v> autodeposit off (not buying)", s.proto))
}
}
// Add(n)
// n > 0 called when promised/provided n units of service
// n < 0 called when used/requested n units of service
func (self *Swap) Add(n int) error {
defer self.lock.Unlock()
self.lock.Lock()
self.balance += n
if !self.Sells && self.balance > 0 {
log.Trace(fmt.Sprintf("<%v> remote peer cannot have debt (balance: %v)", self.proto, self.balance))
self.proto.Drop()
return fmt.Errorf("[SWAP] <%v> remote peer cannot have debt (balance: %v)", self.proto, self.balance)
func (s *Swap) Add(n int) error {
defer s.lock.Unlock()
s.lock.Lock()
s.balance += n
if !s.Sells && s.balance > 0 {
log.Trace(fmt.Sprintf("<%v> remote peer cannot have debt (balance: %v)", s.proto, s.balance))
s.proto.Drop()
return fmt.Errorf("[SWAP] <%v> remote peer cannot have debt (balance: %v)", s.proto, s.balance)
}
if !self.Buys && self.balance < 0 {
log.Trace(fmt.Sprintf("<%v> we cannot have debt (balance: %v)", self.proto, self.balance))
return fmt.Errorf("[SWAP] <%v> we cannot have debt (balance: %v)", self.proto, self.balance)
if !s.Buys && s.balance < 0 {
log.Trace(fmt.Sprintf("<%v> we cannot have debt (balance: %v)", s.proto, s.balance))
return fmt.Errorf("[SWAP] <%v> we cannot have debt (balance: %v)", s.proto, s.balance)
}
if self.balance >= int(self.local.DropAt) {
log.Trace(fmt.Sprintf("<%v> remote peer has too much debt (balance: %v, disconnect threshold: %v)", self.proto, self.balance, self.local.DropAt))
self.proto.Drop()
return fmt.Errorf("[SWAP] <%v> remote peer has too much debt (balance: %v, disconnect threshold: %v)", self.proto, self.balance, self.local.DropAt)
} else if self.balance <= -int(self.remote.PayAt) {
self.send()
if s.balance >= int(s.local.DropAt) {
log.Trace(fmt.Sprintf("<%v> remote peer has too much debt (balance: %v, disconnect threshold: %v)", s.proto, s.balance, s.local.DropAt))
s.proto.Drop()
return fmt.Errorf("[SWAP] <%v> remote peer has too much debt (balance: %v, disconnect threshold: %v)", s.proto, s.balance, s.local.DropAt)
} else if s.balance <= -int(s.remote.PayAt) {
s.send()
}
return nil
}
func (self *Swap) Balance() int {
defer self.lock.Unlock()
self.lock.Lock()
return self.balance
func (s *Swap) Balance() int {
defer s.lock.Unlock()
s.lock.Lock()
return s.balance
}
// send(units) is called when payment is due
// In case of insolvency no promise is issued and sent, safe against fraud
// No return value: no error = payment is opportunistic = hang in till dropped
func (self *Swap) send() {
if self.local.BuyAt != nil && self.balance < 0 {
amount := big.NewInt(int64(-self.balance))
amount.Mul(amount, self.remote.SellAt)
promise, err := self.Out.Issue(amount)
func (s *Swap) send() {
if s.local.BuyAt != nil && s.balance < 0 {
amount := big.NewInt(int64(-s.balance))
amount.Mul(amount, s.remote.SellAt)
promise, err := s.Out.Issue(amount)
if err != nil {
log.Warn(fmt.Sprintf("<%v> cannot issue cheque (amount: %v, channel: %v): %v", self.proto, amount, self.Out, err))
log.Warn(fmt.Sprintf("<%v> cannot issue cheque (amount: %v, channel: %v): %v", s.proto, amount, s.Out, err))
} else {
log.Warn(fmt.Sprintf("<%v> cheque issued (amount: %v, channel: %v)", self.proto, amount, self.Out))
self.proto.Pay(-self.balance, promise)
self.balance = 0
log.Warn(fmt.Sprintf("<%v> cheque issued (amount: %v, channel: %v)", s.proto, amount, s.Out))
s.proto.Pay(-s.balance, promise)
s.balance = 0
}
}
}
// receive(units, promise) is called by the protocol when a payment msg is received
// returns error if promise is invalid.
func (self *Swap) Receive(units int, promise Promise) error {
func (s *Swap) Receive(units int, promise Promise) error {
if units <= 0 {
return fmt.Errorf("invalid units: %v <= 0", units)
}
price := new(big.Int).SetInt64(int64(units))
price.Mul(price, self.local.SellAt)
price.Mul(price, s.local.SellAt)
amount, err := self.In.Receive(promise)
amount, err := s.In.Receive(promise)
if err != nil {
err = fmt.Errorf("invalid promise: %v", err)
} else if price.Cmp(amount) != 0 {
// verify amount = units * unit sale price
return fmt.Errorf("invalid amount: %v = %v * %v (units sent in msg * agreed sale unit price) != %v (signed in cheque)", price, units, self.local.SellAt, amount)
return fmt.Errorf("invalid amount: %v = %v * %v (units sent in msg * agreed sale unit price) != %v (signed in cheque)", price, units, s.local.SellAt, amount)
}
if err != nil {
log.Trace(fmt.Sprintf("<%v> invalid promise (amount: %v, channel: %v): %v", self.proto, amount, self.In, err))
log.Trace(fmt.Sprintf("<%v> invalid promise (amount: %v, channel: %v): %v", s.proto, amount, s.In, err))
return err
}
// credit remote peer with units
self.Add(-units)
log.Trace(fmt.Sprintf("<%v> received promise (amount: %v, channel: %v): %v", self.proto, amount, self.In, promise))
s.Add(-units)
log.Trace(fmt.Sprintf("<%v> received promise (amount: %v, channel: %v): %v", s.proto, amount, s.In, promise))
return nil
}
// stop() causes autocash loop to terminate.
// Called after protocol handle loop terminates.
func (self *Swap) Stop() {
defer self.lock.Unlock()
self.lock.Lock()
if self.Buys {
self.Out.Stop()
func (s *Swap) Stop() {
defer s.lock.Unlock()
s.lock.Lock()
if s.Buys {
s.Out.Stop()
}
if self.Sells {
self.In.Stop()
if s.Sells {
s.In.Stop()
}
}

42
swarm/services/swap/swap/swap_test.go
View file

@ -34,20 +34,20 @@ type testPromise struct {
amount *big.Int
}
func (self *testInPayment) Receive(promise Promise) (*big.Int, error) {
func (t *testInPayment) Receive(promise Promise) (*big.Int, error) {
p := promise.(*testPromise)
self.received = append(self.received, p)
t.received = append(t.received, p)
return p.amount, nil
}
func (self *testInPayment) AutoCash(interval time.Duration, limit *big.Int) {
self.autocashInterval = interval
self.autocashLimit = limit
func (t *testInPayment) AutoCash(interval time.Duration, limit *big.Int) {
t.autocashInterval = interval
t.autocashLimit = limit
}
func (self *testInPayment) Cash() (string, error) { return "", nil }
func (t *testInPayment) Cash() (string, error) { return "", nil }
func (self *testInPayment) Stop() {}
func (t *testInPayment) Stop() {}
type testOutPayment struct {
deposits []*big.Int
@ -56,22 +56,22 @@ type testOutPayment struct {
autodepositBuffer *big.Int
}
func (self *testOutPayment) Issue(amount *big.Int) (promise Promise, err error) {
func (t *testOutPayment) Issue(amount *big.Int) (promise Promise, err error) {
return &testPromise{amount}, nil
}
func (self *testOutPayment) Deposit(amount *big.Int) (string, error) {
self.deposits = append(self.deposits, amount)
func (t *testOutPayment) Deposit(amount *big.Int) (string, error) {
t.deposits = append(t.deposits, amount)
return "", nil
}
func (self *testOutPayment) AutoDeposit(interval time.Duration, threshold, buffer *big.Int) {
self.autodepositInterval = interval
self.autodepositThreshold = threshold
self.autodepositBuffer = buffer
func (t *testOutPayment) AutoDeposit(interval time.Duration, threshold, buffer *big.Int) {
t.autodepositInterval = interval
t.autodepositThreshold = threshold
t.autodepositBuffer = buffer
}
func (self *testOutPayment) Stop() {}
func (t *testOutPayment) Stop() {}
type testProtocol struct {
drop bool
@ -79,18 +79,18 @@ type testProtocol struct {
promises []*testPromise
}
func (self *testProtocol) Drop() {
self.drop = true
func (t *testProtocol) Drop() {
t.drop = true
}
func (self *testProtocol) String() string {
func (t *testProtocol) String() string {
return ""
}
func (self *testProtocol) Pay(amount int, promise Promise) {
func (t *testProtocol) Pay(amount int, promise Promise) {
p := promise.(*testPromise)
self.promises = append(self.promises, p)
self.amounts = append(self.amounts, amount)
t.promises = append(t.promises, p)
t.amounts = append(t.amounts, amount)
}
func TestSwap(t *testing.T) {

138
swarm/storage/chunker.go
View file

@ -74,30 +74,30 @@ type TreeChunker struct {
branches int64
hashFunc SwarmHasher
// calculated
hashSize int64 // self.hashFunc.New().Size()
hashSize int64 // hashFunc.New().Size()
chunkSize int64 // hashSize*branches
workerCount int64 // the number of worker routines used
workerLock sync.RWMutex // lock for the worker count
}
func NewTreeChunker(params *ChunkerParams) (self *TreeChunker) {
self = &TreeChunker{}
self.hashFunc = MakeHashFunc(params.Hash)
self.branches = params.Branches
self.hashSize = int64(self.hashFunc().Size())
self.chunkSize = self.hashSize * self.branches
self.workerCount = 0
func NewTreeChunker(params *ChunkerParams) (chunker *TreeChunker) {
chunker = &TreeChunker{}
chunker.hashFunc = MakeHashFunc(params.Hash)
chunker.branches = params.Branches
chunker.hashSize = int64(chunker.hashFunc().Size())
chunker.chunkSize = chunker.hashSize * chunker.branches
chunker.workerCount = 0
return
}
// func (self *TreeChunker) KeySize() int64 {
// return self.hashSize
// func (chunker *TreeChunker) KeySize() int64 {
// return chunker.hashSize
// }
// String() for pretty printing
func (self *Chunk) String() string {
return fmt.Sprintf("Key: %v TreeSize: %v Chunksize: %v", self.Key.Log(), self.Size, len(self.SData))
func (c *Chunk) String() string {
return fmt.Sprintf("Key: %v TreeSize: %v Chunksize: %v", c.Key.Log(), c.Size, len(c.SData))
}
type hashJob struct {
@ -107,26 +107,26 @@ type hashJob struct {
parentWg *sync.WaitGroup
}
func (self *TreeChunker) incrementWorkerCount() {
self.workerLock.Lock()
defer self.workerLock.Unlock()
self.workerCount += 1
func (chunker *TreeChunker) incrementWorkerCount() {
chunker.workerLock.Lock()
defer chunker.workerLock.Unlock()
chunker.workerCount += 1
}
func (self *TreeChunker) getWorkerCount() int64 {
self.workerLock.RLock()
defer self.workerLock.RUnlock()
return self.workerCount
func (chunker *TreeChunker) getWorkerCount() int64 {
chunker.workerLock.RLock()
defer chunker.workerLock.RUnlock()
return chunker.workerCount
}
func (self *TreeChunker) decrementWorkerCount() {
self.workerLock.Lock()
defer self.workerLock.Unlock()
self.workerCount -= 1
func (chunker *TreeChunker) decrementWorkerCount() {
chunker.workerLock.Lock()
defer chunker.workerLock.Unlock()
chunker.workerCount -= 1
}
func (self *TreeChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, swg, wwg *sync.WaitGroup) (Key, error) {
if self.chunkSize <= 0 {
func (chunker *TreeChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, swg, wwg *sync.WaitGroup) (Key, error) {
if chunker.chunkSize <= 0 {
panic("chunker must be initialised")
}
@ -140,23 +140,23 @@ func (self *TreeChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, s
wwg.Add(1)
}
self.incrementWorkerCount()
go self.hashWorker(jobC, chunkC, errC, quitC, swg, wwg)
chunker.incrementWorkerCount()
go chunker.hashWorker(jobC, chunkC, errC, quitC, swg, wwg)
depth := 0
treeSize := self.chunkSize
treeSize := chunker.chunkSize
// takes lowest depth such that chunksize*HashCount^(depth+1) > size
// power series, will find the order of magnitude of the data size in base hashCount or numbers of levels of branching in the resulting tree.
for ; treeSize < size; treeSize *= self.branches {
for ; treeSize < size; treeSize *= chunker.branches {
depth++
}
key := make([]byte, self.hashFunc().Size())
key := make([]byte, chunker.hashFunc().Size())
// this waitgroup member is released after the root hash is calculated
wg.Add(1)
//launch actual recursive function passing the waitgroups
go self.split(depth, treeSize/self.branches, key, data, size, jobC, chunkC, errC, quitC, wg, swg, wwg)
go chunker.split(depth, treeSize/chunker.branches, key, data, size, jobC, chunkC, errC, quitC, wg, swg, wwg)
// closes internal error channel if all subprocesses in the workgroup finished
go func() {
@ -182,12 +182,12 @@ func (self *TreeChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, s
return key, nil
}
func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reader, size int64, jobC chan *hashJob, chunkC chan *Chunk, errC chan error, quitC chan bool, parentWg, swg, wwg *sync.WaitGroup) {
func (chunker *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reader, size int64, jobC chan *hashJob, chunkC chan *Chunk, errC chan error, quitC chan bool, parentWg, swg, wwg *sync.WaitGroup) {
//
for depth > 0 && size < treeSize {
treeSize /= self.branches
treeSize /= chunker.branches
depth--
}
@ -214,7 +214,7 @@ func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reade
// intermediate chunk containing child nodes hashes
branchCnt := (size + treeSize - 1) / treeSize
var chunk = make([]byte, branchCnt*self.hashSize+8)
var chunk = make([]byte, branchCnt*chunker.hashSize+8)
var pos, i int64
binary.LittleEndian.PutUint64(chunk[0:8], uint64(size))
@ -229,10 +229,10 @@ func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reade
secSize = treeSize
}
// the hash of that data
subTreeKey := chunk[8+i*self.hashSize : 8+(i+1)*self.hashSize]
subTreeKey := chunk[8+i*chunker.hashSize : 8+(i+1)*chunker.hashSize]
childrenWg.Add(1)
self.split(depth-1, treeSize/self.branches, subTreeKey, data, secSize, jobC, chunkC, errC, quitC, childrenWg, swg, wwg)
chunker.split(depth-1, treeSize/chunker.branches, subTreeKey, data, secSize, jobC, chunkC, errC, quitC, childrenWg, swg, wwg)
i++
pos += treeSize
@ -242,13 +242,13 @@ func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reade
// go func() {
childrenWg.Wait()
worker := self.getWorkerCount()
worker := chunker.getWorkerCount()
if int64(len(jobC)) > worker && worker < ChunkProcessors {
if wwg != nil {
wwg.Add(1)
}
self.incrementWorkerCount()
go self.hashWorker(jobC, chunkC, errC, quitC, swg, wwg)
chunker.incrementWorkerCount()
go chunker.hashWorker(jobC, chunkC, errC, quitC, swg, wwg)
}
select {
@ -257,10 +257,10 @@ func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reade
}
}
func (self *TreeChunker) hashWorker(jobC chan *hashJob, chunkC chan *Chunk, errC chan error, quitC chan bool, swg, wwg *sync.WaitGroup) {
defer self.decrementWorkerCount()
func (chunker *TreeChunker) hashWorker(jobC chan *hashJob, chunkC chan *Chunk, errC chan error, quitC chan bool, swg, wwg *sync.WaitGroup) {
defer chunker.decrementWorkerCount()
hasher := self.hashFunc()
hasher := chunker.hashFunc()
if wwg != nil {
defer wwg.Done()
}
@ -272,7 +272,7 @@ func (self *TreeChunker) hashWorker(jobC chan *hashJob, chunkC chan *Chunk, errC
return
}
// now we got the hashes in the chunk, then hash the chunks
self.hashChunk(hasher, job, chunkC, swg)
chunker.hashChunk(hasher, job, chunkC, swg)
case <-quitC:
return
}
@ -282,7 +282,7 @@ func (self *TreeChunker) hashWorker(jobC chan *hashJob, chunkC chan *Chunk, errC
// The treeChunkers own Hash hashes together
// - the size (of the subtree encoded in the Chunk)
// - the Chunk, ie. the contents read from the input reader
func (self *TreeChunker) hashChunk(hasher SwarmHash, job *hashJob, chunkC chan *Chunk, swg *sync.WaitGroup) {
func (chunker *TreeChunker) hashChunk(hasher SwarmHash, job *hashJob, chunkC chan *Chunk, swg *sync.WaitGroup) {
hasher.ResetWithLength(job.chunk[:8]) // 8 bytes of length
hasher.Write(job.chunk[8:]) // minus 8 []byte length
h := hasher.Sum(nil)
@ -316,7 +316,7 @@ func (self *TreeChunker) hashChunk(hasher SwarmHash, job *hashJob, chunkC chan *
}
}
func (self *TreeChunker) Append(key Key, data io.Reader, chunkC chan *Chunk, swg, wwg *sync.WaitGroup) (Key, error) {
func (chunker *TreeChunker) Append(key Key, data io.Reader, chunkC chan *Chunk, swg, wwg *sync.WaitGroup) (Key, error) {
return nil, errAppendOppNotSuported
}
@ -332,44 +332,44 @@ type LazyChunkReader struct {
}
// implements the Joiner interface
func (self *TreeChunker) Join(key Key, chunkC chan *Chunk) LazySectionReader {
func (chunker *TreeChunker) Join(key Key, chunkC chan *Chunk) LazySectionReader {
return &LazyChunkReader{
key: key,
chunkC: chunkC,
chunkSize: self.chunkSize,
branches: self.branches,
hashSize: self.hashSize,
chunkSize: chunker.chunkSize,
branches: chunker.branches,
hashSize: chunker.hashSize,
}
}
// Size is meant to be called on the LazySectionReader
func (self *LazyChunkReader) Size(quitC chan bool) (n int64, err error) {
if self.chunk != nil {
return self.chunk.Size, nil
func (reader *LazyChunkReader) Size(quitC chan bool) (n int64, err error) {
if reader.chunk != nil {
return reader.chunk.Size, nil
}
chunk := retrieve(self.key, self.chunkC, quitC)
chunk := retrieve(reader.key, reader.chunkC, quitC)
if chunk == nil {
select {
case <-quitC:
return 0, errors.New("aborted")
default:
return 0, fmt.Errorf("root chunk not found for %v", self.key.Hex())
return 0, fmt.Errorf("root chunk not found for %v", reader.key.Hex())
}
}
self.chunk = chunk
reader.chunk = chunk
return chunk.Size, nil
}
// read at can be called numerous times
// concurrent reads are allowed
// Size() needs to be called synchronously on the LazyChunkReader first
func (self *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) {
func (reader *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) {
// this is correct, a swarm doc cannot be zero length, so no EOF is expected
if len(b) == 0 {
return 0, nil
}
quitC := make(chan bool)
size, err := self.Size(quitC)
size, err := reader.Size(quitC)
if err != nil {
return 0, err
}
@ -380,13 +380,13 @@ func (self *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) {
var treeSize int64
var depth int
// calculate depth and max treeSize
treeSize = self.chunkSize
for ; treeSize < size; treeSize *= self.branches {
treeSize = reader.chunkSize
for ; treeSize < size; treeSize *= reader.branches {
depth++
}
wg := sync.WaitGroup{}
wg.Add(1)
go self.join(b, off, off+int64(len(b)), depth, treeSize/self.branches, self.chunk, &wg, errC, quitC)
go reader.join(b, off, off+int64(len(b)), depth, treeSize/reader.branches, reader.chunk, &wg, errC, quitC)
go func() {
wg.Wait()
close(errC)
@ -404,7 +404,7 @@ func (self *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) {
return len(b), nil
}
func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeSize int64, chunk *Chunk, parentWg *sync.WaitGroup, errC chan error, quitC chan bool) {
func (reader *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeSize int64, chunk *Chunk, parentWg *sync.WaitGroup, errC chan error, quitC chan bool) {
defer parentWg.Done()
// return NewDPA(&LocalStore{})
@ -412,7 +412,7 @@ func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, tr
// find appropriate block level
for chunk.Size < treeSize && depth > 0 {
treeSize /= self.branches
treeSize /= reader.branches
depth--
}
@ -449,8 +449,8 @@ func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, tr
}
wg.Add(1)
go func(j int64) {
childKey := chunk.SData[8+j*self.hashSize : 8+(j+1)*self.hashSize]
chunk := retrieve(childKey, self.chunkC, quitC)
childKey := chunk.SData[8+j*reader.hashSize : 8+(j+1)*reader.hashSize]
chunk := retrieve(childKey, reader.chunkC, quitC)
if chunk == nil {
select {
case errC <- fmt.Errorf("chunk %v-%v not found", off, off+treeSize):
@ -461,7 +461,7 @@ func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, tr
if soff < off {
soff = off
}
self.join(b[soff-off:seoff-off], soff-roff, seoff-roff, depth-1, treeSize/self.branches, chunk, wg, errC, quitC)
reader.join(b[soff-off:seoff-off], soff-roff, seoff-roff, depth-1, treeSize/reader.branches, chunk, wg, errC, quitC)
}(i)
} //for
}
@ -496,10 +496,10 @@ func retrieve(key Key, chunkC chan *Chunk, quitC chan bool) *Chunk {
}
// Read keeps a cursor so cannot be called simulateously, see ReadAt
func (self *LazyChunkReader) Read(b []byte) (read int, err error) {
read, err = self.ReadAt(b, self.off)
func (reader *LazyChunkReader) Read(b []byte) (read int, err error) {
read, err = reader.ReadAt(b, reader.off)
self.off += int64(read)
reader.off += int64(read)
return
}

22
swarm/storage/chunker_test.go
View file

@ -45,12 +45,12 @@ type chunkerTester struct {
t test
}
func (self *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, chunkC chan *Chunk, swg *sync.WaitGroup, expectedError error) (key Key, err error) {
func (tester *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, chunkC chan *Chunk, swg *sync.WaitGroup, expectedError error) (key Key, err error) {
// reset
self.chunks = make(map[string]*Chunk)
tester.chunks = make(map[string]*Chunk)
if self.inputs == nil {
self.inputs = make(map[uint64][]byte)
if tester.inputs == nil {
tester.inputs = make(map[uint64][]byte)
}
quitC := make(chan bool)
@ -64,8 +64,8 @@ func (self *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, c
case <-quitC:
return nil
case chunk := <-chunkC:
// self.chunks = append(self.chunks, chunk)
self.chunks[chunk.Key.String()] = chunk
// tester.chunks = append(tester.chunks, chunk)
tester.chunks[chunk.Key.String()] = chunk
if chunk.wg != nil {
chunk.wg.Done()
}
@ -89,7 +89,7 @@ func (self *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, c
return key, err
}
func (self *chunkerTester) Append(chunker Splitter, rootKey Key, data io.Reader, chunkC chan *Chunk, swg *sync.WaitGroup, expectedError error) (key Key, err error) {
func (tester *chunkerTester) Append(chunker Splitter, rootKey Key, data io.Reader, chunkC chan *Chunk, swg *sync.WaitGroup, expectedError error) (key Key, err error) {
quitC := make(chan bool)
timeout := time.After(60 * time.Second)
if chunkC != nil {
@ -102,10 +102,10 @@ func (self *chunkerTester) Append(chunker Splitter, rootKey Key, data io.Reader,
return nil
case chunk := <-chunkC:
if chunk != nil {
stored, success := self.chunks[chunk.Key.String()]
stored, success := tester.chunks[chunk.Key.String()]
if !success {
// Requesting data
self.chunks[chunk.Key.String()] = chunk
tester.chunks[chunk.Key.String()] = chunk
if chunk.wg != nil {
chunk.wg.Done()
}
@ -135,7 +135,7 @@ func (self *chunkerTester) Append(chunker Splitter, rootKey Key, data io.Reader,
return key, err
}
func (self *chunkerTester) Join(chunker Chunker, key Key, c int, chunkC chan *Chunk, quitC chan bool) LazySectionReader {
func (tester *chunkerTester) Join(chunker Chunker, key Key, c int, chunkC chan *Chunk, quitC chan bool) LazySectionReader {
// reset but not the chunks
reader := chunker.Join(key, chunkC)
@ -153,7 +153,7 @@ func (self *chunkerTester) Join(chunker Chunker, key Key, c int, chunkC chan *Ch
return nil
}
// this just mocks the behaviour of a chunk store retrieval
stored, success := self.chunks[chunk.Key.String()]
stored, success := tester.chunks[chunk.Key.String()]
if !success {
return errors.New("Not found")
}

8
swarm/storage/common_test.go
View file

@ -46,12 +46,12 @@ func testDataReader(l int) (r io.Reader) {
return io.LimitReader(rand.Reader, int64(l))
}
func (self *brokenLimitedReader) Read(buf []byte) (int, error) {
if self.off+len(buf) > self.errAt {
func (reader *brokenLimitedReader) Read(buf []byte) (int, error) {
if reader.off+len(buf) > reader.errAt {
return 0, fmt.Errorf("Broken reader")
}
self.off += len(buf)
return self.lr.Read(buf)
reader.off += len(buf)
return reader.lr.Read(buf)
}
func testDataReaderAndSlice(l int) (r io.Reader, slice []byte) {

28
swarm/storage/database.go
View file

@ -45,27 +45,27 @@ func NewLDBDatabase(file string) (*LDBDatabase, error) {
return database, nil
}
func (self *LDBDatabase) Put(key []byte, value []byte) {
err := self.db.Put(key, value, nil)
func (database *LDBDatabase) Put(key []byte, value []byte) {
err := database.db.Put(key, value, nil)
if err != nil {
fmt.Println("Error put", err)
}
}
func (self *LDBDatabase) Get(key []byte) ([]byte, error) {
dat, err := self.db.Get(key, nil)
func (database *LDBDatabase) Get(key []byte) ([]byte, error) {
dat, err := database.db.Get(key, nil)
if err != nil {
return nil, err
}
return dat, nil
}
func (self *LDBDatabase) Delete(key []byte) error {
return self.db.Delete(key, nil)
func (database *LDBDatabase) Delete(key []byte) error {
return database.db.Delete(key, nil)
}
func (self *LDBDatabase) LastKnownTD() []byte {
data, _ := self.Get([]byte("LTD"))
func (database *LDBDatabase) LastKnownTD() []byte {
data, _ := database.Get([]byte("LTD"))
if len(data) == 0 {
data = []byte{0x0}
@ -74,15 +74,15 @@ func (self *LDBDatabase) LastKnownTD() []byte {
return data
}
func (self *LDBDatabase) NewIterator() iterator.Iterator {
return self.db.NewIterator(nil, nil)
func (database *LDBDatabase) NewIterator() iterator.Iterator {
return database.db.NewIterator(nil, nil)
}
func (self *LDBDatabase) Write(batch *leveldb.Batch) error {
return self.db.Write(batch, nil)
func (database *LDBDatabase) Write(batch *leveldb.Batch) error {
return database.db.Write(batch, nil)
}
func (self *LDBDatabase) Close() {
func (database *LDBDatabase) Close() {
// Close the leveldb database
self.db.Close()
database.db.Close()
}

24
swarm/storage/dbstore.go
View file

@ -559,12 +559,12 @@ type dbSyncIterator struct {
}
// initialises a sync iterator from a syncToken (passed in with the handshake)
func (self *DbStore) NewSyncIterator(state DbSyncState) (si *dbSyncIterator, err error) {
func (store *DbStore) NewSyncIterator(state DbSyncState) (si *dbSyncIterator, err error) {
if state.First > state.Last {
return nil, fmt.Errorf("no entries found")
}
si = &dbSyncIterator{
it: self.db.NewIterator(),
it: store.db.NewIterator(),
DbSyncState: state,
}
si.it.Seek(getIndexKey(state.Start))
@ -573,28 +573,28 @@ func (self *DbStore) NewSyncIterator(state DbSyncState) (si *dbSyncIterator, err
// walk the area from Start to Stop and returns items within time interval
// First to Last
func (self *dbSyncIterator) Next() (key Key) {
for self.it.Valid() {
dbkey := self.it.Key()
func (iterator *dbSyncIterator) Next() (key Key) {
for iterator.it.Valid() {
dbkey := iterator.it.Key()
if dbkey[0] != 0 {
break
}
key = Key(make([]byte, len(dbkey)-1))
copy(key[:], dbkey[1:])
if bytes.Compare(key[:], self.Start) <= 0 {
self.it.Next()
if bytes.Compare(key[:], iterator.Start) <= 0 {
iterator.it.Next()
continue
}
if bytes.Compare(key[:], self.Stop) > 0 {
if bytes.Compare(key[:], iterator.Stop) > 0 {
break
}
var index dpaDBIndex
decodeIndex(self.it.Value(), &index)
self.it.Next()
if (index.Idx >= self.First) && (index.Idx < self.Last) {
decodeIndex(iterator.it.Value(), &index)
iterator.it.Next()
if (index.Idx >= iterator.First) && (index.Idx < iterator.Last) {
return
}
}
self.it.Release()
iterator.it.Release()
return nil
}

82
swarm/storage/dpa.go
View file

@ -90,53 +90,53 @@ func NewDPA(store ChunkStore, params *ChunkerParams) *DPA {
// FS-aware API and httpaccess
// Chunk retrieval blocks on netStore requests with a timeout so reader will
// report error if retrieval of chunks within requested range time out.
func (self *DPA) Retrieve(key Key) LazySectionReader {
return self.Chunker.Join(key, self.retrieveC)
func (dpa *DPA) Retrieve(key Key) LazySectionReader {
return dpa.Chunker.Join(key, dpa.retrieveC)
}
// Public API. Main entry point for document storage directly. Used by the
// FS-aware API and httpaccess
func (self *DPA) Store(data io.Reader, size int64, swg *sync.WaitGroup, wwg *sync.WaitGroup) (key Key, err error) {
return self.Chunker.Split(data, size, self.storeC, swg, wwg)
func (dpa *DPA) Store(data io.Reader, size int64, swg *sync.WaitGroup, wwg *sync.WaitGroup) (key Key, err error) {
return dpa.Chunker.Split(data, size, dpa.storeC, swg, wwg)
}
func (self *DPA) Start() {
self.lock.Lock()
defer self.lock.Unlock()
if self.running {
func (dpa *DPA) Start() {
dpa.lock.Lock()
defer dpa.lock.Unlock()
if dpa.running {
return
}
self.running = true
self.retrieveC = make(chan *Chunk, retrieveChanCapacity)
self.storeC = make(chan *Chunk, storeChanCapacity)
self.quitC = make(chan bool)
self.storeLoop()
self.retrieveLoop()
dpa.running = true
dpa.retrieveC = make(chan *Chunk, retrieveChanCapacity)
dpa.storeC = make(chan *Chunk, storeChanCapacity)
dpa.quitC = make(chan bool)
dpa.storeLoop()
dpa.retrieveLoop()
}
func (self *DPA) Stop() {
self.lock.Lock()
defer self.lock.Unlock()
if !self.running {
func (dpa *DPA) Stop() {
dpa.lock.Lock()
defer dpa.lock.Unlock()
if !dpa.running {
return
}
self.running = false
close(self.quitC)
dpa.running = false
close(dpa.quitC)
}
// retrieveLoop dispatches the parallel chunk retrieval requests received on the
// retrieve channel to its ChunkStore (NetStore or LocalStore)
func (self *DPA) retrieveLoop() {
func (dpa *DPA) retrieveLoop() {
for i := 0; i < maxRetrieveProcesses; i++ {
go self.retrieveWorker()
go dpa.retrieveWorker()
}
log.Trace(fmt.Sprintf("dpa: retrieve loop spawning %v workers", maxRetrieveProcesses))
}
func (self *DPA) retrieveWorker() {
for chunk := range self.retrieveC {
func (dpa *DPA) retrieveWorker() {
for chunk := range dpa.retrieveC {
log.Trace(fmt.Sprintf("dpa: retrieve loop : chunk %v", chunk.Key.Log()))
storedChunk, err := self.Get(chunk.Key)
storedChunk, err := dpa.Get(chunk.Key)
if err == notFound {
log.Trace(fmt.Sprintf("chunk %v not found", chunk.Key.Log()))
} else if err != nil {
@ -148,7 +148,7 @@ func (self *DPA) retrieveWorker() {
close(chunk.C)
select {
case <-self.quitC:
case <-dpa.quitC:
return
default:
}
@ -157,24 +157,24 @@ func (self *DPA) retrieveWorker() {
// storeLoop dispatches the parallel chunk store request processors
// received on the store channel to its ChunkStore (NetStore or LocalStore)
func (self *DPA) storeLoop() {
func (dpa *DPA) storeLoop() {
for i := 0; i < maxStoreProcesses; i++ {
go self.storeWorker()
go dpa.storeWorker()
}
log.Trace(fmt.Sprintf("dpa: store spawning %v workers", maxStoreProcesses))
}
func (self *DPA) storeWorker() {
func (dpa *DPA) storeWorker() {
for chunk := range self.storeC {
self.Put(chunk)
for chunk := range dpa.storeC {
dpa.Put(chunk)
if chunk.wg != nil {
log.Trace(fmt.Sprintf("dpa: store processor %v", chunk.Key.Log()))
chunk.wg.Done()
}
select {
case <-self.quitC:
case <-dpa.quitC:
return
default:
}
@ -198,14 +198,14 @@ func NewDpaChunkStore(localStore, netStore ChunkStore) *dpaChunkStore {
// Get is the entrypoint for local retrieve requests
// waits for response or times out
func (self *dpaChunkStore) Get(key Key) (chunk *Chunk, err error) {
chunk, err = self.netStore.Get(key)
func (store *dpaChunkStore) Get(key Key) (chunk *Chunk, err error) {
chunk, err = store.netStore.Get(key)
// timeout := time.Now().Add(searchTimeout)
if chunk.SData != nil {
log.Trace(fmt.Sprintf("DPA.Get: %v found locally, %d bytes", key.Log(), len(chunk.SData)))
return
}
// TODO: use self.timer time.Timer and reset with defer disableTimer
// TODO: use store.timer time.Timer and reset with defer disableTimer
timer := time.After(searchTimeout)
select {
case <-timer:
@ -218,8 +218,8 @@ func (self *dpaChunkStore) Get(key Key) (chunk *Chunk, err error) {
}
// Put is the entrypoint for local store requests coming from storeLoop
func (self *dpaChunkStore) Put(entry *Chunk) {
chunk, err := self.localStore.Get(entry.Key)
func (store *dpaChunkStore) Put(entry *Chunk) {
chunk, err := store.localStore.Get(entry.Key)
if err != nil {
log.Trace(fmt.Sprintf("DPA.Put: %v new chunk. call netStore.Put", entry.Key.Log()))
chunk = entry
@ -232,10 +232,10 @@ func (self *dpaChunkStore) Put(entry *Chunk) {
return
}
// from this point on the storage logic is the same with network storage requests
log.Trace(fmt.Sprintf("DPA.Put %v: %v", self.n, chunk.Key.Log()))
self.n++
self.netStore.Put(chunk)
log.Trace(fmt.Sprintf("DPA.Put %v: %v", store.n, chunk.Key.Log()))
store.n++
store.netStore.Put(chunk)
}
// Close chunk store
func (self *dpaChunkStore) Close() {}
func (store *dpaChunkStore) Close() {}

26
swarm/storage/localstore.go
View file

@ -46,25 +46,25 @@ func NewLocalStore(hash SwarmHasher, params *StoreParams) (*LocalStore, error) {
}, nil
}
func (self *LocalStore) CacheCounter() uint64 {
return uint64(self.memStore.(*MemStore).Counter())
func (store *LocalStore) CacheCounter() uint64 {
return uint64(store.memStore.(*MemStore).Counter())
}
func (self *LocalStore) DbCounter() uint64 {
return self.DbStore.(*DbStore).Counter()
func (store *LocalStore) DbCounter() uint64 {
return store.DbStore.(*DbStore).Counter()
}
// LocalStore is itself a chunk store
// LocalStore is itstore a chunk store
// unsafe, in that the data is not integrity checked
func (self *LocalStore) Put(chunk *Chunk) {
func (store *LocalStore) Put(chunk *Chunk) {
chunk.dbStored = make(chan bool)
self.memStore.Put(chunk)
store.memStore.Put(chunk)
if chunk.wg != nil {
chunk.wg.Add(1)
}
go func() {
dbStorePutCounter.Inc(1)
self.DbStore.Put(chunk)
store.DbStore.Put(chunk)
if chunk.wg != nil {
chunk.wg.Done()
}
@ -75,19 +75,19 @@ func (self *LocalStore) Put(chunk *Chunk) {
// This method is blocking until the chunk is retrieved
// so additional timeout may be needed to wrap this call if
// ChunkStores are remote and can have long latency
func (self *LocalStore) Get(key Key) (chunk *Chunk, err error) {
chunk, err = self.memStore.Get(key)
func (store *LocalStore) Get(key Key) (chunk *Chunk, err error) {
chunk, err = store.memStore.Get(key)
if err == nil {
return
}
chunk, err = self.DbStore.Get(key)
chunk, err = store.DbStore.Get(key)
if err != nil {
return
}
chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8]))
self.memStore.Put(chunk)
store.memStore.Put(chunk)
return
}
// Close local store
func (self *LocalStore) Close() {}
func (store *LocalStore) Close() {}

26
swarm/storage/netstore.go
View file

@ -58,7 +58,7 @@ type StoreParams struct {
}
//create params with default values
func NewDefaultStoreParams() (self *StoreParams) {
func NewDefaultStoreParams() *StoreParams {
return &StoreParams{
DbCapacity: defaultDbCapacity,
CacheCapacity: defaultCacheCapacity,
@ -68,8 +68,8 @@ func NewDefaultStoreParams() (self *StoreParams) {
//this can only finally be set after all config options (file, cmd line, env vars)
//have been evaluated
func (self *StoreParams) Init(path string) {
self.ChunkDbPath = filepath.Join(path, "chunks")
func (params *StoreParams) Init(path string) {
params.ChunkDbPath = filepath.Join(path, "chunks")
}
// netstore contructor, takes path argument that is used to initialise dbStore,
@ -92,8 +92,8 @@ var (
// ~ unsafe put in localdb no check if exists no extra copy no hash validation
// the chunk is forced to propagate (Cloud.Store) even if locally found!
// caller needs to make sure if that is wanted
func (self *NetStore) Put(entry *Chunk) {
self.localStore.Put(entry)
func (store *NetStore) Put(entry *Chunk) {
store.localStore.Put(entry)
// handle deliveries
if entry.Req != nil {
@ -102,19 +102,19 @@ func (self *NetStore) Put(entry *Chunk) {
// that the chunk is has been retrieved
close(entry.Req.C)
// deliver the chunk to requesters upstream
go self.cloud.Deliver(entry)
go store.cloud.Deliver(entry)
} else {
log.Trace(fmt.Sprintf("NetStore.Put: localStore.Put %v stored locally", entry.Key.Log()))
// handle propagating store requests
// go self.cloud.Store(entry)
go self.cloud.Store(entry)
// go store.cloud.Store(entry)
go store.cloud.Store(entry)
}
}
// retrieve logic common for local and network chunk retrieval requests
func (self *NetStore) Get(key Key) (*Chunk, error) {
func (store *NetStore) Get(key Key) (*Chunk, error) {
var err error
chunk, err := self.localStore.Get(key)
chunk, err := store.localStore.Get(key)
if err == nil {
if chunk.Req == nil {
log.Trace(fmt.Sprintf("NetStore.Get: %v found locally", key))
@ -127,10 +127,10 @@ func (self *NetStore) Get(key Key) (*Chunk, error) {
// no data and no request status
log.Trace(fmt.Sprintf("NetStore.Get: %v not found locally. open new request", key))
chunk = NewChunk(key, newRequestStatus(key))
self.localStore.memStore.Put(chunk)
go self.cloud.Retrieve(chunk)
store.localStore.memStore.Put(chunk)
go store.cloud.Retrieve(chunk)
return chunk, nil
}
// Close netstore
func (self *NetStore) Close() {}
func (store *NetStore) Close() {}

176
swarm/storage/pyramid.go
View file

@ -126,54 +126,54 @@ type PyramidChunker struct {
workerLock sync.RWMutex
}
func NewPyramidChunker(params *ChunkerParams) (self *PyramidChunker) {
self = &PyramidChunker{}
self.hashFunc = MakeHashFunc(params.Hash)
self.branches = params.Branches
self.hashSize = int64(self.hashFunc().Size())
self.chunkSize = self.hashSize * self.branches
self.workerCount = 0
return
func NewPyramidChunker(params *ChunkerParams) *PyramidChunker {
chunker := &PyramidChunker{}
chunker.hashFunc = MakeHashFunc(params.Hash)
chunker.branches = params.Branches
chunker.hashSize = int64(chunker.hashFunc().Size())
chunker.chunkSize = chunker.hashSize * chunker.branches
chunker.workerCount = 0
return chunker
}
func (self *PyramidChunker) Join(key Key, chunkC chan *Chunk) LazySectionReader {
func (chunker *PyramidChunker) Join(key Key, chunkC chan *Chunk) LazySectionReader {
return &LazyChunkReader{
key: key,
chunkC: chunkC,
chunkSize: self.chunkSize,
branches: self.branches,
hashSize: self.hashSize,
chunkSize: chunker.chunkSize,
branches: chunker.branches,
hashSize: chunker.hashSize,
}
}
func (self *PyramidChunker) incrementWorkerCount() {
self.workerLock.Lock()
defer self.workerLock.Unlock()
self.workerCount += 1
func (chunker *PyramidChunker) incrementWorkerCount() {
chunker.workerLock.Lock()
defer chunker.workerLock.Unlock()
chunker.workerCount += 1
}
func (self *PyramidChunker) getWorkerCount() int64 {
self.workerLock.Lock()
defer self.workerLock.Unlock()
return self.workerCount
func (chunker *PyramidChunker) getWorkerCount() int64 {
chunker.workerLock.Lock()
defer chunker.workerLock.Unlock()
return chunker.workerCount
}
func (self *PyramidChunker) decrementWorkerCount() {
self.workerLock.Lock()
defer self.workerLock.Unlock()
self.workerCount -= 1
func (chunker *PyramidChunker) decrementWorkerCount() {
chunker.workerLock.Lock()
defer chunker.workerLock.Unlock()
chunker.workerCount -= 1
}
func (self *PyramidChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, storageWG, processorWG *sync.WaitGroup) (Key, error) {
func (chunker *PyramidChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, storageWG, processorWG *sync.WaitGroup) (Key, error) {
jobC := make(chan *chunkJob, 2*ChunkProcessors)
wg := &sync.WaitGroup{}
errC := make(chan error)
quitC := make(chan bool)
rootKey := make([]byte, self.hashSize)
chunkLevel := make([][]*TreeEntry, self.branches)
rootKey := make([]byte, chunker.hashSize)
chunkLevel := make([][]*TreeEntry, chunker.branches)
wg.Add(1)
go self.prepareChunks(false, chunkLevel, data, rootKey, quitC, wg, jobC, processorWG, chunkC, errC, storageWG)
go chunker.prepareChunks(false, chunkLevel, data, rootKey, quitC, wg, jobC, processorWG, chunkC, errC, storageWG)
// closes internal error channel if all subprocesses in the workgroup finished
go func() {
@ -204,20 +204,20 @@ func (self *PyramidChunker) Split(data io.Reader, size int64, chunkC chan *Chunk
}
func (self *PyramidChunker) Append(key Key, data io.Reader, chunkC chan *Chunk, storageWG, processorWG *sync.WaitGroup) (Key, error) {
func (chunker *PyramidChunker) Append(key Key, data io.Reader, chunkC chan *Chunk, storageWG, processorWG *sync.WaitGroup) (Key, error) {
quitC := make(chan bool)
rootKey := make([]byte, self.hashSize)
chunkLevel := make([][]*TreeEntry, self.branches)
rootKey := make([]byte, chunker.hashSize)
chunkLevel := make([][]*TreeEntry, chunker.branches)
// Load the right most unfinished tree chunks in every level
self.loadTree(chunkLevel, key, chunkC, quitC)
chunker.loadTree(chunkLevel, key, chunkC, quitC)
jobC := make(chan *chunkJob, 2*ChunkProcessors)
wg := &sync.WaitGroup{}
errC := make(chan error)
wg.Add(1)
go self.prepareChunks(true, chunkLevel, data, rootKey, quitC, wg, jobC, processorWG, chunkC, errC, storageWG)
go chunker.prepareChunks(true, chunkLevel, data, rootKey, quitC, wg, jobC, processorWG, chunkC, errC, storageWG)
// closes internal error channel if all subprocesses in the workgroup finished
go func() {
@ -245,10 +245,10 @@ func (self *PyramidChunker) Append(key Key, data io.Reader, chunkC chan *Chunk,
}
func (self *PyramidChunker) processor(id int64, jobC chan *chunkJob, chunkC chan *Chunk, errC chan error, quitC chan bool, swg, wwg *sync.WaitGroup) {
defer self.decrementWorkerCount()
func (chunker *PyramidChunker) processor(id int64, jobC chan *chunkJob, chunkC chan *Chunk, errC chan error, quitC chan bool, swg, wwg *sync.WaitGroup) {
defer chunker.decrementWorkerCount()
hasher := self.hashFunc()
hasher := chunker.hashFunc()
if wwg != nil {
defer wwg.Done()
}
@ -259,14 +259,14 @@ func (self *PyramidChunker) processor(id int64, jobC chan *chunkJob, chunkC chan
if !ok {
return
}
self.processChunk(id, hasher, job, chunkC, swg)
chunker.processChunk(id, hasher, job, chunkC, swg)
case <-quitC:
return
}
}
}
func (self *PyramidChunker) processChunk(id int64, hasher SwarmHash, job *chunkJob, chunkC chan *Chunk, swg *sync.WaitGroup) {
func (chunker *PyramidChunker) processChunk(id int64, hasher SwarmHash, job *chunkJob, chunkC chan *Chunk, swg *sync.WaitGroup) {
hasher.ResetWithLength(job.chunk[:8]) // 8 bytes of length
hasher.Write(job.chunk[8:]) // minus 8 []byte length
h := hasher.Sum(nil)
@ -294,7 +294,7 @@ func (self *PyramidChunker) processChunk(id int64, hasher SwarmHash, job *chunkJ
}
}
func (self *PyramidChunker) loadTree(chunkLevel [][]*TreeEntry, key Key, chunkC chan *Chunk, quitC chan bool) error {
func (chunker *PyramidChunker) loadTree(chunkLevel [][]*TreeEntry, key Key, chunkC chan *Chunk, quitC chan bool) error {
// Get the root chunk to get the total size
chunk := retrieve(key, chunkC, quitC)
if chunk == nil {
@ -302,13 +302,13 @@ func (self *PyramidChunker) loadTree(chunkLevel [][]*TreeEntry, key Key, chunkC
}
//if data size is less than a chunk... add a parent with update as pending
if chunk.Size <= self.chunkSize {
if chunk.Size <= chunker.chunkSize {
newEntry := &TreeEntry{
level: 0,
branchCount: 1,
subtreeSize: uint64(chunk.Size),
chunk: make([]byte, self.chunkSize+8),
key: make([]byte, self.hashSize),
chunk: make([]byte, chunker.chunkSize+8),
key: make([]byte, chunker.hashSize),
index: 0,
updatePending: true,
}
@ -319,13 +319,13 @@ func (self *PyramidChunker) loadTree(chunkLevel [][]*TreeEntry, key Key, chunkC
var treeSize int64
var depth int
treeSize = self.chunkSize
for ; treeSize < chunk.Size; treeSize *= self.branches {
treeSize = chunker.chunkSize
for ; treeSize < chunk.Size; treeSize *= chunker.branches {
depth++
}
// Add the root chunk entry
branchCount := int64(len(chunk.SData)-8) / self.hashSize
branchCount := int64(len(chunk.SData)-8) / chunker.hashSize
newEntry := &TreeEntry{
level: depth - 1,
branchCount: branchCount,
@ -343,14 +343,14 @@ func (self *PyramidChunker) loadTree(chunkLevel [][]*TreeEntry, key Key, chunkC
//TODO(jmozah): instead of loading finished branches and then trim in the end,
//avoid loading them in the first place
for _, ent := range chunkLevel[lvl] {
branchCount = int64(len(ent.chunk)-8) / self.hashSize
branchCount = int64(len(ent.chunk)-8) / chunker.hashSize
for i := int64(0); i < branchCount; i++ {
key := ent.chunk[8+(i*self.hashSize) : 8+((i+1)*self.hashSize)]
key := ent.chunk[8+(i*chunker.hashSize) : 8+((i+1)*chunker.hashSize)]
newChunk := retrieve(key, chunkC, quitC)
if newChunk == nil {
return errLoadingTreeChunk
}
bewBranchCount := int64(len(newChunk.SData)-8) / self.hashSize
bewBranchCount := int64(len(newChunk.SData)-8) / chunker.hashSize
newEntry := &TreeEntry{
level: lvl - 1,
branchCount: bewBranchCount,
@ -365,7 +365,7 @@ func (self *PyramidChunker) loadTree(chunkLevel [][]*TreeEntry, key Key, chunkC
}
// We need to get only the right most unfinished branch.. so trim all finished branches
if int64(len(chunkLevel[lvl-1])) >= self.branches {
if int64(len(chunkLevel[lvl-1])) >= chunker.branches {
chunkLevel[lvl-1] = nil
}
}
@ -374,7 +374,7 @@ func (self *PyramidChunker) loadTree(chunkLevel [][]*TreeEntry, key Key, chunkC
return nil
}
func (self *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEntry, data io.Reader, rootKey []byte, quitC chan bool, wg *sync.WaitGroup, jobC chan *chunkJob, processorWG *sync.WaitGroup, chunkC chan *Chunk, errC chan error, storageWG *sync.WaitGroup) {
func (chunker *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEntry, data io.Reader, rootKey []byte, quitC chan bool, wg *sync.WaitGroup, jobC chan *chunkJob, processorWG *sync.WaitGroup, chunkC chan *Chunk, errC chan error, storageWG *sync.WaitGroup) {
defer wg.Done()
chunkWG := &sync.WaitGroup{}
@ -385,10 +385,10 @@ func (self *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEnt
processorWG.Add(1)
}
self.incrementWorkerCount()
go self.processor(self.workerCount, jobC, chunkC, errC, quitC, storageWG, processorWG)
chunker.incrementWorkerCount()
go chunker.processor(chunker.workerCount, jobC, chunkC, errC, quitC, storageWG, processorWG)
parent := NewTreeEntry(self)
parent := NewTreeEntry(chunker)
var unFinishedChunk *Chunk
if isAppend && len(chunkLevel[0]) != 0 {
@ -396,7 +396,7 @@ func (self *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEnt
lastIndex := len(chunkLevel[0]) - 1
ent := chunkLevel[0][lastIndex]
if ent.branchCount < self.branches {
if ent.branchCount < chunker.branches {
parent = &TreeEntry{
level: 0,
branchCount: ent.branchCount,
@ -408,10 +408,10 @@ func (self *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEnt
}
lastBranch := parent.branchCount - 1
lastKey := parent.chunk[8+lastBranch*self.hashSize : 8+(lastBranch+1)*self.hashSize]
lastKey := parent.chunk[8+lastBranch*chunker.hashSize : 8+(lastBranch+1)*chunker.hashSize]
unFinishedChunk = retrieve(lastKey, chunkC, quitC)
if unFinishedChunk.Size < self.chunkSize {
if unFinishedChunk.Size < chunker.chunkSize {
parent.subtreeSize = parent.subtreeSize - uint64(unFinishedChunk.Size)
parent.branchCount = parent.branchCount - 1
@ -425,7 +425,7 @@ func (self *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEnt
var n int
var err error
chunkData := make([]byte, self.chunkSize+8)
chunkData := make([]byte, chunker.chunkSize+8)
if unFinishedChunk != nil {
copy(chunkData, unFinishedChunk.SData)
n, err = data.Read(chunkData[8+unFinishedChunk.Size:])
@ -441,7 +441,7 @@ func (self *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEnt
if parent.branchCount == 1 {
// Data is exactly one chunk.. pick the last chunk key as root
chunkWG.Wait()
lastChunksKey := parent.chunk[8 : 8+self.hashSize]
lastChunksKey := parent.chunk[8 : 8+chunker.hashSize]
copy(rootKey, lastChunksKey)
break
}
@ -453,18 +453,18 @@ func (self *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEnt
// Data ended in chunk boundary.. just signal to start bulding tree
if n == 0 {
self.buildTree(isAppend, chunkLevel, parent, chunkWG, jobC, quitC, true, rootKey)
chunker.buildTree(isAppend, chunkLevel, parent, chunkWG, jobC, quitC, true, rootKey)
break
} else {
pkey := self.enqueueDataChunk(chunkData, uint64(n), parent, chunkWG, jobC, quitC)
pkey := chunker.enqueueDataChunk(chunkData, uint64(n), parent, chunkWG, jobC, quitC)
// update tree related parent data structures
parent.subtreeSize += uint64(n)
parent.branchCount++
// Data got exhausted... signal to send any parent tree related chunks
if int64(n) < self.chunkSize {
if int64(n) < chunker.chunkSize {
// only one data chunk .. so dont add any parent chunk
if parent.branchCount <= 1 {
@ -473,39 +473,39 @@ func (self *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEnt
break
}
self.buildTree(isAppend, chunkLevel, parent, chunkWG, jobC, quitC, true, rootKey)
chunker.buildTree(isAppend, chunkLevel, parent, chunkWG, jobC, quitC, true, rootKey)
break
}
if parent.branchCount == self.branches {
self.buildTree(isAppend, chunkLevel, parent, chunkWG, jobC, quitC, false, rootKey)
parent = NewTreeEntry(self)
if parent.branchCount == chunker.branches {
chunker.buildTree(isAppend, chunkLevel, parent, chunkWG, jobC, quitC, false, rootKey)
parent = NewTreeEntry(chunker)
}
}
workers := self.getWorkerCount()
workers := chunker.getWorkerCount()
if int64(len(jobC)) > workers && workers < ChunkProcessors {
if processorWG != nil {
processorWG.Add(1)
}
self.incrementWorkerCount()
go self.processor(self.workerCount, jobC, chunkC, errC, quitC, storageWG, processorWG)
chunker.incrementWorkerCount()
go chunker.processor(chunker.workerCount, jobC, chunkC, errC, quitC, storageWG, processorWG)
}
}
}
func (self *PyramidChunker) buildTree(isAppend bool, chunkLevel [][]*TreeEntry, ent *TreeEntry, chunkWG *sync.WaitGroup, jobC chan *chunkJob, quitC chan bool, last bool, rootKey []byte) {
func (chunker *PyramidChunker) buildTree(isAppend bool, chunkLevel [][]*TreeEntry, ent *TreeEntry, chunkWG *sync.WaitGroup, jobC chan *chunkJob, quitC chan bool, last bool, rootKey []byte) {
chunkWG.Wait()
self.enqueueTreeChunk(chunkLevel, ent, chunkWG, jobC, quitC, last)
chunker.enqueueTreeChunk(chunkLevel, ent, chunkWG, jobC, quitC, last)
compress := false
endLvl := self.branches
for lvl := int64(0); lvl < self.branches; lvl++ {
endLvl := chunker.branches
for lvl := int64(0); lvl < chunker.branches; lvl++ {
lvlCount := int64(len(chunkLevel[lvl]))
if lvlCount >= self.branches {
if lvlCount >= chunker.branches {
endLvl = lvl + 1
compress = true
break
@ -527,9 +527,9 @@ func (self *PyramidChunker) buildTree(isAppend bool, chunkLevel [][]*TreeEntry,
return
}
for startCount := int64(0); startCount < lvlCount; startCount += self.branches {
for startCount := int64(0); startCount < lvlCount; startCount += chunker.branches {
endCount := startCount + self.branches
endCount := startCount + chunker.branches
if endCount > lvlCount {
endCount = lvlCount
}
@ -545,18 +545,18 @@ func (self *PyramidChunker) buildTree(isAppend bool, chunkLevel [][]*TreeEntry,
level: int(lvl + 1),
branchCount: 0,
subtreeSize: 0,
chunk: make([]byte, self.chunkSize+8),
key: make([]byte, self.hashSize),
chunk: make([]byte, chunker.chunkSize+8),
key: make([]byte, chunker.hashSize),
index: int(nextLvlCount),
updatePending: true,
}
for index := int64(0); index < lvlCount; index++ {
updateEntry.branchCount++
updateEntry.subtreeSize += chunkLevel[lvl][index].subtreeSize
copy(updateEntry.chunk[8+(index*self.hashSize):8+((index+1)*self.hashSize)], chunkLevel[lvl][index].key[:self.hashSize])
copy(updateEntry.chunk[8+(index*chunker.hashSize):8+((index+1)*chunker.hashSize)], chunkLevel[lvl][index].key[:chunker.hashSize])
}
self.enqueueTreeChunk(chunkLevel, updateEntry, chunkWG, jobC, quitC, last)
chunker.enqueueTreeChunk(chunkLevel, updateEntry, chunkWG, jobC, quitC, last)
} else {
@ -565,8 +565,8 @@ func (self *PyramidChunker) buildTree(isAppend bool, chunkLevel [][]*TreeEntry,
level: int(lvl + 1),
branchCount: noOfBranches,
subtreeSize: 0,
chunk: make([]byte, (noOfBranches*self.hashSize)+8),
key: make([]byte, self.hashSize),
chunk: make([]byte, (noOfBranches*chunker.hashSize)+8),
key: make([]byte, chunker.hashSize),
index: int(nextLvlCount),
updatePending: false,
}
@ -575,11 +575,11 @@ func (self *PyramidChunker) buildTree(isAppend bool, chunkLevel [][]*TreeEntry,
for i := startCount; i < endCount; i++ {
entry := chunkLevel[lvl][i]
newEntry.subtreeSize += entry.subtreeSize
copy(newEntry.chunk[8+(index*self.hashSize):8+((index+1)*self.hashSize)], entry.key[:self.hashSize])
copy(newEntry.chunk[8+(index*chunker.hashSize):8+((index+1)*chunker.hashSize)], entry.key[:chunker.hashSize])
index++
}
self.enqueueTreeChunk(chunkLevel, newEntry, chunkWG, jobC, quitC, last)
chunker.enqueueTreeChunk(chunkLevel, newEntry, chunkWG, jobC, quitC, last)
}
@ -595,7 +595,7 @@ func (self *PyramidChunker) buildTree(isAppend bool, chunkLevel [][]*TreeEntry,
}
func (self *PyramidChunker) enqueueTreeChunk(chunkLevel [][]*TreeEntry, ent *TreeEntry, chunkWG *sync.WaitGroup, jobC chan *chunkJob, quitC chan bool, last bool) {
func (chunker *PyramidChunker) enqueueTreeChunk(chunkLevel [][]*TreeEntry, ent *TreeEntry, chunkWG *sync.WaitGroup, jobC chan *chunkJob, quitC chan bool, last bool) {
if ent != nil {
// wait for data chunks to get over before processing the tree chunk
@ -604,10 +604,10 @@ func (self *PyramidChunker) enqueueTreeChunk(chunkLevel [][]*TreeEntry, ent *Tre
}
binary.LittleEndian.PutUint64(ent.chunk[:8], ent.subtreeSize)
ent.key = make([]byte, self.hashSize)
ent.key = make([]byte, chunker.hashSize)
chunkWG.Add(1)
select {
case jobC <- &chunkJob{ent.key, ent.chunk[:ent.branchCount*self.hashSize+8], int64(ent.subtreeSize), chunkWG, TreeChunk, 0}:
case jobC <- &chunkJob{ent.key, ent.chunk[:ent.branchCount*chunker.hashSize+8], int64(ent.subtreeSize), chunkWG, TreeChunk, 0}:
case <-quitC:
}
@ -622,9 +622,9 @@ func (self *PyramidChunker) enqueueTreeChunk(chunkLevel [][]*TreeEntry, ent *Tre
}
}
func (self *PyramidChunker) enqueueDataChunk(chunkData []byte, size uint64, parent *TreeEntry, chunkWG *sync.WaitGroup, jobC chan *chunkJob, quitC chan bool) Key {
func (chunker *PyramidChunker) enqueueDataChunk(chunkData []byte, size uint64, parent *TreeEntry, chunkWG *sync.WaitGroup, jobC chan *chunkJob, quitC chan bool) Key {
binary.LittleEndian.PutUint64(chunkData[:8], size)
pkey := parent.chunk[8+parent.branchCount*self.hashSize : 8+(parent.branchCount+1)*self.hashSize]
pkey := parent.chunk[8+parent.branchCount*chunker.hashSize : 8+(parent.branchCount+1)*chunker.hashSize]
chunkWG.Add(1)
select {

6
swarm/storage/swarmhasher.go
View file

@ -34,7 +34,7 @@ type HashWithLength struct {
hash.Hash
}
func (self *HashWithLength) ResetWithLength(length []byte) {
self.Reset()
self.Write(length)
func (h *HashWithLength) ResetWithLength(length []byte) {
h.Reset()
h.Write(length)
}

4
swarm/storage/types.go
View file

@ -243,6 +243,6 @@ type LazyTestSectionReader struct {
*io.SectionReader
}
func (self *LazyTestSectionReader) Size(chan bool) (int64, error) {
return self.SectionReader.Size(), nil
func (reader *LazyTestSectionReader) Size(chan bool) (int64, error) {
return reader.SectionReader.Size(), nil
}

130
swarm/swarm.go
View file

@ -82,17 +82,17 @@ type SwarmAPI struct {
PrvKey *ecdsa.PrivateKey
}
func (self *Swarm) API() *SwarmAPI {
func (s *Swarm) API() *SwarmAPI {
return &SwarmAPI{
Api: self.api,
Backend: self.backend,
PrvKey: self.privateKey,
Api: s.api,
Backend: s.backend,
PrvKey: s.privateKey,
}
}
// creates a new swarm service instance
// implements node.Service
func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api.Config) (self *Swarm, err error) {
func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api.Config) (s *Swarm, err error) {
if bytes.Equal(common.FromHex(config.PublicKey), storage.ZeroKey) {
return nil, fmt.Errorf("empty public key")
}
@ -100,7 +100,7 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api.
return nil, fmt.Errorf("empty bzz key")
}
self = &Swarm{
s = &Swarm{
config: config,
swapEnabled: config.SwapEnabled,
backend: backend,
@ -110,7 +110,7 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api.
log.Debug(fmt.Sprintf("Setting up Swarm service components"))
hash := storage.MakeHashFunc(config.ChunkerParams.Hash)
self.lstore, err = storage.NewLocalStore(hash, config.StoreParams)
s.lstore, err = storage.NewLocalStore(hash, config.StoreParams)
if err != nil {
return
}
@ -118,12 +118,12 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api.
// setup local store
log.Debug(fmt.Sprintf("Set up local storage"))
self.dbAccess = network.NewDbAccess(self.lstore)
s.dbAccess = network.NewDbAccess(s.lstore)
log.Debug(fmt.Sprintf("Set up local db access (iterator/counter)"))
// set up the kademlia hive
self.hive = network.NewHive(
common.HexToHash(self.config.BzzKey), // key to hive (kademlia base address)
s.hive = network.NewHive(
common.HexToHash(s.config.BzzKey), // key to hive (kademlia base address)
config.HiveParams, // configuration parameters
config.SwapEnabled, // SWAP enabled
config.SyncEnabled, // syncronisation enabled
@ -131,22 +131,22 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api.
log.Debug(fmt.Sprintf("Set up swarm network with Kademlia hive"))
// setup cloud storage backend
self.cloud = network.NewForwarder(self.hive)
s.cloud = network.NewForwarder(s.hive)
log.Debug(fmt.Sprintf("-> set swarm forwarder as cloud storage backend"))
// setup cloud storage internal access layer
self.storage = storage.NewNetStore(hash, self.lstore, self.cloud, config.StoreParams)
s.storage = storage.NewNetStore(hash, s.lstore, s.cloud, config.StoreParams)
log.Debug(fmt.Sprintf("-> swarm net store shared access layer to Swarm Chunk Store"))
// set up Depo (storage handler = cloud storage access layer for incoming remote requests)
self.depo = network.NewDepo(hash, self.lstore, self.storage)
s.depo = network.NewDepo(hash, s.lstore, s.storage)
log.Debug(fmt.Sprintf("-> REmote Access to CHunks"))
// set up DPA, the cloud storage local access layer
dpaChunkStore := storage.NewDpaChunkStore(self.lstore, self.storage)
dpaChunkStore := storage.NewDpaChunkStore(s.lstore, s.storage)
log.Debug(fmt.Sprintf("-> Local Access to Swarm"))
// Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage
self.dpa = storage.NewDPA(dpaChunkStore, self.config.ChunkerParams)
s.dpa = storage.NewDPA(dpaChunkStore, s.config.ChunkerParams)
log.Debug(fmt.Sprintf("-> Content Store API"))
if len(config.EnsAPIs) > 0 {
@ -159,17 +159,17 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api.
}
opts = append(opts, api.MultiResolverOptionWithResolver(r, tld))
}
self.dns = api.NewMultiResolver(opts...)
s.dns = api.NewMultiResolver(opts...)
}
self.api = api.NewApi(self.dpa, self.dns)
s.api = api.NewApi(s.dpa, s.dns)
// Manifests for Smart Hosting
log.Debug(fmt.Sprintf("-> Web3 virtual server API"))
self.sfs = fuse.NewSwarmFS(self.api)
s.sfs = fuse.NewSwarmFS(s.api)
log.Debug("-> Initializing Fuse file system")
return self, nil
return s, nil
}
// parseEnsAPIAddress parses string according to format
@ -272,7 +272,7 @@ Start is called when the stack is started
* TODO: start subservices like sword, swear, swarmdns
*/
// implements the node.Service interface
func (self *Swarm) Start(srv *p2p.Server) error {
func (s *Swarm) Start(srv *p2p.Server) error {
startTime = time.Now()
connectPeer := func(url string) error {
node, err := discover.ParseNode(url)
@ -283,85 +283,85 @@ func (self *Swarm) Start(srv *p2p.Server) error {
return nil
}
// set chequebook
if self.swapEnabled {
if s.swapEnabled {
ctx := context.Background() // The initial setup has no deadline.
err := self.SetChequebook(ctx)
err := s.SetChequebook(ctx)
if err != nil {
return fmt.Errorf("Unable to set chequebook for SWAP: %v", err)
}
log.Debug(fmt.Sprintf("-> cheque book for SWAP: %v", self.config.Swap.Chequebook()))
log.Debug(fmt.Sprintf("-> cheque book for SWAP: %v", s.config.Swap.Chequebook()))
} else {
log.Debug(fmt.Sprintf("SWAP disabled: no cheque book set"))
}
log.Warn(fmt.Sprintf("Starting Swarm service"))
self.hive.Start(
s.hive.Start(
discover.PubkeyID(&srv.PrivateKey.PublicKey),
func() string { return srv.ListenAddr },
connectPeer,
)
log.Info(fmt.Sprintf("Swarm network started on bzz address: %v", self.hive.Addr()))
log.Info(fmt.Sprintf("Swarm network started on bzz address: %v", s.hive.Addr()))
self.dpa.Start()
s.dpa.Start()
log.Debug(fmt.Sprintf("Swarm DPA started"))
// start swarm http proxy server
if self.config.Port != "" {
addr := net.JoinHostPort(self.config.ListenAddr, self.config.Port)
go httpapi.StartHttpServer(self.api, &httpapi.ServerConfig{
if s.config.Port != "" {
addr := net.JoinHostPort(s.config.ListenAddr, s.config.Port)
go httpapi.StartHttpServer(s.api, &httpapi.ServerConfig{
Addr: addr,
CorsString: self.corsString,
CorsString: s.corsString,
})
log.Info(fmt.Sprintf("Swarm http proxy started on %v", addr))
if self.corsString != "" {
log.Debug(fmt.Sprintf("Swarm http proxy started with corsdomain: %v", self.corsString))
if s.corsString != "" {
log.Debug(fmt.Sprintf("Swarm http proxy started with corsdomain: %v", s.corsString))
}
}
self.periodicallyUpdateGauges()
s.periodicallyUpdateGauges()
startCounter.Inc(1)
return nil
}
func (self *Swarm) periodicallyUpdateGauges() {
func (s *Swarm) periodicallyUpdateGauges() {
ticker := time.NewTicker(updateGaugesPeriod)
go func() {
for range ticker.C {
self.updateGauges()
s.updateGauges()
}
}()
}
func (self *Swarm) updateGauges() {
dbSizeGauge.Update(int64(self.lstore.DbCounter()))
cacheSizeGauge.Update(int64(self.lstore.CacheCounter()))
func (s *Swarm) updateGauges() {
dbSizeGauge.Update(int64(s.lstore.DbCounter()))
cacheSizeGauge.Update(int64(s.lstore.CacheCounter()))
uptimeGauge.Update(time.Since(startTime).Nanoseconds())
}
// implements the node.Service interface
// stops all component services.
func (self *Swarm) Stop() error {
self.dpa.Stop()
err := self.hive.Stop()
if ch := self.config.Swap.Chequebook(); ch != nil {
func (s *Swarm) Stop() error {
s.dpa.Stop()
err := s.hive.Stop()
if ch := s.config.Swap.Chequebook(); ch != nil {
ch.Stop()
ch.Save()
}
if self.lstore != nil {
self.lstore.DbStore.Close()
if s.lstore != nil {
s.lstore.DbStore.Close()
}
self.sfs.Stop()
s.sfs.Stop()
stopCounter.Inc(1)
return err
}
// implements the node.Service interface
func (self *Swarm) Protocols() []p2p.Protocol {
proto, err := network.Bzz(self.depo, self.backend, self.hive, self.dbAccess, self.config.Swap, self.config.SyncParams, self.config.NetworkId)
func (s *Swarm) Protocols() []p2p.Protocol {
proto, err := network.Bzz(s.depo, s.backend, s.hive, s.dbAccess, s.config.Swap, s.config.SyncParams, s.config.NetworkId)
if err != nil {
return nil
}
@ -370,32 +370,32 @@ func (self *Swarm) Protocols() []p2p.Protocol {
// implements node.Service
// Apis returns the RPC Api descriptors the Swarm implementation offers
func (self *Swarm) APIs() []rpc.API {
func (s *Swarm) APIs() []rpc.API {
return []rpc.API{
// public APIs
{
Namespace: "bzz",
Version: "0.1",
Service: &Info{self.config, chequebook.ContractParams},
Service: &Info{s.config, chequebook.ContractParams},
Public: true,
},
// admin APIs
{
Namespace: "bzz",
Version: "0.1",
Service: api.NewControl(self.api, self.hive),
Service: api.NewControl(s.api, s.hive),
Public: false,
},
{
Namespace: "chequebook",
Version: chequebook.Version,
Service: chequebook.NewApi(self.config.Swap.Chequebook),
Service: chequebook.NewApi(s.config.Swap.Chequebook),
Public: false,
},
{
Namespace: "swarmfs",
Version: fuse.Swarmfs_Version,
Service: self.sfs,
Service: s.sfs,
Public: false,
},
// storage APIs
@ -403,36 +403,36 @@ func (self *Swarm) APIs() []rpc.API {
{
Namespace: "bzz",
Version: "0.1",
Service: api.NewStorage(self.api),
Service: api.NewStorage(s.api),
Public: true,
},
{
Namespace: "bzz",
Version: "0.1",
Service: api.NewFileSystem(self.api),
Service: api.NewFileSystem(s.api),
Public: false,
},
// {Namespace, Version, api.NewAdmin(self), false},
// {Namespace, Version, api.NewAdmin(s), false},
}
}
func (self *Swarm) Api() *api.Api {
return self.api
func (s *Swarm) Api() *api.Api {
return s.api
}
// SetChequebook ensures that the local checquebook is set up on chain.
func (self *Swarm) SetChequebook(ctx context.Context) error {
err := self.config.Swap.SetChequebook(ctx, self.backend, self.config.Path)
func (s *Swarm) SetChequebook(ctx context.Context) error {
err := s.config.Swap.SetChequebook(ctx, s.backend, s.config.Path)
if err != nil {
return err
}
log.Info(fmt.Sprintf("new chequebook set (%v): saving config file, resetting all connections in the hive", self.config.Swap.Contract.Hex()))
self.hive.DropAll()
log.Info(fmt.Sprintf("new chequebook set (%v): saving config file, resetting all connections in the hive", s.config.Swap.Contract.Hex()))
s.hive.DropAll()
return nil
}
// Local swarm without netStore
func NewLocalSwarm(datadir, port string) (self *Swarm, err error) {
func NewLocalSwarm(datadir, port string) (s *Swarm, err error) {
prvKey, err := crypto.GenerateKey()
if err != nil {
@ -449,7 +449,7 @@ func NewLocalSwarm(datadir, port string) (self *Swarm, err error) {
return
}
self = &Swarm{
s = &Swarm{
api: api.NewApi(dpa, nil),
config: config,
}
@ -463,6 +463,6 @@ type Info struct {
*chequebook.Params
}
func (self *Info) Info() *Info {
return self
func (s *Info) Info() *Info {
return s
}