forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-06-21 22:24:32 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

remove swarm as unused like eth

Browse source
This commit is contained in:
Liam Lai 2024-05-13 21:27:14 +08:00
parent 6832f228fa
commit 288e4c46a5
81 changed files with 0 additions and 18212 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

492
swarm/api/api.go
View file

@ -1,492 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"fmt"
"io"
"net/http"
"path"
"regexp"
"strings"
"sync"
"bytes"
"mime"
"path/filepath"
"time"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/metrics"
"github.com/XinFinOrg/XDPoSChain/swarm/storage"
)
var hashMatcher = regexp.MustCompile("^[0-9A-Fa-f]{64}")
//setup metrics
var (
apiResolveCount = metrics.NewRegisteredCounter("api.resolve.count", nil)
apiResolveFail = metrics.NewRegisteredCounter("api.resolve.fail", nil)
apiPutCount = metrics.NewRegisteredCounter("api.put.count", nil)
apiPutFail = metrics.NewRegisteredCounter("api.put.fail", nil)
apiGetCount = metrics.NewRegisteredCounter("api.get.count", nil)
apiGetNotFound = metrics.NewRegisteredCounter("api.get.notfound", nil)
apiGetHttp300 = metrics.NewRegisteredCounter("api.get.http.300", nil)
apiModifyCount = metrics.NewRegisteredCounter("api.modify.count", nil)
apiModifyFail = metrics.NewRegisteredCounter("api.modify.fail", nil)
apiAddFileCount = metrics.NewRegisteredCounter("api.addfile.count", nil)
apiAddFileFail = metrics.NewRegisteredCounter("api.addfile.fail", nil)
apiRmFileCount = metrics.NewRegisteredCounter("api.removefile.count", nil)
apiRmFileFail = metrics.NewRegisteredCounter("api.removefile.fail", nil)
apiAppendFileCount = metrics.NewRegisteredCounter("api.appendfile.count", nil)
apiAppendFileFail = metrics.NewRegisteredCounter("api.appendfile.fail", nil)
)
type Resolver interface {
Resolve(string) (common.Hash, error)
}
// NoResolverError is returned by MultiResolver.Resolve if no resolver
// can be found for the address.
type NoResolverError struct {
TLD string
}
func NewNoResolverError(tld string) *NoResolverError {
return &NoResolverError{TLD: tld}
}
func (e *NoResolverError) Error() string {
if e.TLD == "" {
return "no ENS resolver"
}
return fmt.Sprintf("no ENS endpoint configured to resolve .%s TLD names", e.TLD)
}
// MultiResolver is used to resolve URL addresses based on their TLDs.
// Each TLD can have multiple resolvers, and the resoluton from the
// first one in the sequence will be returned.
type MultiResolver struct {
resolvers map[string][]Resolver
}
// MultiResolverOption sets options for MultiResolver and is used as
// arguments for its constructor.
type MultiResolverOption func(*MultiResolver)
// MultiResolverOptionWithResolver adds a Resolver to a list of resolvers
// for a specific TLD. If TLD is an empty string, the resolver will be added
// to the list of default resolver, the ones that will be used for resolution
// of addresses which do not have their TLD resolver specified.
func MultiResolverOptionWithResolver(r Resolver, tld string) MultiResolverOption {
return func(m *MultiResolver) {
m.resolvers[tld] = append(m.resolvers[tld], r)
}
}
// NewMultiResolver creates a new instance of MultiResolver.
func NewMultiResolver(opts ...MultiResolverOption) (m *MultiResolver) {
m = &MultiResolver{
resolvers: make(map[string][]Resolver),
}
for _, o := range opts {
o(m)
}
return m
}
// Resolve resolves address by choosing a Resolver by TLD.
// If there are more default Resolvers, or for a specific TLD,
// the Hash from the the first one which does not return error
// will be returned.
func (m MultiResolver) Resolve(addr string) (h common.Hash, err error) {
rs := m.resolvers[""]
tld := path.Ext(addr)
if tld != "" {
tld = tld[1:]
rstld, ok := m.resolvers[tld]
if ok {
rs = rstld
}
}
if rs == nil {
return h, NewNoResolverError(tld)
}
for _, r := range rs {
h, err = r.Resolve(addr)
if err == nil {
return
}
}
return
}
/*
Api implements webserver/file system related content storage and retrieval
on top of the dpa
it is the public interface of the dpa which is included in the ethereum stack
*/
type Api struct {
dpa *storage.DPA
dns Resolver
}
//the api constructor initialises
func NewApi(dpa *storage.DPA, dns Resolver) (self *Api) {
self = &Api{
dpa: dpa,
dns: dns,
}
return
}
// to be used only in TEST
func (self *Api) Upload(uploadDir, index string) (hash string, err error) {
fs := NewFileSystem(self)
hash, err = fs.Upload(uploadDir, index)
return hash, err
}
// DPA reader API
func (self *Api) Retrieve(key storage.Key) storage.LazySectionReader {
return self.dpa.Retrieve(key)
}
func (self *Api) Store(data io.Reader, size int64, wg *sync.WaitGroup) (key storage.Key, err error) {
return self.dpa.Store(data, size, wg, nil)
}
type ErrResolve error
// DNS Resolver
func (self *Api) Resolve(uri *URI) (storage.Key, error) {
apiResolveCount.Inc(1)
log.Trace(fmt.Sprintf("Resolving : %v", uri.Addr))
// if the URI is immutable, check if the address is a hash
isHash := hashMatcher.MatchString(uri.Addr)
if uri.Immutable() || uri.DeprecatedImmutable() {
if !isHash {
return nil, fmt.Errorf("immutable address not a content hash: %q", uri.Addr)
}
return common.Hex2Bytes(uri.Addr), nil
}
// if DNS is not configured, check if the address is a hash
if self.dns == nil {
if !isHash {
apiResolveFail.Inc(1)
return nil, fmt.Errorf("no DNS to resolve name: %q", uri.Addr)
}
return common.Hex2Bytes(uri.Addr), nil
}
// try and resolve the address
resolved, err := self.dns.Resolve(uri.Addr)
if err == nil {
return resolved[:], nil
} else if !isHash {
apiResolveFail.Inc(1)
return nil, err
}
return common.Hex2Bytes(uri.Addr), nil
}
// Put provides singleton manifest creation on top of dpa store
func (self *Api) Put(content, contentType string) (storage.Key, error) {
apiPutCount.Inc(1)
r := strings.NewReader(content)
wg := &sync.WaitGroup{}
key, err := self.dpa.Store(r, int64(len(content)), wg, nil)
if err != nil {
apiPutFail.Inc(1)
return nil, err
}
manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType)
r = strings.NewReader(manifest)
key, err = self.dpa.Store(r, int64(len(manifest)), wg, nil)
if err != nil {
apiPutFail.Inc(1)
return nil, err
}
wg.Wait()
return key, nil
}
// Get uses iterative manifest retrieval and prefix matching
// to resolve basePath to content using dpa retrieve
// it returns a section reader, mimeType, status and an error
func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionReader, mimeType string, status int, err error) {
apiGetCount.Inc(1)
trie, err := loadManifest(self.dpa, key, nil)
if err != nil {
apiGetNotFound.Inc(1)
status = http.StatusNotFound
log.Warn(fmt.Sprintf("loadManifestTrie error: %v", err))
return
}
log.Trace(fmt.Sprintf("getEntry(%s)", path))
entry, _ := trie.getEntry(path)
if entry != nil {
key = common.Hex2Bytes(entry.Hash)
status = entry.Status
if status == http.StatusMultipleChoices {
apiGetHttp300.Inc(1)
return
} else {
mimeType = entry.ContentType
log.Trace(fmt.Sprintf("content lookup key: '%v' (%v)", key, mimeType))
reader = self.dpa.Retrieve(key)
}
} else {
status = http.StatusNotFound
apiGetNotFound.Inc(1)
err = fmt.Errorf("manifest entry for '%s' not found", path)
log.Warn(fmt.Sprintf("%v", err))
}
return
}
func (self *Api) Modify(key storage.Key, path, contentHash, contentType string) (storage.Key, error) {
apiModifyCount.Inc(1)
quitC := make(chan bool)
trie, err := loadManifest(self.dpa, key, quitC)
if err != nil {
apiModifyFail.Inc(1)
return nil, err
}
if contentHash != "" {
entry := newManifestTrieEntry(&ManifestEntry{
Path: path,
ContentType: contentType,
}, nil)
entry.Hash = contentHash
trie.addEntry(entry, quitC)
} else {
trie.deleteEntry(path, quitC)
}
if err := trie.recalcAndStore(); err != nil {
apiModifyFail.Inc(1)
return nil, err
}
return trie.hash, nil
}
func (self *Api) AddFile(mhash, path, fname string, content []byte, nameresolver bool) (storage.Key, string, error) {
apiAddFileCount.Inc(1)
uri, err := Parse("bzz:/" + mhash)
if err != nil {
apiAddFileFail.Inc(1)
return nil, "", err
}
mkey, err := self.Resolve(uri)
if err != nil {
apiAddFileFail.Inc(1)
return nil, "", err
}
// trim the root dir we added
if path[:1] == "/" {
path = path[1:]
}
entry := &ManifestEntry{
Path: filepath.Join(path, fname),
ContentType: mime.TypeByExtension(filepath.Ext(fname)),
Mode: 0700,
Size: int64(len(content)),
ModTime: time.Now(),
}
mw, err := self.NewManifestWriter(mkey, nil)
if err != nil {
apiAddFileFail.Inc(1)
return nil, "", err
}
fkey, err := mw.AddEntry(bytes.NewReader(content), entry)
if err != nil {
apiAddFileFail.Inc(1)
return nil, "", err
}
newMkey, err := mw.Store()
if err != nil {
apiAddFileFail.Inc(1)
return nil, "", err
}
return fkey, newMkey.String(), nil
}
func (self *Api) RemoveFile(mhash, path, fname string, nameresolver bool) (string, error) {
apiRmFileCount.Inc(1)
uri, err := Parse("bzz:/" + mhash)
if err != nil {
apiRmFileFail.Inc(1)
return "", err
}
mkey, err := self.Resolve(uri)
if err != nil {
apiRmFileFail.Inc(1)
return "", err
}
// trim the root dir we added
if path[:1] == "/" {
path = path[1:]
}
mw, err := self.NewManifestWriter(mkey, nil)
if err != nil {
apiRmFileFail.Inc(1)
return "", err
}
err = mw.RemoveEntry(filepath.Join(path, fname))
if err != nil {
apiRmFileFail.Inc(1)
return "", err
}
newMkey, err := mw.Store()
if err != nil {
apiRmFileFail.Inc(1)
return "", err
}
return newMkey.String(), nil
}
func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, content []byte, oldKey storage.Key, offset int64, addSize int64, nameresolver bool) (storage.Key, string, error) {
apiAppendFileCount.Inc(1)
buffSize := offset + addSize
if buffSize < existingSize {
buffSize = existingSize
}
buf := make([]byte, buffSize)
oldReader := self.Retrieve(oldKey)
io.ReadAtLeast(oldReader, buf, int(offset))
newReader := bytes.NewReader(content)
io.ReadAtLeast(newReader, buf[offset:], int(addSize))
if buffSize < existingSize {
io.ReadAtLeast(oldReader, buf[addSize:], int(buffSize))
}
combinedReader := bytes.NewReader(buf)
totalSize := int64(len(buf))
// TODO(jmozah): to append using pyramid chunker when it is ready
//oldReader := self.Retrieve(oldKey)
//newReader := bytes.NewReader(content)
//combinedReader := io.MultiReader(oldReader, newReader)
uri, err := Parse("bzz:/" + mhash)
if err != nil {
apiAppendFileFail.Inc(1)
return nil, "", err
}
mkey, err := self.Resolve(uri)
if err != nil {
apiAppendFileFail.Inc(1)
return nil, "", err
}
// trim the root dir we added
if path[:1] == "/" {
path = path[1:]
}
mw, err := self.NewManifestWriter(mkey, nil)
if err != nil {
apiAppendFileFail.Inc(1)
return nil, "", err
}
err = mw.RemoveEntry(filepath.Join(path, fname))
if err != nil {
apiAppendFileFail.Inc(1)
return nil, "", err
}
entry := &ManifestEntry{
Path: filepath.Join(path, fname),
ContentType: mime.TypeByExtension(filepath.Ext(fname)),
Mode: 0700,
Size: totalSize,
ModTime: time.Now(),
}
fkey, err := mw.AddEntry(io.Reader(combinedReader), entry)
if err != nil {
apiAppendFileFail.Inc(1)
return nil, "", err
}
newMkey, err := mw.Store()
if err != nil {
apiAppendFileFail.Inc(1)
return nil, "", err
}
return fkey, newMkey.String(), nil
}
func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storage.Key, manifestEntryMap map[string]*manifestTrieEntry, err error) {
uri, err := Parse("bzz:/" + mhash)
if err != nil {
return nil, nil, err
}
key, err = self.Resolve(uri)
if err != nil {
return nil, nil, err
}
quitC := make(chan bool)
rootTrie, err := loadManifest(self.dpa, key, quitC)
if err != nil {
return nil, nil, fmt.Errorf("can't load manifest %v: %v", key.String(), err)
}
manifestEntryMap = map[string]*manifestTrieEntry{}
err = rootTrie.listWithPrefix(uri.Path, quitC, func(entry *manifestTrieEntry, suffix string) {
manifestEntryMap[suffix] = entry
})
if err != nil {
return nil, nil, fmt.Errorf("list with prefix failed %v: %v", key.String(), err)
}
return key, manifestEntryMap, nil
}

363
swarm/api/api_test.go
View file

@ -1,363 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"errors"
"fmt"
"io"
"os"
"testing"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/swarm/storage"
)
func testApi(t *testing.T, f func(*Api)) {
datadir, err := os.MkdirTemp("", "bzz-test")
if err != nil {
t.Fatalf("unable to create temp dir: %v", err)
}
os.RemoveAll(datadir)
defer os.RemoveAll(datadir)
dpa, err := storage.NewLocalDPA(datadir)
if err != nil {
return
}
api := NewApi(dpa, nil)
dpa.Start()
f(api)
dpa.Stop()
}
type testResponse struct {
reader storage.LazySectionReader
*Response
}
func checkResponse(t *testing.T, resp *testResponse, exp *Response) {
if resp.MimeType != exp.MimeType {
t.Errorf("incorrect mimeType. expected '%s', got '%s'", exp.MimeType, resp.MimeType)
}
if resp.Status != exp.Status {
t.Errorf("incorrect status. expected '%d', got '%d'", exp.Status, resp.Status)
}
if resp.Size != exp.Size {
t.Errorf("incorrect size. expected '%d', got '%d'", exp.Size, resp.Size)
}
if resp.reader != nil {
content := make([]byte, resp.Size)
read, _ := resp.reader.Read(content)
if int64(read) != exp.Size {
t.Errorf("incorrect content length. expected '%d...', got '%d...'", read, exp.Size)
}
resp.Content = string(content)
}
if resp.Content != exp.Content {
// if !bytes.Equal(resp.Content, exp.Content)
t.Errorf("incorrect content. expected '%s...', got '%s...'", string(exp.Content), string(resp.Content))
}
}
// func expResponse(content []byte, mimeType string, status int) *Response {
func expResponse(content string, mimeType string, status int) *Response {
log.Trace(fmt.Sprintf("expected content (%v): %v ", len(content), content))
return &Response{mimeType, status, int64(len(content)), content}
}
// func testGet(t *testing.T, api *Api, bzzhash string) *testResponse {
func testGet(t *testing.T, api *Api, bzzhash, path string) *testResponse {
key := storage.Key(common.Hex2Bytes(bzzhash))
reader, mimeType, status, err := api.Get(key, path)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
quitC := make(chan bool)
size, err := reader.Size(quitC)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
log.Trace(fmt.Sprintf("reader size: %v ", size))
s := make([]byte, size)
_, err = reader.Read(s)
if err != io.EOF {
t.Fatalf("unexpected error: %v", err)
}
reader.Seek(0, 0)
return &testResponse{reader, &Response{mimeType, status, size, string(s)}}
// return &testResponse{reader, &Response{mimeType, status, reader.Size(), nil}}
}
func TestApiPut(t *testing.T) {
testApi(t, func(api *Api) {
content := "hello"
exp := expResponse(content, "text/plain", 0)
// exp := expResponse([]byte(content), "text/plain", 0)
key, err := api.Put(content, exp.MimeType)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
resp := testGet(t, api, key.String(), "")
checkResponse(t, resp, exp)
})
}
// testResolver implements the Resolver interface and either returns the given
// hash if it is set, or returns a "name not found" error
type testResolver struct {
hash *common.Hash
}
func newTestResolver(addr string) *testResolver {
r := &testResolver{}
if addr != "" {
hash := common.HexToHash(addr)
r.hash = &hash
}
return r
}
func (t *testResolver) Resolve(addr string) (common.Hash, error) {
if t.hash == nil {
return common.Hash{}, fmt.Errorf("DNS name not found: %q", addr)
}
return *t.hash, nil
}
// TestAPIResolve tests resolving URIs which can either contain content hashes
// or ENS names
func TestAPIResolve(t *testing.T) {
ensAddr := "swarm.eth"
hashAddr := "1111111111111111111111111111111111111111111111111111111111111111"
resolvedAddr := "2222222222222222222222222222222222222222222222222222222222222222"
doesResolve := newTestResolver(resolvedAddr)
doesntResolve := newTestResolver("")
type test struct {
desc string
dns Resolver
addr string
immutable bool
result string
expectErr error
}
tests := []*test{
{
desc: "DNS not configured, hash address, returns hash address",
dns: nil,
addr: hashAddr,
result: hashAddr,
},
{
desc: "DNS not configured, ENS address, returns error",
dns: nil,
addr: ensAddr,
expectErr: errors.New(`no DNS to resolve name: "swarm.eth"`),
},
{
desc: "DNS configured, hash address, hash resolves, returns resolved address",
dns: doesResolve,
addr: hashAddr,
result: resolvedAddr,
},
{
desc: "DNS configured, immutable hash address, hash resolves, returns hash address",
dns: doesResolve,
addr: hashAddr,
immutable: true,
result: hashAddr,
},
{
desc: "DNS configured, hash address, hash doesn't resolve, returns hash address",
dns: doesntResolve,
addr: hashAddr,
result: hashAddr,
},
{
desc: "DNS configured, ENS address, name resolves, returns resolved address",
dns: doesResolve,
addr: ensAddr,
result: resolvedAddr,
},
{
desc: "DNS configured, immutable ENS address, name resolves, returns error",
dns: doesResolve,
addr: ensAddr,
immutable: true,
expectErr: errors.New(`immutable address not a content hash: "swarm.eth"`),
},
{
desc: "DNS configured, ENS address, name doesn't resolve, returns error",
dns: doesntResolve,
addr: ensAddr,
expectErr: errors.New(`DNS name not found: "swarm.eth"`),
},
}
for _, x := range tests {
t.Run(x.desc, func(t *testing.T) {
api := &Api{dns: x.dns}
uri := &URI{Addr: x.addr, Scheme: "bzz"}
if x.immutable {
uri.Scheme = "bzz-immutable"
}
res, err := api.Resolve(uri)
if err == nil {
if x.expectErr != nil {
t.Fatalf("expected error %q, got result %q", x.expectErr, res)
}
if res.String() != x.result {
t.Fatalf("expected result %q, got %q", x.result, res)
}
} else {
if x.expectErr == nil {
t.Fatalf("expected no error, got %q", err)
}
if err.Error() != x.expectErr.Error() {
t.Fatalf("expected error %q, got %q", x.expectErr, err)
}
}
})
}
}
func TestMultiResolver(t *testing.T) {
doesntResolve := newTestResolver("")
ethAddr := "swarm.eth"
ethHash := "0x2222222222222222222222222222222222222222222222222222222222222222"
ethResolve := newTestResolver(ethHash)
testAddr := "swarm.test"
testHash := "0x1111111111111111111111111111111111111111111111111111111111111111"
testResolve := newTestResolver(testHash)
tests := []struct {
desc string
r Resolver
addr string
result string
err error
}{
{
desc: "No resolvers, returns error",
r: NewMultiResolver(),
err: NewNoResolverError(""),
},
{
desc: "One default resolver, returns resolved address",
r: NewMultiResolver(MultiResolverOptionWithResolver(ethResolve, "")),
addr: ethAddr,
result: ethHash,
},
{
desc: "Two default resolvers, returns resolved address",
r: NewMultiResolver(
MultiResolverOptionWithResolver(ethResolve, ""),
MultiResolverOptionWithResolver(ethResolve, ""),
),
addr: ethAddr,
result: ethHash,
},
{
desc: "Two default resolvers, first doesn't resolve, returns resolved address",
r: NewMultiResolver(
MultiResolverOptionWithResolver(doesntResolve, ""),
MultiResolverOptionWithResolver(ethResolve, ""),
),
addr: ethAddr,
result: ethHash,
},
{
desc: "Default resolver doesn't resolve, tld resolver resolve, returns resolved address",
r: NewMultiResolver(
MultiResolverOptionWithResolver(doesntResolve, ""),
MultiResolverOptionWithResolver(ethResolve, "eth"),
),
addr: ethAddr,
result: ethHash,
},
{
desc: "Three TLD resolvers, third resolves, returns resolved address",
r: NewMultiResolver(
MultiResolverOptionWithResolver(doesntResolve, "eth"),
MultiResolverOptionWithResolver(doesntResolve, "eth"),
MultiResolverOptionWithResolver(ethResolve, "eth"),
),
addr: ethAddr,
result: ethHash,
},
{
desc: "One TLD resolver doesn't resolve, returns error",
r: NewMultiResolver(
MultiResolverOptionWithResolver(doesntResolve, ""),
MultiResolverOptionWithResolver(ethResolve, "eth"),
),
addr: ethAddr,
result: ethHash,
},
{
desc: "One defautl and one TLD resolver, all doesn't resolve, returns error",
r: NewMultiResolver(
MultiResolverOptionWithResolver(doesntResolve, ""),
MultiResolverOptionWithResolver(doesntResolve, "eth"),
),
addr: ethAddr,
result: ethHash,
err: errors.New(`DNS name not found: "swarm.eth"`),
},
{
desc: "Two TLD resolvers, both resolve, returns resolved address",
r: NewMultiResolver(
MultiResolverOptionWithResolver(ethResolve, "eth"),
MultiResolverOptionWithResolver(testResolve, "test"),
),
addr: testAddr,
result: testHash,
},
{
desc: "One TLD resolver, no default resolver, returns error for different TLD",
r: NewMultiResolver(
MultiResolverOptionWithResolver(ethResolve, "eth"),
),
addr: testAddr,
err: NewNoResolverError("test"),
},
}
for _, x := range tests {
t.Run(x.desc, func(t *testing.T) {
res, err := x.r.Resolve(x.addr)
if err == nil {
if x.err != nil {
t.Fatalf("expected error %q, got result %q", x.err, res.Hex())
}
if res.Hex() != x.result {
t.Fatalf("expected result %q, got %q", x.result, res.Hex())
}
} else {
if x.err == nil {
t.Fatalf("expected no error, got %q", err)
}
if err.Error() != x.err.Error() {
t.Fatalf("expected error %q, got %q", x.err, err)
}
}
})
}
}

464
swarm/api/client/client.go
View file

@ -1,464 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package client
import (
"archive/tar"
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"mime"
"mime/multipart"
"net/http"
"net/textproto"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/XinFinOrg/XDPoSChain/swarm/api"
)
var (
DefaultGateway = "http://localhost:8500"
DefaultClient = NewClient(DefaultGateway)
)
func NewClient(gateway string) *Client {
return &Client{
Gateway: gateway,
}
}
// Client wraps interaction with a swarm HTTP gateway.
type Client struct {
Gateway string
}
// UploadRaw uploads raw data to swarm and returns the resulting hash
func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) {
if size <= 0 {
return "", errors.New("data size must be greater than zero")
}
req, err := http.NewRequest("POST", c.Gateway+"/bzz-raw:/", r)
if err != nil {
return "", err
}
req.ContentLength = size
res, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
data, err := io.ReadAll(res.Body)
if err != nil {
return "", err
}
return string(data), nil
}
// DownloadRaw downloads raw data from swarm
func (c *Client) DownloadRaw(hash string) (io.ReadCloser, error) {
uri := c.Gateway + "/bzz-raw:/" + hash
res, err := http.DefaultClient.Get(uri)
if err != nil {
return nil, err
}
if res.StatusCode != http.StatusOK {
res.Body.Close()
return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
return res.Body, nil
}
// File represents a file in a swarm manifest and is used for uploading and
// downloading content to and from swarm
type File struct {
io.ReadCloser
api.ManifestEntry
}
// Open opens a local file which can then be passed to client.Upload to upload
// it to swarm
func Open(path string) (*File, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
stat, err := f.Stat()
if err != nil {
f.Close()
return nil, err
}
return &File{
ReadCloser: f,
ManifestEntry: api.ManifestEntry{
ContentType: mime.TypeByExtension(filepath.Ext(path)),
Mode: int64(stat.Mode()),
Size: stat.Size(),
ModTime: stat.ModTime(),
},
}, nil
}
// Upload uploads a file to swarm and either adds it to an existing manifest
// (if the manifest argument is non-empty) or creates a new manifest containing
// the file, returning the resulting manifest hash (the file will then be
// available at bzz:/<hash>/<path>)
func (c *Client) Upload(file *File, manifest string) (string, error) {
if file.Size <= 0 {
return "", errors.New("file size must be greater than zero")
}
return c.TarUpload(manifest, &FileUploader{file})
}
// Download downloads a file with the given path from the swarm manifest with
// the given hash (i.e. it gets bzz:/<hash>/<path>)
func (c *Client) Download(hash, path string) (*File, error) {
uri := c.Gateway + "/bzz:/" + hash + "/" + path
res, err := http.DefaultClient.Get(uri)
if err != nil {
return nil, err
}
if res.StatusCode != http.StatusOK {
res.Body.Close()
return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
return &File{
ReadCloser: res.Body,
ManifestEntry: api.ManifestEntry{
ContentType: res.Header.Get("Content-Type"),
Size: res.ContentLength,
},
}, nil
}
// UploadDirectory uploads a directory tree to swarm and either adds the files
// to an existing manifest (if the manifest argument is non-empty) or creates a
// new manifest, returning the resulting manifest hash (files from the
// directory will then be available at bzz:/<hash>/path/to/file), with
// the file specified in defaultPath being uploaded to the root of the manifest
// (i.e. bzz:/<hash>/)
func (c *Client) UploadDirectory(dir, defaultPath, manifest string) (string, error) {
stat, err := os.Stat(dir)
if err != nil {
return "", err
} else if !stat.IsDir() {
return "", fmt.Errorf("not a directory: %s", dir)
}
return c.TarUpload(manifest, &DirectoryUploader{dir, defaultPath})
}
// DownloadDirectory downloads the files contained in a swarm manifest under
// the given path into a local directory (existing files will be overwritten)
func (c *Client) DownloadDirectory(hash, path, destDir string) error {
stat, err := os.Stat(destDir)
if err != nil {
return err
} else if !stat.IsDir() {
return fmt.Errorf("not a directory: %s", destDir)
}
uri := c.Gateway + "/bzz:/" + hash + "/" + path
req, err := http.NewRequest("GET", uri, nil)
if err != nil {
return err
}
req.Header.Set("Accept", "application/x-tar")
res, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
tr := tar.NewReader(res.Body)
for {
hdr, err := tr.Next()
if err == io.EOF {
return nil
} else if err != nil {
return err
}
// ignore the default path file
if hdr.Name == "" {
continue
}
dstPath := filepath.Join(destDir, filepath.Clean(strings.TrimPrefix(hdr.Name, path)))
if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
return err
}
var mode os.FileMode = 0644
if hdr.Mode > 0 {
mode = os.FileMode(hdr.Mode)
}
dst, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode)
if err != nil {
return err
}
n, err := io.Copy(dst, tr)
dst.Close()
if err != nil {
return err
} else if n != hdr.Size {
return fmt.Errorf("expected %s to be %d bytes but got %d", hdr.Name, hdr.Size, n)
}
}
}
// UploadManifest uploads the given manifest to swarm
func (c *Client) UploadManifest(m *api.Manifest) (string, error) {
data, err := json.Marshal(m)
if err != nil {
return "", err
}
return c.UploadRaw(bytes.NewReader(data), int64(len(data)))
}
// DownloadManifest downloads a swarm manifest
func (c *Client) DownloadManifest(hash string) (*api.Manifest, error) {
res, err := c.DownloadRaw(hash)
if err != nil {
return nil, err
}
defer res.Close()
var manifest api.Manifest
if err := json.NewDecoder(res).Decode(&manifest); err != nil {
return nil, err
}
return &manifest, nil
}
// List list files in a swarm manifest which have the given prefix, grouping
// common prefixes using "/" as a delimiter.
//
// For example, if the manifest represents the following directory structure:
//
// file1.txt
// file2.txt
// dir1/file3.txt
// dir1/dir2/file4.txt
//
// Then:
//
// - a prefix of "" would return [dir1/, file1.txt, file2.txt]
// - a prefix of "file" would return [file1.txt, file2.txt]
// - a prefix of "dir1/" would return [dir1/dir2/, dir1/file3.txt]
//
// where entries ending with "/" are common prefixes.
func (c *Client) List(hash, prefix string) (*api.ManifestList, error) {
res, err := http.DefaultClient.Get(c.Gateway + "/bzz-list:/" + hash + "/" + prefix)
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
var list api.ManifestList
if err := json.NewDecoder(res.Body).Decode(&list); err != nil {
return nil, err
}
return &list, nil
}
// Uploader uploads files to swarm using a provided UploadFn
type Uploader interface {
Upload(UploadFn) error
}
type UploaderFunc func(UploadFn) error
func (u UploaderFunc) Upload(upload UploadFn) error {
return u(upload)
}
// DirectoryUploader uploads all files in a directory, optionally uploading
// a file to the default path
type DirectoryUploader struct {
Dir string
DefaultPath string
}
// Upload performs the upload of the directory and default path
func (d *DirectoryUploader) Upload(upload UploadFn) error {
if d.DefaultPath != "" {
file, err := Open(d.DefaultPath)
if err != nil {
return err
}
if err := upload(file); err != nil {
return err
}
}
return filepath.Walk(d.Dir, func(path string, f os.FileInfo, err error) error {
if err != nil {
return err
}
if f.IsDir() {
return nil
}
file, err := Open(path)
if err != nil {
return err
}
relPath, err := filepath.Rel(d.Dir, path)
if err != nil {
return err
}
file.Path = filepath.ToSlash(relPath)
return upload(file)
})
}
// FileUploader uploads a single file
type FileUploader struct {
File *File
}
// Upload performs the upload of the file
func (f *FileUploader) Upload(upload UploadFn) error {
return upload(f.File)
}
// UploadFn is the type of function passed to an Uploader to perform the upload
// of a single file (for example, a directory uploader would call a provided
// UploadFn for each file in the directory tree)
type UploadFn func(file *File) error
// TarUpload uses the given Uploader to upload files to swarm as a tar stream,
// returning the resulting manifest hash
func (c *Client) TarUpload(hash string, uploader Uploader) (string, error) {
reqR, reqW := io.Pipe()
defer reqR.Close()
req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR)
if err != nil {
return "", err
}
req.Header.Set("Content-Type", "application/x-tar")
// use 'Expect: 100-continue' so we don't send the request body if
// the server refuses the request
req.Header.Set("Expect", "100-continue")
tw := tar.NewWriter(reqW)
// define an UploadFn which adds files to the tar stream
uploadFn := func(file *File) error {
hdr := &tar.Header{
Name: file.Path,
Mode: file.Mode,
Size: file.Size,
ModTime: file.ModTime,
Xattrs: map[string]string{
"user.swarm.content-type": file.ContentType,
},
}
if err := tw.WriteHeader(hdr); err != nil {
return err
}
_, err = io.Copy(tw, file)
return err
}
// run the upload in a goroutine so we can send the request headers and
// wait for a '100 Continue' response before sending the tar stream
go func() {
err := uploader.Upload(uploadFn)
if err == nil {
err = tw.Close()
}
reqW.CloseWithError(err)
}()
res, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
data, err := io.ReadAll(res.Body)
if err != nil {
return "", err
}
return string(data), nil
}
// MultipartUpload uses the given Uploader to upload files to swarm as a
// multipart form, returning the resulting manifest hash
func (c *Client) MultipartUpload(hash string, uploader Uploader) (string, error) {
reqR, reqW := io.Pipe()
defer reqR.Close()
req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR)
if err != nil {
return "", err
}
// use 'Expect: 100-continue' so we don't send the request body if
// the server refuses the request
req.Header.Set("Expect", "100-continue")
mw := multipart.NewWriter(reqW)
req.Header.Set("Content-Type", fmt.Sprintf("multipart/form-data; boundary=%q", mw.Boundary()))
// define an UploadFn which adds files to the multipart form
uploadFn := func(file *File) error {
hdr := make(textproto.MIMEHeader)
hdr.Set("Content-Disposition", fmt.Sprintf("form-data; name=%q", file.Path))
hdr.Set("Content-Type", file.ContentType)
hdr.Set("Content-Length", strconv.FormatInt(file.Size, 10))
w, err := mw.CreatePart(hdr)
if err != nil {
return err
}
_, err = io.Copy(w, file)
return err
}
// run the upload in a goroutine so we can send the request headers and
// wait for a '100 Continue' response before sending the multipart form
go func() {
err := uploader.Upload(uploadFn)
if err == nil {
err = mw.Close()
}
reqW.CloseWithError(err)
}()
res, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
data, err := io.ReadAll(res.Body)
if err != nil {
return "", err
}
return string(data), nil
}

325
swarm/api/client/client_test.go
View file

@ -1,325 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package client
import (
"bytes"
"io"
"os"
"path/filepath"
"reflect"
"sort"
"testing"
"github.com/XinFinOrg/XDPoSChain/swarm/api"
"github.com/XinFinOrg/XDPoSChain/swarm/testutil"
)
// TestClientUploadDownloadRaw test uploading and downloading raw data to swarm
func TestClientUploadDownloadRaw(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
defer srv.Close()
client := NewClient(srv.URL)
// upload some raw data
data := []byte("foo123")
hash, err := client.UploadRaw(bytes.NewReader(data), int64(len(data)))
if err != nil {
t.Fatal(err)
}
// check we can download the same data
res, err := client.DownloadRaw(hash)
if err != nil {
t.Fatal(err)
}
defer res.Close()
gotData, err := io.ReadAll(res)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(gotData, data) {
t.Fatalf("expected downloaded data to be %q, got %q", data, gotData)
}
}
// TestClientUploadDownloadFiles test uploading and downloading files to swarm
// manifests
func TestClientUploadDownloadFiles(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
defer srv.Close()
client := NewClient(srv.URL)
upload := func(manifest, path string, data []byte) string {
file := &File{
ReadCloser: io.NopCloser(bytes.NewReader(data)),
ManifestEntry: api.ManifestEntry{
Path: path,
ContentType: "text/plain",
Size: int64(len(data)),
},
}
hash, err := client.Upload(file, manifest)
if err != nil {
t.Fatal(err)
}
return hash
}
checkDownload := func(manifest, path string, expected []byte) {
file, err := client.Download(manifest, path)
if err != nil {
t.Fatal(err)
}
defer file.Close()
if file.Size != int64(len(expected)) {
t.Fatalf("expected downloaded file to be %d bytes, got %d", len(expected), file.Size)
}
if file.ContentType != "text/plain" {
t.Fatalf("expected downloaded file to have type %q, got %q", "text/plain", file.ContentType)
}
data, err := io.ReadAll(file)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(data, expected) {
t.Fatalf("expected downloaded data to be %q, got %q", expected, data)
}
}
// upload a file to the root of a manifest
rootData := []byte("some-data")
rootHash := upload("", "", rootData)
// check we can download the root file
checkDownload(rootHash, "", rootData)
// upload another file to the same manifest
otherData := []byte("some-other-data")
newHash := upload(rootHash, "some/other/path", otherData)
// check we can download both files from the new manifest
checkDownload(newHash, "", rootData)
checkDownload(newHash, "some/other/path", otherData)
// replace the root file with different data
newHash = upload(newHash, "", otherData)
// check both files have the other data
checkDownload(newHash, "", otherData)
checkDownload(newHash, "some/other/path", otherData)
}
var testDirFiles = []string{
"file1.txt",
"file2.txt",
"dir1/file3.txt",
"dir1/file4.txt",
"dir2/file5.txt",
"dir2/dir3/file6.txt",
"dir2/dir4/file7.txt",
"dir2/dir4/file8.txt",
}
func newTestDirectory(t *testing.T) string {
dir, err := os.MkdirTemp("", "swarm-client-test")
if err != nil {
t.Fatal(err)
}
for _, file := range testDirFiles {
path := filepath.Join(dir, file)
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
os.RemoveAll(dir)
t.Fatalf("error creating dir for %s: %s", path, err)
}
if err := os.WriteFile(path, []byte(file), 0644); err != nil {
os.RemoveAll(dir)
t.Fatalf("error writing file %s: %s", path, err)
}
}
return dir
}
// TestClientUploadDownloadDirectory tests uploading and downloading a
// directory of files to a swarm manifest
func TestClientUploadDownloadDirectory(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
defer srv.Close()
dir := newTestDirectory(t)
defer os.RemoveAll(dir)
// upload the directory
client := NewClient(srv.URL)
defaultPath := filepath.Join(dir, testDirFiles[0])
hash, err := client.UploadDirectory(dir, defaultPath, "")
if err != nil {
t.Fatalf("error uploading directory: %s", err)
}
// check we can download the individual files
checkDownloadFile := func(path string, expected []byte) {
file, err := client.Download(hash, path)
if err != nil {
t.Fatal(err)
}
defer file.Close()
data, err := io.ReadAll(file)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(data, expected) {
t.Fatalf("expected data to be %q, got %q", expected, data)
}
}
for _, file := range testDirFiles {
checkDownloadFile(file, []byte(file))
}
// check we can download the default path
checkDownloadFile("", []byte(testDirFiles[0]))
// check we can download the directory
tmp, err := os.MkdirTemp("", "swarm-client-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmp)
if err := client.DownloadDirectory(hash, "", tmp); err != nil {
t.Fatal(err)
}
for _, file := range testDirFiles {
data, err := os.ReadFile(filepath.Join(tmp, file))
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(data, []byte(file)) {
t.Fatalf("expected data to be %q, got %q", file, data)
}
}
}
// TestClientFileList tests listing files in a swarm manifest
func TestClientFileList(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
defer srv.Close()
dir := newTestDirectory(t)
defer os.RemoveAll(dir)
client := NewClient(srv.URL)
hash, err := client.UploadDirectory(dir, "", "")
if err != nil {
t.Fatalf("error uploading directory: %s", err)
}
ls := func(prefix string) []string {
list, err := client.List(hash, prefix)
if err != nil {
t.Fatal(err)
}
paths := make([]string, 0, len(list.CommonPrefixes)+len(list.Entries))
paths = append(paths, list.CommonPrefixes...)
for _, entry := range list.Entries {
paths = append(paths, entry.Path)
}
sort.Strings(paths)
return paths
}
tests := map[string][]string{
"": {"dir1/", "dir2/", "file1.txt", "file2.txt"},
"file": {"file1.txt", "file2.txt"},
"file1": {"file1.txt"},
"file2.txt": {"file2.txt"},
"file12": {},
"dir": {"dir1/", "dir2/"},
"dir1": {"dir1/"},
"dir1/": {"dir1/file3.txt", "dir1/file4.txt"},
"dir1/file": {"dir1/file3.txt", "dir1/file4.txt"},
"dir1/file3.txt": {"dir1/file3.txt"},
"dir1/file34": {},
"dir2/": {"dir2/dir3/", "dir2/dir4/", "dir2/file5.txt"},
"dir2/file": {"dir2/file5.txt"},
"dir2/dir": {"dir2/dir3/", "dir2/dir4/"},
"dir2/dir3/": {"dir2/dir3/file6.txt"},
"dir2/dir4/": {"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"},
"dir2/dir4/file": {"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"},
"dir2/dir4/file7.txt": {"dir2/dir4/file7.txt"},
"dir2/dir4/file78": {},
}
for prefix, expected := range tests {
actual := ls(prefix)
if !reflect.DeepEqual(actual, expected) {
t.Fatalf("expected prefix %q to return %v, got %v", prefix, expected, actual)
}
}
}
// TestClientMultipartUpload tests uploading files to swarm using a multipart
// upload
func TestClientMultipartUpload(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
defer srv.Close()
// define an uploader which uploads testDirFiles with some data
data := []byte("some-data")
uploader := UploaderFunc(func(upload UploadFn) error {
for _, name := range testDirFiles {
file := &File{
ReadCloser: io.NopCloser(bytes.NewReader(data)),
ManifestEntry: api.ManifestEntry{
Path: name,
ContentType: "text/plain",
Size: int64(len(data)),
},
}
if err := upload(file); err != nil {
return err
}
}
return nil
})
// upload the files as a multipart upload
client := NewClient(srv.URL)
hash, err := client.MultipartUpload("", uploader)
if err != nil {
t.Fatal(err)
}
// check we can download the individual files
checkDownloadFile := func(path string) {
file, err := client.Download(hash, path)
if err != nil {
t.Fatal(err)
}
defer file.Close()
gotData, err := io.ReadAll(file)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(gotData, data) {
t.Fatalf("expected data to be %q, got %q", data, gotData)
}
}
for _, file := range testDirFiles {
checkDownloadFile(file)
}
}

113
swarm/api/config.go
View file

@ -1,113 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"crypto/ecdsa"
"fmt"
"os"
"path/filepath"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/contracts/ens"
"github.com/XinFinOrg/XDPoSChain/crypto"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/node"
"github.com/XinFinOrg/XDPoSChain/swarm/network"
"github.com/XinFinOrg/XDPoSChain/swarm/services/swap"
"github.com/XinFinOrg/XDPoSChain/swarm/storage"
)
const (
DefaultHTTPListenAddr = "127.0.0.1"
DefaultHTTPPort = "8500"
)
// separate bzz directories
// allow several bzz nodes running in parallel
type Config struct {
// serialised/persisted fields
*storage.StoreParams
*storage.ChunkerParams
*network.HiveParams
Swap *swap.SwapParams
*network.SyncParams
Contract common.Address
EnsRoot common.Address
EnsAPIs []string
Path string
ListenAddr string
Port string
PublicKey string
BzzKey string
NetworkId uint64
SwapEnabled bool
SyncEnabled bool
SwapApi string
Cors string
BzzAccount string
BootNodes string
}
//create a default config with all parameters to set to defaults
func NewDefaultConfig() (self *Config) {
self = &Config{
StoreParams: storage.NewDefaultStoreParams(),
ChunkerParams: storage.NewChunkerParams(),
HiveParams: network.NewDefaultHiveParams(),
SyncParams: network.NewDefaultSyncParams(),
Swap: swap.NewDefaultSwapParams(),
ListenAddr: DefaultHTTPListenAddr,
Port: DefaultHTTPPort,
Path: node.DefaultDataDir(),
EnsAPIs: nil,
EnsRoot: ens.TestNetAddress,
NetworkId: network.NetworkId,
SwapEnabled: false,
SyncEnabled: true,
SwapApi: "",
BootNodes: "",
}
return
}
//some config params need to be initialized after the complete
//config building phase is completed (e.g. due to overriding flags)
func (self *Config) Init(prvKey *ecdsa.PrivateKey) {
address := crypto.PubkeyToAddress(prvKey.PublicKey)
self.Path = filepath.Join(self.Path, "bzz-"+common.Bytes2Hex(address.Bytes()))
err := os.MkdirAll(self.Path, os.ModePerm)
if err != nil {
log.Error(fmt.Sprintf("Error creating root swarm data directory: %v", err))
return
}
pubkey := crypto.FromECDSAPub(&prvKey.PublicKey)
pubkeyhex := common.ToHex(pubkey)
keyhex := crypto.Keccak256Hash(pubkey).Hex()
self.PublicKey = pubkeyhex
self.BzzKey = keyhex
self.Swap.Init(self.Contract, prvKey)
self.SyncParams.Init(self.Path)
self.HiveParams.Init(self.Path)
self.StoreParams.Init(self.Path)
}

69
swarm/api/config_test.go
View file

@ -1,69 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"reflect"
"testing"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/crypto"
)
func TestConfig(t *testing.T) {
var hexprvkey = "65138b2aa745041b372153550584587da326ab440576b2a1191dd95cee30039c"
prvkey, err := crypto.HexToECDSA(hexprvkey)
if err != nil {
t.Fatalf("failed to load private key: %v", err)
}
one := NewDefaultConfig()
two := NewDefaultConfig()
if equal := reflect.DeepEqual(one, two); !equal {
t.Fatal("Two default configs are not equal")
}
one.Init(prvkey)
//the init function should set the following fields
if one.BzzKey == "" {
t.Fatal("Expected BzzKey to be set")
}
if one.PublicKey == "" {
t.Fatal("Expected PublicKey to be set")
}
//the Init function should append subdirs to the given path
if one.Swap.PayProfile.Beneficiary == (common.Address{}) {
t.Fatal("Failed to correctly initialize SwapParams")
}
if one.SyncParams.RequestDbPath == one.Path {
t.Fatal("Failed to correctly initialize SyncParams")
}
if one.HiveParams.KadDbPath == one.Path {
t.Fatal("Failed to correctly initialize HiveParams")
}
if one.StoreParams.ChunkDbPath == one.Path {
t.Fatal("Failed to correctly initialize StoreParams")
}
}

288
swarm/api/filesystem.go
View file

@ -1,288 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"bufio"
"fmt"
"io"
"net/http"
"os"
"path"
"path/filepath"
"sync"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/swarm/storage"
)
const maxParallelFiles = 5
type FileSystem struct {
api *Api
}
func NewFileSystem(api *Api) *FileSystem {
return &FileSystem{api}
}
// Upload replicates a local directory as a manifest file and uploads it
// using dpa store
// TODO: localpath should point to a manifest
//
// DEPRECATED: Use the HTTP API instead
func (self *FileSystem) Upload(lpath, index string) (string, error) {
var list []*manifestTrieEntry
localpath, err := filepath.Abs(filepath.Clean(lpath))
if err != nil {
return "", err
}
f, err := os.Open(localpath)
if err != nil {
return "", err
}
stat, err := f.Stat()
if err != nil {
return "", err
}
var start int
if stat.IsDir() {
start = len(localpath)
log.Debug(fmt.Sprintf("uploading '%s'", localpath))
err = filepath.Walk(localpath, func(path string, info os.FileInfo, err error) error {
if (err == nil) && !info.IsDir() {
if len(path) <= start {
return fmt.Errorf("Path is too short")
}
if path[:start] != localpath {
return fmt.Errorf("Path prefix of '%s' does not match localpath '%s'", path, localpath)
}
entry := newManifestTrieEntry(&ManifestEntry{Path: filepath.ToSlash(path)}, nil)
list = append(list, entry)
}
return err
})
if err != nil {
return "", err
}
} else {
dir := filepath.Dir(localpath)
start = len(dir)
if len(localpath) <= start {
return "", fmt.Errorf("Path is too short")
}
if localpath[:start] != dir {
return "", fmt.Errorf("Path prefix of '%s' does not match dir '%s'", localpath, dir)
}
entry := newManifestTrieEntry(&ManifestEntry{Path: filepath.ToSlash(localpath)}, nil)
list = append(list, entry)
}
cnt := len(list)
errors := make([]error, cnt)
done := make(chan bool, maxParallelFiles)
dcnt := 0
awg := &sync.WaitGroup{}
for i, entry := range list {
if i >= dcnt+maxParallelFiles {
<-done
dcnt++
}
awg.Add(1)
go func(i int, entry *manifestTrieEntry, done chan bool) {
f, err := os.Open(entry.Path)
if err == nil {
stat, _ := f.Stat()
var hash storage.Key
wg := &sync.WaitGroup{}
hash, err = self.api.dpa.Store(f, stat.Size(), wg, nil)
if hash != nil {
list[i].Hash = hash.String()
}
wg.Wait()
awg.Done()
if err == nil {
first512 := make([]byte, 512)
fread, _ := f.ReadAt(first512, 0)
if fread > 0 {
mimeType := http.DetectContentType(first512[:fread])
if filepath.Ext(entry.Path) == ".css" {
mimeType = "text/css"
}
list[i].ContentType = mimeType
}
}
f.Close()
}
errors[i] = err
done <- true
}(i, entry, done)
}
for dcnt < cnt {
<-done
dcnt++
}
trie := &manifestTrie{
dpa: self.api.dpa,
}
quitC := make(chan bool)
for i, entry := range list {
if errors[i] != nil {
return "", errors[i]
}
entry.Path = RegularSlashes(entry.Path[start:])
if entry.Path == index {
ientry := newManifestTrieEntry(&ManifestEntry{
ContentType: entry.ContentType,
}, nil)
ientry.Hash = entry.Hash
trie.addEntry(ientry, quitC)
}
trie.addEntry(entry, quitC)
}
err2 := trie.recalcAndStore()
var hs string
if err2 == nil {
hs = trie.hash.String()
}
awg.Wait()
return hs, err2
}
// Download replicates the manifest basePath structure on the local filesystem
// under localpath
//
// DEPRECATED: Use the HTTP API instead
func (self *FileSystem) Download(bzzpath, localpath string) error {
lpath, err := filepath.Abs(filepath.Clean(localpath))
if err != nil {
return err
}
err = os.MkdirAll(lpath, os.ModePerm)
if err != nil {
return err
}
//resolving host and port
uri, err := Parse(path.Join("bzz:/", bzzpath))
if err != nil {
return err
}
key, err := self.api.Resolve(uri)
if err != nil {
return err
}
path := uri.Path
if len(path) > 0 {
path += "/"
}
quitC := make(chan bool)
trie, err := loadManifest(self.api.dpa, key, quitC)
if err != nil {
log.Warn(fmt.Sprintf("fs.Download: loadManifestTrie error: %v", err))
return err
}
type downloadListEntry struct {
key storage.Key
path string
}
var list []*downloadListEntry
var mde error
prevPath := lpath
err = trie.listWithPrefix(path, quitC, func(entry *manifestTrieEntry, suffix string) {
log.Trace(fmt.Sprintf("fs.Download: %#v", entry))
key = common.Hex2Bytes(entry.Hash)
path := lpath + "/" + suffix
dir := filepath.Dir(path)
if dir != prevPath {
mde = os.MkdirAll(dir, os.ModePerm)
prevPath = dir
}
if (mde == nil) && (path != dir+"/") {
list = append(list, &downloadListEntry{key: key, path: path})
}
})
if err != nil {
return err
}
wg := sync.WaitGroup{}
errC := make(chan error)
done := make(chan bool, maxParallelFiles)
for i, entry := range list {
select {
case done <- true:
wg.Add(1)
case <-quitC:
return fmt.Errorf("aborted")
}
go func(i int, entry *downloadListEntry) {
defer wg.Done()
err := retrieveToFile(quitC, self.api.dpa, entry.key, entry.path)
if err != nil {
select {
case errC <- err:
case <-quitC:
}
return
}
<-done
}(i, entry)
}
go func() {
wg.Wait()
close(errC)
}()
select {
case err = <-errC:
return err
case <-quitC:
return fmt.Errorf("aborted")
}
}
func retrieveToFile(quitC chan bool, dpa *storage.DPA, key storage.Key, path string) error {
f, err := os.Create(path) // TODO: basePath separators
if err != nil {
return err
}
reader := dpa.Retrieve(key)
writer := bufio.NewWriter(f)
size, err := reader.Size(quitC)
if err != nil {
return err
}
if _, err = io.CopyN(writer, reader, size); err != nil {
return err
}
if err := writer.Flush(); err != nil {
return err
}
return f.Close()
}

193
swarm/api/filesystem_test.go
View file

@ -1,193 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"bytes"
"os"
"path/filepath"
"sync"
"testing"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/swarm/storage"
)
var testDownloadDir, _ = os.MkdirTemp(os.TempDir(), "bzz-test")
func testFileSystem(t *testing.T, f func(*FileSystem)) {
testApi(t, func(api *Api) {
f(NewFileSystem(api))
})
}
func readPath(t *testing.T, parts ...string) string {
file := filepath.Join(parts...)
content, err := os.ReadFile(file)
if err != nil {
t.Fatalf("unexpected error reading '%v': %v", file, err)
}
return string(content)
}
func TestApiDirUpload0(t *testing.T) {
testFileSystem(t, func(fs *FileSystem) {
api := fs.api
bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
content := readPath(t, "testdata", "test0", "index.html")
resp := testGet(t, api, bzzhash, "index.html")
exp := expResponse(content, "text/html; charset=utf-8", 0)
checkResponse(t, resp, exp)
content = readPath(t, "testdata", "test0", "index.css")
resp = testGet(t, api, bzzhash, "index.css")
exp = expResponse(content, "text/css", 0)
checkResponse(t, resp, exp)
key := storage.Key(common.Hex2Bytes(bzzhash))
_, _, _, err = api.Get(key, "")
if err == nil {
t.Fatalf("expected error: %v", err)
}
downloadDir := filepath.Join(testDownloadDir, "test0")
defer os.RemoveAll(downloadDir)
err = fs.Download(bzzhash, downloadDir)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
newbzzhash, err := fs.Upload(downloadDir, "")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if bzzhash != newbzzhash {
t.Fatalf("download %v reuploaded has incorrect hash, expected %v, got %v", downloadDir, bzzhash, newbzzhash)
}
})
}
func TestApiDirUploadModify(t *testing.T) {
testFileSystem(t, func(fs *FileSystem) {
api := fs.api
bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "")
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
key := storage.Key(common.Hex2Bytes(bzzhash))
key, err = api.Modify(key, "index.html", "", "")
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
index, err := os.ReadFile(filepath.Join("testdata", "test0", "index.html"))
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
wg := &sync.WaitGroup{}
hash, err := api.Store(bytes.NewReader(index), int64(len(index)), wg)
wg.Wait()
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
key, err = api.Modify(key, "index2.html", hash.Hex(), "text/html; charset=utf-8")
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
key, err = api.Modify(key, "img/logo.png", hash.Hex(), "text/html; charset=utf-8")
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
bzzhash = key.String()
content := readPath(t, "testdata", "test0", "index.html")
resp := testGet(t, api, bzzhash, "index2.html")
exp := expResponse(content, "text/html; charset=utf-8", 0)
checkResponse(t, resp, exp)
resp = testGet(t, api, bzzhash, "img/logo.png")
exp = expResponse(content, "text/html; charset=utf-8", 0)
checkResponse(t, resp, exp)
content = readPath(t, "testdata", "test0", "index.css")
resp = testGet(t, api, bzzhash, "index.css")
exp = expResponse(content, "text/css", 0)
checkResponse(t, resp, exp)
_, _, _, err = api.Get(key, "")
if err == nil {
t.Errorf("expected error: %v", err)
}
})
}
func TestApiDirUploadWithRootFile(t *testing.T) {
testFileSystem(t, func(fs *FileSystem) {
api := fs.api
bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "index.html")
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
content := readPath(t, "testdata", "test0", "index.html")
resp := testGet(t, api, bzzhash, "")
exp := expResponse(content, "text/html; charset=utf-8", 0)
checkResponse(t, resp, exp)
})
}
func TestApiFileUpload(t *testing.T) {
testFileSystem(t, func(fs *FileSystem) {
api := fs.api
bzzhash, err := fs.Upload(filepath.Join("testdata", "test0", "index.html"), "")
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
content := readPath(t, "testdata", "test0", "index.html")
resp := testGet(t, api, bzzhash, "index.html")
exp := expResponse(content, "text/html; charset=utf-8", 0)
checkResponse(t, resp, exp)
})
}
func TestApiFileUploadWithRootFile(t *testing.T) {
testFileSystem(t, func(fs *FileSystem) {
api := fs.api
bzzhash, err := fs.Upload(filepath.Join("testdata", "test0", "index.html"), "index.html")
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
content := readPath(t, "testdata", "test0", "index.html")
resp := testGet(t, api, bzzhash, "")
exp := expResponse(content, "text/html; charset=utf-8", 0)
checkResponse(t, resp, exp)
})
}

196
swarm/api/http/error.go
View file

@ -1,196 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
/*
Show nicely (but simple) formatted HTML error pages (or respond with JSON
if the appropriate `Accept` header is set)) for the http package.
*/
package http
import (
"encoding/json"
"fmt"
"html/template"
"net/http"
"strings"
"time"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/metrics"
"github.com/XinFinOrg/XDPoSChain/swarm/api"
)
//templateMap holds a mapping of an HTTP error code to a template
var templateMap map[int]*template.Template
var caseErrors []CaseError
//metrics variables
var (
htmlCounter = metrics.NewRegisteredCounter("api.http.errorpage.html.count", nil)
jsonCounter = metrics.NewRegisteredCounter("api.http.errorpage.json.count", nil)
)
//parameters needed for formatting the correct HTML page
type ErrorParams struct {
Msg string
Code int
Timestamp string
template *template.Template
Details template.HTML
}
//a custom error case struct that would be used to store validators and
//additional error info to display with client responses.
type CaseError struct {
Validator func(*Request) bool
Msg func(*Request) string
}
//we init the error handling right on boot time, so lookup and http response is fast
func init() {
initErrHandling()
}
func initErrHandling() {
//pages are saved as strings - get these strings
genErrPage := GetGenericErrorPage()
notFoundPage := GetNotFoundErrorPage()
multipleChoicesPage := GetMultipleChoicesErrorPage()
//map the codes to the available pages
tnames := map[int]string{
0: genErrPage, //default
http.StatusBadRequest: genErrPage,
http.StatusNotFound: notFoundPage,
http.StatusMultipleChoices: multipleChoicesPage,
http.StatusInternalServerError: genErrPage,
}
templateMap = make(map[int]*template.Template)
for code, tname := range tnames {
//assign formatted HTML to the code
templateMap[code] = template.Must(template.New(fmt.Sprintf("%d", code)).Parse(tname))
}
caseErrors = []CaseError{
{
Validator: func(r *Request) bool { return r.uri != nil && r.uri.Addr != "" && strings.HasPrefix(r.uri.Addr, "0x") },
Msg: func(r *Request) string {
uriCopy := r.uri
uriCopy.Addr = strings.TrimPrefix(uriCopy.Addr, "0x")
return fmt.Sprintf(`The requested hash seems to be prefixed with '0x'. You will be redirected to the correct URL within 5 seconds.<br/>
Please click <a href='%[1]s'>here</a> if your browser does not redirect you.<script>setTimeout("location.href='%[1]s';",5000);</script>`, "/"+uriCopy.String())
},
}}
}
//ValidateCaseErrors is a method that process the request object through certain validators
//that assert if certain conditions are met for further information to log as an error
func ValidateCaseErrors(r *Request) string {
for _, err := range caseErrors {
if err.Validator(r) {
return err.Msg(r)
}
}
return ""
}
//ShowMultipeChoices is used when a user requests a resource in a manifest which results
//in ambiguous results. It returns a HTML page with clickable links of each of the entry
//in the manifest which fits the request URI ambiguity.
//For example, if the user requests bzz:/<hash>/read and that manifest contains entries
//"readme.md" and "readinglist.txt", a HTML page is returned with this two links.
//This only applies if the manifest has no default entry
func ShowMultipleChoices(w http.ResponseWriter, r *Request, list api.ManifestList) {
msg := ""
if list.Entries == nil {
ShowError(w, r, "Could not resolve", http.StatusInternalServerError)
return
}
//make links relative
//requestURI comes with the prefix of the ambiguous path, e.g. "read" for "readme.md" and "readinglist.txt"
//to get clickable links, need to remove the ambiguous path, i.e. "read"
idx := strings.LastIndex(r.RequestURI, "/")
if idx == -1 {
ShowError(w, r, "Internal Server Error", http.StatusInternalServerError)
return
}
//remove ambiguous part
base := r.RequestURI[:idx+1]
for _, e := range list.Entries {
//create clickable link for each entry
msg += "<a href='" + base + e.Path + "'>" + e.Path + "</a><br/>"
}
respond(w, &r.Request, &ErrorParams{
Code: http.StatusMultipleChoices,
Details: template.HTML(msg),
Timestamp: time.Now().Format(time.RFC1123),
template: getTemplate(http.StatusMultipleChoices),
})
}
//ShowError is used to show an HTML error page to a client.
//If there is an `Accept` header of `application/json`, JSON will be returned instead
//The function just takes a string message which will be displayed in the error page.
//The code is used to evaluate which template will be displayed
//(and return the correct HTTP status code)
func ShowError(w http.ResponseWriter, r *Request, msg string, code int) {
additionalMessage := ValidateCaseErrors(r)
if code == http.StatusInternalServerError {
log.Error(msg)
}
respond(w, &r.Request, &ErrorParams{
Code: code,
Msg: msg,
Details: template.HTML(additionalMessage),
Timestamp: time.Now().Format(time.RFC1123),
template: getTemplate(code),
})
}
//evaluate if client accepts html or json response
func respond(w http.ResponseWriter, r *http.Request, params *ErrorParams) {
w.WriteHeader(params.Code)
if r.Header.Get("Accept") == "application/json" {
respondJson(w, params)
} else {
respondHtml(w, params)
}
}
//return a HTML page
func respondHtml(w http.ResponseWriter, params *ErrorParams) {
htmlCounter.Inc(1)
err := params.template.Execute(w, params)
if err != nil {
log.Error(err.Error())
}
}
//return JSON
func respondJson(w http.ResponseWriter, params *ErrorParams) {
jsonCounter.Inc(1)
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(params)
}
//get the HTML template for a given code
func getTemplate(code int) *template.Template {
if val, tmpl := templateMap[code]; tmpl {
return val
} else {
return templateMap[0]
}
}

564
swarm/api/http/error_templates.go
View file

File diff suppressed because one or more lines are too long

173
swarm/api/http/error_test.go
View file

@ -1,173 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package http_test
import (
"encoding/json"
"io"
"net/http"
"strings"
"testing"
"golang.org/x/net/html"
"github.com/XinFinOrg/XDPoSChain/swarm/testutil"
)
func TestError(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
defer srv.Close()
var resp *http.Response
var respbody []byte
url := srv.URL + "/this_should_fail_as_no_bzz_protocol_present"
resp, err := http.Get(url)
if err != nil {
t.Fatalf("Request failed: %v", err)
}
defer resp.Body.Close()
respbody, err = io.ReadAll(resp.Body)
if resp.StatusCode != 400 && !strings.Contains(string(respbody), "Invalid URI &#34;/this_should_fail_as_no_bzz_protocol_present&#34;: unknown scheme") {
t.Fatalf("Response body does not match, expected: %v, to contain: %v; received code %d, expected code: %d", string(respbody), "Invalid bzz URI: unknown scheme", 400, resp.StatusCode)
}
_, err = html.Parse(strings.NewReader(string(respbody)))
if err != nil {
t.Fatalf("HTML validation failed for error page returned!")
}
}
func Test404Page(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
defer srv.Close()
var resp *http.Response
var respbody []byte
url := srv.URL + "/bzz:/1234567890123456789012345678901234567890123456789012345678901234"
resp, err := http.Get(url)
if err != nil {
t.Fatalf("Request failed: %v", err)
}
defer resp.Body.Close()
respbody, err = io.ReadAll(resp.Body)
if resp.StatusCode != 404 || !strings.Contains(string(respbody), "404") {
t.Fatalf("Invalid Status Code received, expected 404, got %d", resp.StatusCode)
}
_, err = html.Parse(strings.NewReader(string(respbody)))
if err != nil {
t.Fatalf("HTML validation failed for error page returned!")
}
}
func Test500Page(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
defer srv.Close()
var resp *http.Response
var respbody []byte
url := srv.URL + "/bzz:/thisShouldFailWith500Code"
resp, err := http.Get(url)
if err != nil {
t.Fatalf("Request failed: %v", err)
}
defer resp.Body.Close()
respbody, err = io.ReadAll(resp.Body)
if resp.StatusCode != 404 {
t.Fatalf("Invalid Status Code received, expected 404, got %d", resp.StatusCode)
}
_, err = html.Parse(strings.NewReader(string(respbody)))
if err != nil {
t.Fatalf("HTML validation failed for error page returned!")
}
}
func Test500PageWith0xHashPrefix(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
defer srv.Close()
var resp *http.Response
var respbody []byte
url := srv.URL + "/bzz:/0xthisShouldFailWith500CodeAndAHelpfulMessage"
resp, err := http.Get(url)
if err != nil {
t.Fatalf("Request failed: %v", err)
}
defer resp.Body.Close()
respbody, err = io.ReadAll(resp.Body)
if resp.StatusCode != 404 {
t.Fatalf("Invalid Status Code received, expected 404, got %d", resp.StatusCode)
}
if !strings.Contains(string(respbody), "The requested hash seems to be prefixed with") {
t.Fatalf("Did not receive the expected error message")
}
_, err = html.Parse(strings.NewReader(string(respbody)))
if err != nil {
t.Fatalf("HTML validation failed for error page returned!")
}
}
func TestJsonResponse(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
defer srv.Close()
var resp *http.Response
var respbody []byte
url := srv.URL + "/bzz:/thisShouldFailWith500Code/"
req, err := http.NewRequest("GET", url, nil)
if err != nil {
t.Fatalf("Request failed: %v", err)
}
req.Header.Set("Accept", "application/json")
resp, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("Request failed: %v", err)
}
defer resp.Body.Close()
respbody, err = io.ReadAll(resp.Body)
if resp.StatusCode != 404 {
t.Fatalf("Invalid Status Code received, expected 404, got %d", resp.StatusCode)
}
if !isJSON(string(respbody)) {
t.Fatalf("Expected response to be JSON, received invalid JSON: %s", string(respbody))
}
}
func isJSON(s string) bool {
var js map[string]interface{}
return json.Unmarshal([]byte(s), &js) == nil
}

66
swarm/api/http/roundtripper.go
View file

@ -1,66 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package http
import (
"fmt"
"net/http"
"github.com/XinFinOrg/XDPoSChain/log"
)
/*
http roundtripper to register for bzz url scheme
see https://github.com/XinFinOrg/XDPoSChain/issues/2040
Usage:
import (
"github.com/XinFinOrg/XDPoSChain/common/httpclient"
"github.com/XinFinOrg/XDPoSChain/swarm/api/http"
)
client := httpclient.New()
// for (private) swarm proxy running locally
client.RegisterScheme("bzz", &http.RoundTripper{Port: port})
client.RegisterScheme("bzz-immutable", &http.RoundTripper{Port: port})
client.RegisterScheme("bzz-raw", &http.RoundTripper{Port: port})
The port you give the Roundtripper is the port the swarm proxy is listening on.
If Host is left empty, localhost is assumed.
Using a public gateway, the above few lines gives you the leanest
bzz-scheme aware read-only http client. You really only ever need this
if you need go-native swarm access to bzz addresses.
*/
type RoundTripper struct {
Host string
Port string
}
func (self *RoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) {
host := self.Host
if len(host) == 0 {
host = "localhost"
}
url := fmt.Sprintf("http://%s:%s/%s:/%s/%s", host, self.Port, req.Proto, req.URL.Host, req.URL.Path)
log.Info(fmt.Sprintf("roundtripper: proxying request '%s' to '%s'", req.RequestURI, url))
reqProxy, err := http.NewRequest(req.Method, url, req.Body)
if err != nil {
return nil, err
}
return http.DefaultClient.Do(reqProxy)
}

69
swarm/api/http/roundtripper_test.go
View file

@ -1,69 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package http
import (
"io"
"net"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
func TestRoundTripper(t *testing.T) {
serveMux := http.NewServeMux()
serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
if r.Method == "GET" {
w.Header().Set("Content-Type", "text/plain")
http.ServeContent(w, r, "", time.Unix(0, 0), strings.NewReader(r.RequestURI))
} else {
http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed)
}
})
srv := httptest.NewServer(serveMux)
defer srv.Close()
host, port, _ := net.SplitHostPort(srv.Listener.Addr().String())
rt := &RoundTripper{Host: host, Port: port}
trans := &http.Transport{}
trans.RegisterProtocol("bzz", rt)
client := &http.Client{Transport: trans}
resp, err := client.Get("bzz://test.com/path")
if err != nil {
t.Errorf("expected no error, got %v", err)
return
}
defer func() {
if resp != nil {
resp.Body.Close()
}
}()
content, err := io.ReadAll(resp.Body)
if err != nil {
t.Errorf("expected no error, got %v", err)
return
}
if string(content) != "/HTTP/1.1:/test.com/path" {
t.Errorf("incorrect response from http server: expected '%v', got '%v'", "/HTTP/1.1:/test.com/path", string(content))
}
}

768
swarm/api/http/server.go
View file

@ -1,768 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
/*
A simple http server interface to Swarm
*/
package http
import (
"archive/tar"
"encoding/json"
"errors"
"fmt"
"io"
"mime"
"mime/multipart"
"net/http"
"os"
"path"
"strconv"
"strings"
"time"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/metrics"
"github.com/XinFinOrg/XDPoSChain/swarm/api"
"github.com/XinFinOrg/XDPoSChain/swarm/storage"
"github.com/rs/cors"
)
// setup metrics
var (
postRawCount = metrics.NewRegisteredCounter("api.http.post.raw.count", nil)
postRawFail = metrics.NewRegisteredCounter("api.http.post.raw.fail", nil)
postFilesCount = metrics.NewRegisteredCounter("api.http.post.files.count", nil)
postFilesFail = metrics.NewRegisteredCounter("api.http.post.files.fail", nil)
deleteCount = metrics.NewRegisteredCounter("api.http.delete.count", nil)
deleteFail = metrics.NewRegisteredCounter("api.http.delete.fail", nil)
getCount = metrics.NewRegisteredCounter("api.http.get.count", nil)
getFail = metrics.NewRegisteredCounter("api.http.get.fail", nil)
getFileCount = metrics.NewRegisteredCounter("api.http.get.file.count", nil)
getFileNotFound = metrics.NewRegisteredCounter("api.http.get.file.notfound", nil)
getFileFail = metrics.NewRegisteredCounter("api.http.get.file.fail", nil)
getFilesCount = metrics.NewRegisteredCounter("api.http.get.files.count", nil)
getFilesFail = metrics.NewRegisteredCounter("api.http.get.files.fail", nil)
getListCount = metrics.NewRegisteredCounter("api.http.get.list.count", nil)
getListFail = metrics.NewRegisteredCounter("api.http.get.list.fail", nil)
requestCount = metrics.NewRegisteredCounter("http.request.count", nil)
htmlRequestCount = metrics.NewRegisteredCounter("http.request.html.count", nil)
jsonRequestCount = metrics.NewRegisteredCounter("http.request.json.count", nil)
requestTimer = metrics.NewRegisteredResettingTimer("http.request.time", nil)
)
// ServerConfig is the basic configuration needed for the HTTP server and also
// includes CORS settings.
type ServerConfig struct {
Addr string
CorsString string
}
// browser API for registering bzz url scheme handlers:
// https://developer.mozilla.org/en/docs/Web-based_protocol_handlers
// electron (chromium) api for registering bzz url scheme handlers:
// https://github.com/atom/electron/blob/master/docs/api/protocol.md
// starts up http server
func StartHttpServer(api *api.Api, config *ServerConfig) {
var allowedOrigins []string
for _, domain := range strings.Split(config.CorsString, ",") {
allowedOrigins = append(allowedOrigins, strings.TrimSpace(domain))
}
c := cors.New(cors.Options{
AllowedOrigins: allowedOrigins,
AllowedMethods: []string{"POST", "GET", "DELETE", "PATCH", "PUT"},
MaxAge: 600,
AllowedHeaders: []string{"*"},
})
hdlr := c.Handler(NewServer(api))
go http.ListenAndServe(config.Addr, hdlr)
}
func NewServer(api *api.Api) *Server {
return &Server{api}
}
type Server struct {
api *api.Api
}
// Request wraps http.Request and also includes the parsed bzz URI
type Request struct {
http.Request
uri *api.URI
}
// HandlePostRaw handles a POST request to a raw bzz-raw:/ URI, stores the request
// body in swarm and returns the resulting storage key as a text/plain response
func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) {
postRawCount.Inc(1)
if r.uri.Path != "" {
postRawFail.Inc(1)
s.BadRequest(w, r, "raw POST request cannot contain a path")
return
}
if r.Header.Get("Content-Length") == "" {
postRawFail.Inc(1)
s.BadRequest(w, r, "missing Content-Length header in request")
return
}
key, err := s.api.Store(r.Body, r.ContentLength, nil)
if err != nil {
postRawFail.Inc(1)
s.Error(w, r, err)
return
}
s.logDebug("content for %s stored", key.Log())
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, key)
}
// HandlePostFiles handles a POST request (or deprecated PUT request) to
// bzz:/<hash>/<path> which contains either a single file or multiple files
// (either a tar archive or multipart form), adds those files either to an
// existing manifest or to a new manifest under <path> and returns the
// resulting manifest hash as a text/plain response
func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) {
postFilesCount.Inc(1)
contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
if err != nil {
postFilesFail.Inc(1)
s.BadRequest(w, r, err.Error())
return
}
var key storage.Key
if r.uri.Addr != "" {
key, err = s.api.Resolve(r.uri)
if err != nil {
postFilesFail.Inc(1)
s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
return
}
} else {
key, err = s.api.NewManifest()
if err != nil {
postFilesFail.Inc(1)
s.Error(w, r, err)
return
}
}
newKey, err := s.updateManifest(key, func(mw *api.ManifestWriter) error {
switch contentType {
case "application/x-tar":
return s.handleTarUpload(r, mw)
case "multipart/form-data":
return s.handleMultipartUpload(r, params["boundary"], mw)
default:
return s.handleDirectUpload(r, mw)
}
})
if err != nil {
postFilesFail.Inc(1)
s.Error(w, r, fmt.Errorf("error creating manifest: %s", err))
return
}
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, newKey)
}
func (s *Server) handleTarUpload(req *Request, mw *api.ManifestWriter) error {
tr := tar.NewReader(req.Body)
for {
hdr, err := tr.Next()
if err == io.EOF {
return nil
} else if err != nil {
return fmt.Errorf("error reading tar stream: %s", err)
}
// only store regular files
if !hdr.FileInfo().Mode().IsRegular() {
continue
}
// add the entry under the path from the request
path := path.Join(req.uri.Path, hdr.Name)
entry := &api.ManifestEntry{
Path: path,
ContentType: hdr.Xattrs["user.swarm.content-type"],
Mode: hdr.Mode,
Size: hdr.Size,
ModTime: hdr.ModTime,
}
s.logDebug("adding %s (%d bytes) to new manifest", entry.Path, entry.Size)
contentKey, err := mw.AddEntry(tr, entry)
if err != nil {
return fmt.Errorf("error adding manifest entry from tar stream: %s", err)
}
s.logDebug("content for %s stored", contentKey.Log())
}
}
func (s *Server) handleMultipartUpload(req *Request, boundary string, mw *api.ManifestWriter) error {
mr := multipart.NewReader(req.Body, boundary)
for {
part, err := mr.NextPart()
if err == io.EOF {
return nil
} else if err != nil {
return fmt.Errorf("error reading multipart form: %s", err)
}
var size int64
var reader io.Reader = part
if contentLength := part.Header.Get("Content-Length"); contentLength != "" {
size, err = strconv.ParseInt(contentLength, 10, 64)
if err != nil {
return fmt.Errorf("error parsing multipart content length: %s", err)
}
reader = part
} else {
// copy the part to a tmp file to get its size
tmp, err := os.CreateTemp("", "swarm-multipart")
if err != nil {
return err
}
defer os.Remove(tmp.Name())
defer tmp.Close()
size, err = io.Copy(tmp, part)
if err != nil {
return fmt.Errorf("error copying multipart content: %s", err)
}
if _, err := tmp.Seek(0, io.SeekStart); err != nil {
return fmt.Errorf("error copying multipart content: %s", err)
}
reader = tmp
}
// add the entry under the path from the request
name := part.FileName()
if name == "" {
name = part.FormName()
}
path := path.Join(req.uri.Path, name)
entry := &api.ManifestEntry{
Path: path,
ContentType: part.Header.Get("Content-Type"),
Size: size,
ModTime: time.Now(),
}
s.logDebug("adding %s (%d bytes) to new manifest", entry.Path, entry.Size)
contentKey, err := mw.AddEntry(reader, entry)
if err != nil {
return fmt.Errorf("error adding manifest entry from multipart form: %s", err)
}
s.logDebug("content for %s stored", contentKey.Log())
}
}
func (s *Server) handleDirectUpload(req *Request, mw *api.ManifestWriter) error {
key, err := mw.AddEntry(req.Body, &api.ManifestEntry{
Path: req.uri.Path,
ContentType: req.Header.Get("Content-Type"),
Mode: 0644,
Size: req.ContentLength,
ModTime: time.Now(),
})
if err != nil {
return err
}
s.logDebug("content for %s stored", key.Log())
return nil
}
// HandleDelete handles a DELETE request to bzz:/<manifest>/<path>, removes
// <path> from <manifest> and returns the resulting manifest hash as a
// text/plain response
func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) {
deleteCount.Inc(1)
key, err := s.api.Resolve(r.uri)
if err != nil {
deleteFail.Inc(1)
s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
return
}
newKey, err := s.updateManifest(key, func(mw *api.ManifestWriter) error {
s.logDebug("removing %s from manifest %s", r.uri.Path, key.Log())
return mw.RemoveEntry(r.uri.Path)
})
if err != nil {
deleteFail.Inc(1)
s.Error(w, r, fmt.Errorf("error updating manifest: %s", err))
return
}
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, newKey)
}
// HandleGet handles a GET request to
// - bzz-raw://<key> and responds with the raw content stored at the
// given storage key
// - bzz-hash://<key> and responds with the hash of the content stored
// at the given storage key as a text/plain response
func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
getCount.Inc(1)
key, err := s.api.Resolve(r.uri)
if err != nil {
getFail.Inc(1)
s.NotFound(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
return
}
// if path is set, interpret <key> as a manifest and return the
// raw entry at the given path
if r.uri.Path != "" {
walker, err := s.api.NewManifestWalker(key, nil)
if err != nil {
getFail.Inc(1)
s.BadRequest(w, r, fmt.Sprintf("%s is not a manifest", key))
return
}
var entry *api.ManifestEntry
walker.Walk(func(e *api.ManifestEntry) error {
// if the entry matches the path, set entry and stop
// the walk
if e.Path == r.uri.Path {
entry = e
// return an error to cancel the walk
return errors.New("found")
}
// ignore non-manifest files
if e.ContentType != api.ManifestType {
return nil
}
// if the manifest's path is a prefix of the
// requested path, recurse into it by returning
// nil and continuing the walk
if strings.HasPrefix(r.uri.Path, e.Path) {
return nil
}
return api.SkipManifest
})
if entry == nil {
getFail.Inc(1)
s.NotFound(w, r, fmt.Errorf("Manifest entry could not be loaded"))
return
}
key = storage.Key(common.Hex2Bytes(entry.Hash))
}
// check the root chunk exists by retrieving the file's size
reader := s.api.Retrieve(key)
if _, err := reader.Size(nil); err != nil {
getFail.Inc(1)
s.NotFound(w, r, fmt.Errorf("Root chunk not found %s: %s", key, err))
return
}
switch {
case r.uri.Raw() || r.uri.DeprecatedRaw():
// allow the request to overwrite the content type using a query
// parameter
contentType := "application/octet-stream"
if typ := r.URL.Query().Get("content_type"); typ != "" {
contentType = typ
}
w.Header().Set("Content-Type", contentType)
http.ServeContent(w, &r.Request, "", time.Now(), reader)
case r.uri.Hash():
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, key)
}
}
// HandleGetFiles handles a GET request to bzz:/<manifest> with an Accept
// header of "application/x-tar" and returns a tar stream of all files
// contained in the manifest
func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) {
getFilesCount.Inc(1)
if r.uri.Path != "" {
getFilesFail.Inc(1)
s.BadRequest(w, r, "files request cannot contain a path")
return
}
key, err := s.api.Resolve(r.uri)
if err != nil {
getFilesFail.Inc(1)
s.NotFound(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
return
}
walker, err := s.api.NewManifestWalker(key, nil)
if err != nil {
getFilesFail.Inc(1)
s.Error(w, r, err)
return
}
tw := tar.NewWriter(w)
defer tw.Close()
w.Header().Set("Content-Type", "application/x-tar")
w.WriteHeader(http.StatusOK)
err = walker.Walk(func(entry *api.ManifestEntry) error {
// ignore manifests (walk will recurse into them)
if entry.ContentType == api.ManifestType {
return nil
}
// retrieve the entry's key and size
reader := s.api.Retrieve(storage.Key(common.Hex2Bytes(entry.Hash)))
size, err := reader.Size(nil)
if err != nil {
return err
}
// write a tar header for the entry
hdr := &tar.Header{
Name: entry.Path,
Mode: entry.Mode,
Size: size,
ModTime: entry.ModTime,
Xattrs: map[string]string{
"user.swarm.content-type": entry.ContentType,
},
}
if err := tw.WriteHeader(hdr); err != nil {
return err
}
// copy the file into the tar stream
n, err := io.Copy(tw, io.LimitReader(reader, hdr.Size))
if err != nil {
return err
} else if n != size {
return fmt.Errorf("error writing %s: expected %d bytes but sent %d", entry.Path, size, n)
}
return nil
})
if err != nil {
getFilesFail.Inc(1)
s.logError("error generating tar stream: %s", err)
}
}
// HandleGetList handles a GET request to bzz-list:/<manifest>/<path> and returns
// a list of all files contained in <manifest> under <path> grouped into
// common prefixes using "/" as a delimiter
func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
getListCount.Inc(1)
// ensure the root path has a trailing slash so that relative URLs work
if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently)
return
}
key, err := s.api.Resolve(r.uri)
if err != nil {
getListFail.Inc(1)
s.NotFound(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
return
}
list, err := s.getManifestList(key, r.uri.Path)
if err != nil {
getListFail.Inc(1)
s.Error(w, r, err)
return
}
// if the client wants HTML (e.g. a browser) then render the list as a
// HTML index with relative URLs
if strings.Contains(r.Header.Get("Accept"), "text/html") {
w.Header().Set("Content-Type", "text/html")
err := htmlListTemplate.Execute(w, &htmlListData{
URI: &api.URI{
Scheme: "bzz",
Addr: r.uri.Addr,
Path: r.uri.Path,
},
List: &list,
})
if err != nil {
getListFail.Inc(1)
s.logError("error rendering list HTML: %s", err)
}
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(&list)
}
func (s *Server) getManifestList(key storage.Key, prefix string) (list api.ManifestList, err error) {
walker, err := s.api.NewManifestWalker(key, nil)
if err != nil {
return
}
err = walker.Walk(func(entry *api.ManifestEntry) error {
// handle non-manifest files
if entry.ContentType != api.ManifestType {
// ignore the file if it doesn't have the specified prefix
if !strings.HasPrefix(entry.Path, prefix) {
return nil
}
// if the path after the prefix contains a slash, add a
// common prefix to the list, otherwise add the entry
suffix := strings.TrimPrefix(entry.Path, prefix)
if index := strings.Index(suffix, "/"); index > -1 {
list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1])
return nil
}
if entry.Path == "" {
entry.Path = "/"
}
list.Entries = append(list.Entries, entry)
return nil
}
// if the manifest's path is a prefix of the specified prefix
// then just recurse into the manifest by returning nil and
// continuing the walk
if strings.HasPrefix(prefix, entry.Path) {
return nil
}
// if the manifest's path has the specified prefix, then if the
// path after the prefix contains a slash, add a common prefix
// to the list and skip the manifest, otherwise recurse into
// the manifest by returning nil and continuing the walk
if strings.HasPrefix(entry.Path, prefix) {
suffix := strings.TrimPrefix(entry.Path, prefix)
if index := strings.Index(suffix, "/"); index > -1 {
list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1])
return api.SkipManifest
}
return nil
}
// the manifest neither has the prefix or needs recursing in to
// so just skip it
return api.SkipManifest
})
return list, nil
}
// HandleGetFile handles a GET request to bzz://<manifest>/<path> and responds
// with the content of the file at <path> from the given <manifest>
func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
getFileCount.Inc(1)
// ensure the root path has a trailing slash so that relative URLs work
if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently)
return
}
key, err := s.api.Resolve(r.uri)
if err != nil {
getFileFail.Inc(1)
s.NotFound(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
return
}
reader, contentType, status, err := s.api.Get(key, r.uri.Path)
if err != nil {
switch status {
case http.StatusNotFound:
getFileNotFound.Inc(1)
s.NotFound(w, r, err)
default:
getFileFail.Inc(1)
s.Error(w, r, err)
}
return
}
//the request results in ambiguous files
//e.g. /read with readme.md and readinglist.txt available in manifest
if status == http.StatusMultipleChoices {
list, err := s.getManifestList(key, r.uri.Path)
if err != nil {
getFileFail.Inc(1)
s.Error(w, r, err)
return
}
s.logDebug(fmt.Sprintf("Multiple choices! --> %v", list))
//show a nice page links to available entries
ShowMultipleChoices(w, r, list)
return
}
// check the root chunk exists by retrieving the file's size
if _, err := reader.Size(nil); err != nil {
getFileNotFound.Inc(1)
s.NotFound(w, r, fmt.Errorf("File not found %s: %s", r.uri, err))
return
}
w.Header().Set("Content-Type", contentType)
http.ServeContent(w, &r.Request, "", time.Now(), reader)
}
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if metrics.Enabled {
//The increment for request count and request timer themselves have a flag check
//for metrics.Enabled. Nevertheless, we introduce the if here because we
//are looking into the header just to see what request type it is (json/html).
//So let's take advantage and add all metrics related stuff here
requestCount.Inc(1)
defer requestTimer.UpdateSince(time.Now())
if r.Header.Get("Accept") == "application/json" {
jsonRequestCount.Inc(1)
} else {
htmlRequestCount.Inc(1)
}
}
s.logDebug("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, r.URL.Host, r.URL.Path, r.Referer(), r.Header.Get("Accept"))
if r.RequestURI == "/" && strings.Contains(r.Header.Get("Accept"), "text/html") {
err := landingPageTemplate.Execute(w, nil)
if err != nil {
s.logError("error rendering landing page: %s", err)
}
return
}
uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/"))
req := &Request{Request: *r, uri: uri}
if err != nil {
s.logError("Invalid URI %q: %s", r.URL.Path, err)
s.BadRequest(w, req, fmt.Sprintf("Invalid URI %q: %s", r.URL.Path, err))
return
}
s.logDebug("%s request received for %s", r.Method, uri)
switch r.Method {
case "POST":
if uri.Raw() || uri.DeprecatedRaw() {
s.HandlePostRaw(w, req)
} else {
s.HandlePostFiles(w, req)
}
case "PUT":
// DEPRECATED:
// clients should send a POST request (the request creates a
// new manifest leaving the existing one intact, so it isn't
// strictly a traditional PUT request which replaces content
// at a URI, and POST is more ubiquitous)
if uri.Raw() || uri.DeprecatedRaw() {
ShowError(w, req, fmt.Sprintf("No PUT to %s allowed.", uri), http.StatusBadRequest)
return
} else {
s.HandlePostFiles(w, req)
}
case "DELETE":
if uri.Raw() || uri.DeprecatedRaw() {
ShowError(w, req, fmt.Sprintf("No DELETE to %s allowed.", uri), http.StatusBadRequest)
return
}
s.HandleDelete(w, req)
case "GET":
if uri.Raw() || uri.Hash() || uri.DeprecatedRaw() {
s.HandleGet(w, req)
return
}
if uri.List() {
s.HandleGetList(w, req)
return
}
if r.Header.Get("Accept") == "application/x-tar" {
s.HandleGetFiles(w, req)
return
}
s.HandleGetFile(w, req)
default:
ShowError(w, req, fmt.Sprintf("Method "+r.Method+" is not supported.", uri), http.StatusMethodNotAllowed)
}
}
func (s *Server) updateManifest(key storage.Key, update func(mw *api.ManifestWriter) error) (storage.Key, error) {
mw, err := s.api.NewManifestWriter(key, nil)
if err != nil {
return nil, err
}
if err := update(mw); err != nil {
return nil, err
}
key, err = mw.Store()
if err != nil {
return nil, err
}
s.logDebug("generated manifest %s", key)
return key, nil
}
func (s *Server) logDebug(format string, v ...interface{}) {
log.Debug(fmt.Sprintf("[BZZ] HTTP: "+format, v...))
}
func (s *Server) logError(format string, v ...interface{}) {
log.Error(fmt.Sprintf("[BZZ] HTTP: "+format, v...))
}
func (s *Server) BadRequest(w http.ResponseWriter, r *Request, reason string) {
ShowError(w, r, fmt.Sprintf("Bad request %s %s: %s", r.Request.Method, r.uri, reason), http.StatusBadRequest)
}
func (s *Server) Error(w http.ResponseWriter, r *Request, err error) {
ShowError(w, r, fmt.Sprintf("Error serving %s %s: %s", r.Request.Method, r.uri, err), http.StatusInternalServerError)
}
func (s *Server) NotFound(w http.ResponseWriter, r *Request, err error) {
ShowError(w, r, fmt.Sprintf("NOT FOUND error serving %s %s: %s", r.Request.Method, r.uri, err), http.StatusNotFound)
}

320
swarm/api/http/server_test.go
View file

File diff suppressed because one or more lines are too long

215
swarm/api/http/templates.go
View file

File diff suppressed because one or more lines are too long

504
swarm/api/manifest.go
View file

@ -1,504 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"sync"
"time"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/swarm/storage"
)
const (
ManifestType = "application/bzz-manifest+json"
)
// Manifest represents a swarm manifest
type Manifest struct {
Entries []ManifestEntry `json:"entries,omitempty"`
}
// ManifestEntry represents an entry in a swarm manifest
type ManifestEntry struct {
Hash string `json:"hash,omitempty"`
Path string `json:"path,omitempty"`
ContentType string `json:"contentType,omitempty"`
Mode int64 `json:"mode,omitempty"`
Size int64 `json:"size,omitempty"`
ModTime time.Time `json:"mod_time,omitempty"`
Status int `json:"status,omitempty"`
}
// ManifestList represents the result of listing files in a manifest
type ManifestList struct {
CommonPrefixes []string `json:"common_prefixes,omitempty"`
Entries []*ManifestEntry `json:"entries,omitempty"`
}
// NewManifest creates and stores a new, empty manifest
func (a *Api) NewManifest() (storage.Key, error) {
var manifest Manifest
data, err := json.Marshal(&manifest)
if err != nil {
return nil, err
}
return a.Store(bytes.NewReader(data), int64(len(data)), &sync.WaitGroup{})
}
// ManifestWriter is used to add and remove entries from an underlying manifest
type ManifestWriter struct {
api *Api
trie *manifestTrie
quitC chan bool
}
func (a *Api) NewManifestWriter(key storage.Key, quitC chan bool) (*ManifestWriter, error) {
trie, err := loadManifest(a.dpa, key, quitC)
if err != nil {
return nil, fmt.Errorf("error loading manifest %s: %s", key, err)
}
return &ManifestWriter{a, trie, quitC}, nil
}
// AddEntry stores the given data and adds the resulting key to the manifest
func (m *ManifestWriter) AddEntry(data io.Reader, e *ManifestEntry) (storage.Key, error) {
key, err := m.api.Store(data, e.Size, nil)
if err != nil {
return nil, err
}
entry := newManifestTrieEntry(e, nil)
entry.Hash = key.String()
m.trie.addEntry(entry, m.quitC)
return key, nil
}
// RemoveEntry removes the given path from the manifest
func (m *ManifestWriter) RemoveEntry(path string) error {
m.trie.deleteEntry(path, m.quitC)
return nil
}
// Store stores the manifest, returning the resulting storage key
func (m *ManifestWriter) Store() (storage.Key, error) {
return m.trie.hash, m.trie.recalcAndStore()
}
// ManifestWalker is used to recursively walk the entries in the manifest and
// all of its submanifests
type ManifestWalker struct {
api *Api
trie *manifestTrie
quitC chan bool
}
func (a *Api) NewManifestWalker(key storage.Key, quitC chan bool) (*ManifestWalker, error) {
trie, err := loadManifest(a.dpa, key, quitC)
if err != nil {
return nil, fmt.Errorf("error loading manifest %s: %s", key, err)
}
return &ManifestWalker{a, trie, quitC}, nil
}
// SkipManifest is used as a return value from WalkFn to indicate that the
// manifest should be skipped
var SkipManifest = errors.New("skip this manifest")
// WalkFn is the type of function called for each entry visited by a recursive
// manifest walk
type WalkFn func(entry *ManifestEntry) error
// Walk recursively walks the manifest calling walkFn for each entry in the
// manifest, including submanifests
func (m *ManifestWalker) Walk(walkFn WalkFn) error {
return m.walk(m.trie, "", walkFn)
}
func (m *ManifestWalker) walk(trie *manifestTrie, prefix string, walkFn WalkFn) error {
for _, entry := range trie.entries {
if entry == nil {
continue
}
entry.Path = prefix + entry.Path
err := walkFn(&entry.ManifestEntry)
if err != nil {
if entry.ContentType == ManifestType && err == SkipManifest {
continue
}
return err
}
if entry.ContentType != ManifestType {
continue
}
if err := trie.loadSubTrie(entry, nil); err != nil {
return err
}
if err := m.walk(entry.subtrie, entry.Path, walkFn); err != nil {
return err
}
}
return nil
}
type manifestTrie struct {
dpa *storage.DPA
entries [257]*manifestTrieEntry // indexed by first character of basePath, entries[256] is the empty basePath entry
hash storage.Key // if hash != nil, it is stored
}
func newManifestTrieEntry(entry *ManifestEntry, subtrie *manifestTrie) *manifestTrieEntry {
return &manifestTrieEntry{
ManifestEntry: *entry,
subtrie: subtrie,
}
}
type manifestTrieEntry struct {
ManifestEntry
subtrie *manifestTrie
}
func loadManifest(dpa *storage.DPA, hash storage.Key, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
log.Trace(fmt.Sprintf("manifest lookup key: '%v'.", hash.Log()))
// retrieve manifest via DPA
manifestReader := dpa.Retrieve(hash)
return readManifest(manifestReader, hash, dpa, quitC)
}
func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dpa *storage.DPA, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
// TODO check size for oversized manifests
size, err := manifestReader.Size(quitC)
if err != nil { // size == 0
// can't determine size means we don't have the root chunk
err = fmt.Errorf("Manifest not Found")
return
}
manifestData := make([]byte, size)
read, err := manifestReader.Read(manifestData)
if int64(read) < size {
log.Trace(fmt.Sprintf("Manifest %v not found.", hash.Log()))
if err == nil {
err = fmt.Errorf("Manifest retrieval cut short: read %v, expect %v", read, size)
}
return
}
log.Trace(fmt.Sprintf("Manifest %v retrieved", hash.Log()))
var man struct {
Entries []*manifestTrieEntry `json:"entries"`
}
err = json.Unmarshal(manifestData, &man)
if err != nil {
err = fmt.Errorf("Manifest %v is malformed: %v", hash.Log(), err)
log.Trace(fmt.Sprintf("%v", err))
return
}
log.Trace(fmt.Sprintf("Manifest %v has %d entries.", hash.Log(), len(man.Entries)))
trie = &manifestTrie{
dpa: dpa,
}
for _, entry := range man.Entries {
trie.addEntry(entry, quitC)
}
return
}
func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
self.hash = nil // trie modified, hash needs to be re-calculated on demand
if len(entry.Path) == 0 {
self.entries[256] = entry
return
}
b := entry.Path[0]
oldentry := self.entries[b]
if (oldentry == nil) || (oldentry.Path == entry.Path && oldentry.ContentType != ManifestType) {
self.entries[b] = entry
return
}
cpl := 0
for (len(entry.Path) > cpl) && (len(oldentry.Path) > cpl) && (entry.Path[cpl] == oldentry.Path[cpl]) {
cpl++
}
if (oldentry.ContentType == ManifestType) && (cpl == len(oldentry.Path)) {
if self.loadSubTrie(oldentry, quitC) != nil {
return
}
entry.Path = entry.Path[cpl:]
oldentry.subtrie.addEntry(entry, quitC)
oldentry.Hash = ""
return
}
commonPrefix := entry.Path[:cpl]
subtrie := &manifestTrie{
dpa: self.dpa,
}
entry.Path = entry.Path[cpl:]
oldentry.Path = oldentry.Path[cpl:]
subtrie.addEntry(entry, quitC)
subtrie.addEntry(oldentry, quitC)
self.entries[b] = newManifestTrieEntry(&ManifestEntry{
Path: commonPrefix,
ContentType: ManifestType,
}, subtrie)
}
func (self *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) {
for _, e := range self.entries {
if e != nil {
cnt++
entry = e
}
}
return
}
func (self *manifestTrie) deleteEntry(path string, quitC chan bool) {
self.hash = nil // trie modified, hash needs to be re-calculated on demand
if len(path) == 0 {
self.entries[256] = nil
return
}
b := path[0]
entry := self.entries[b]
if entry == nil {
return
}
if entry.Path == path {
self.entries[b] = nil
return
}
epl := len(entry.Path)
if (entry.ContentType == ManifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) {
if self.loadSubTrie(entry, quitC) != nil {
return
}
entry.subtrie.deleteEntry(path[epl:], quitC)
entry.Hash = ""
// remove subtree if it has less than 2 elements
cnt, lastentry := entry.subtrie.getCountLast()
if cnt < 2 {
if lastentry != nil {
lastentry.Path = entry.Path + lastentry.Path
}
self.entries[b] = lastentry
}
}
}
func (self *manifestTrie) recalcAndStore() error {
if self.hash != nil {
return nil
}
var buffer bytes.Buffer
buffer.WriteString(`{"entries":[`)
list := &Manifest{}
for _, entry := range self.entries {
if entry != nil {
if entry.Hash == "" { // TODO: paralellize
err := entry.subtrie.recalcAndStore()
if err != nil {
return err
}
entry.Hash = entry.subtrie.hash.String()
}
list.Entries = append(list.Entries, entry.ManifestEntry)
}
}
manifest, err := json.Marshal(list)
if err != nil {
return err
}
sr := bytes.NewReader(manifest)
wg := &sync.WaitGroup{}
key, err2 := self.dpa.Store(sr, int64(len(manifest)), wg, nil)
wg.Wait()
self.hash = key
return err2
}
func (self *manifestTrie) loadSubTrie(entry *manifestTrieEntry, quitC chan bool) (err error) {
if entry.subtrie == nil {
hash := common.Hex2Bytes(entry.Hash)
entry.subtrie, err = loadManifest(self.dpa, hash, quitC)
entry.Hash = "" // might not match, should be recalculated
}
return
}
func (self *manifestTrie) listWithPrefixInt(prefix, rp string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) error {
plen := len(prefix)
var start, stop int
if plen == 0 {
start = 0
stop = 256
} else {
start = int(prefix[0])
stop = start
}
for i := start; i <= stop; i++ {
select {
case <-quitC:
return fmt.Errorf("aborted")
default:
}
entry := self.entries[i]
if entry != nil {
epl := len(entry.Path)
if entry.ContentType == ManifestType {
l := plen
if epl < l {
l = epl
}
if prefix[:l] == entry.Path[:l] {
err := self.loadSubTrie(entry, quitC)
if err != nil {
return err
}
err = entry.subtrie.listWithPrefixInt(prefix[l:], rp+entry.Path[l:], quitC, cb)
if err != nil {
return err
}
}
} else {
if (epl >= plen) && (prefix == entry.Path[:plen]) {
cb(entry, rp+entry.Path[plen:])
}
}
}
}
return nil
}
func (self *manifestTrie) listWithPrefix(prefix string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) (err error) {
return self.listWithPrefixInt(prefix, "", quitC, cb)
}
func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *manifestTrieEntry, pos int) {
log.Trace(fmt.Sprintf("findPrefixOf(%s)", path))
if len(path) == 0 {
return self.entries[256], 0
}
//see if first char is in manifest entries
b := path[0]
entry = self.entries[b]
if entry == nil {
return self.entries[256], 0
}
epl := len(entry.Path)
log.Trace(fmt.Sprintf("path = %v entry.Path = %v epl = %v", path, entry.Path, epl))
if len(path) <= epl {
if entry.Path[:len(path)] == path {
if entry.ContentType == ManifestType {
err := self.loadSubTrie(entry, quitC)
if err == nil && entry.subtrie != nil {
subentries := entry.subtrie.entries
for i := 0; i < len(subentries); i++ {
sub := subentries[i]
if sub != nil && sub.Path == "" {
return sub, len(path)
}
}
}
entry.Status = http.StatusMultipleChoices
}
pos = len(path)
return
}
return nil, 0
}
if path[:epl] == entry.Path {
log.Trace(fmt.Sprintf("entry.ContentType = %v", entry.ContentType))
//the subentry is a manifest, load subtrie
if entry.ContentType == ManifestType && (strings.Contains(entry.Path, path) || strings.Contains(path, entry.Path)) {
err := self.loadSubTrie(entry, quitC)
if err != nil {
return nil, 0
}
sub, pos := entry.subtrie.findPrefixOf(path[epl:], quitC)
if sub != nil {
entry = sub
pos += epl
return sub, pos
} else if path == entry.Path {
entry.Status = http.StatusMultipleChoices
}
} else {
//entry is not a manifest, return it
if path != entry.Path {
return nil, 0
}
pos = epl
}
}
return
}
// file system manifest always contains regularized paths
// no leading or trailing slashes, only single slashes inside
func RegularSlashes(path string) (res string) {
for i := 0; i < len(path); i++ {
if (path[i] != '/') || ((i > 0) && (path[i-1] != '/')) {
res = res + path[i:i+1]
}
}
if (len(res) > 0) && (res[len(res)-1] == '/') {
res = res[:len(res)-1]
}
return
}
func (self *manifestTrie) getEntry(spath string) (entry *manifestTrieEntry, fullpath string) {
path := RegularSlashes(spath)
var pos int
quitC := make(chan bool)
entry, pos = self.findPrefixOf(path, quitC)
return entry, path[:pos]
}

146
swarm/api/manifest_test.go
View file

@ -1,146 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"testing"
"github.com/XinFinOrg/XDPoSChain/swarm/storage"
)
func manifest(paths ...string) (manifestReader storage.LazySectionReader) {
var entries []string
for _, path := range paths {
entry := fmt.Sprintf(`{"path":"%s"}`, path)
entries = append(entries, entry)
}
manifest := fmt.Sprintf(`{"entries":[%s]}`, strings.Join(entries, ","))
return &storage.LazyTestSectionReader{
SectionReader: io.NewSectionReader(strings.NewReader(manifest), 0, int64(len(manifest))),
}
}
func testGetEntry(t *testing.T, path, match string, multiple bool, paths ...string) *manifestTrie {
quitC := make(chan bool)
trie, err := readManifest(manifest(paths...), nil, nil, quitC)
if err != nil {
t.Errorf("unexpected error making manifest: %v", err)
}
checkEntry(t, path, match, multiple, trie)
return trie
}
func checkEntry(t *testing.T, path, match string, multiple bool, trie *manifestTrie) {
entry, fullpath := trie.getEntry(path)
if match == "-" && entry != nil {
t.Errorf("expected no match for '%s', got '%s'", path, fullpath)
} else if entry == nil {
if match != "-" {
t.Errorf("expected entry '%s' to match '%s', got no match", match, path)
}
} else if fullpath != match {
t.Errorf("incorrect entry retrieved for '%s'. expected path '%v', got '%s'", path, match, fullpath)
}
if multiple && entry.Status != http.StatusMultipleChoices {
t.Errorf("Expected %d Multiple Choices Status for path %s, match %s, got %d", http.StatusMultipleChoices, path, match, entry.Status)
} else if !multiple && entry != nil && entry.Status == http.StatusMultipleChoices {
t.Errorf("Were not expecting %d Multiple Choices Status for path %s, match %s, but got it", http.StatusMultipleChoices, path, match)
}
}
func TestGetEntry(t *testing.T) {
// file system manifest always contains regularized paths
testGetEntry(t, "a", "a", false, "a")
testGetEntry(t, "b", "-", false, "a")
testGetEntry(t, "/a//", "a", false, "a")
// fallback
testGetEntry(t, "/a", "", false, "")
testGetEntry(t, "/a/b", "a/b", false, "a/b")
// longest/deepest math
testGetEntry(t, "read", "read", true, "readme.md", "readit.md")
testGetEntry(t, "rf", "-", false, "readme.md", "readit.md")
testGetEntry(t, "readme", "readme", false, "readme.md")
testGetEntry(t, "readme", "-", false, "readit.md")
testGetEntry(t, "readme.md", "readme.md", false, "readme.md")
testGetEntry(t, "readme.md", "-", false, "readit.md")
testGetEntry(t, "readmeAmd", "-", false, "readit.md")
testGetEntry(t, "readme.mdffff", "-", false, "readme.md")
testGetEntry(t, "ab", "ab", true, "ab/cefg", "ab/cedh", "ab/kkkkkk")
testGetEntry(t, "ab/ce", "ab/ce", true, "ab/cefg", "ab/cedh", "ab/ceuuuuuuuuuu")
testGetEntry(t, "abc", "abc", true, "abcd", "abczzzzef", "abc/def", "abc/e/g")
testGetEntry(t, "a/b", "a/b", true, "a", "a/bc", "a/ba", "a/b/c")
testGetEntry(t, "a/b", "a/b", false, "a", "a/b", "a/bb", "a/b/c")
testGetEntry(t, "//a//b//", "a/b", false, "a", "a/b", "a/bb", "a/b/c")
}
func TestExactMatch(t *testing.T) {
quitC := make(chan bool)
mf := manifest("shouldBeExactMatch.css", "shouldBeExactMatch.css.map")
trie, err := readManifest(mf, nil, nil, quitC)
if err != nil {
t.Errorf("unexpected error making manifest: %v", err)
}
entry, _ := trie.getEntry("shouldBeExactMatch.css")
if entry.Path != "" {
t.Errorf("Expected entry to match %s, got: %s", "shouldBeExactMatch.css", entry.Path)
}
if entry.Status == http.StatusMultipleChoices {
t.Errorf("Got status %d, which is unexepcted", http.StatusMultipleChoices)
}
}
func TestDeleteEntry(t *testing.T) {
}
// TestAddFileWithManifestPath tests that adding an entry at a path which
// already exists as a manifest just adds the entry to the manifest rather
// than replacing the manifest with the entry
func TestAddFileWithManifestPath(t *testing.T) {
// create a manifest containing "ab" and "ac"
manifest, _ := json.Marshal(&Manifest{
Entries: []ManifestEntry{
{Path: "ab", Hash: "ab"},
{Path: "ac", Hash: "ac"},
},
})
reader := &storage.LazyTestSectionReader{
SectionReader: io.NewSectionReader(bytes.NewReader(manifest), 0, int64(len(manifest))),
}
trie, err := readManifest(reader, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
checkEntry(t, "ab", "ab", false, trie)
checkEntry(t, "ac", "ac", false, trie)
// now add path "a" and check we can still get "ab" and "ac"
entry := &manifestTrieEntry{}
entry.Path = "a"
entry.Hash = "a"
trie.addEntry(entry, nil)
checkEntry(t, "ab", "ab", false, trie)
checkEntry(t, "ac", "ac", false, trie)
checkEntry(t, "a", "a", false, trie)
}

104
swarm/api/storage.go
View file

@ -1,104 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import "path"
type Response struct {
MimeType string
Status int
Size int64
// Content []byte
Content string
}
// implements a service
//
// DEPRECATED: Use the HTTP API instead
type Storage struct {
api *Api
}
func NewStorage(api *Api) *Storage {
return &Storage{api}
}
// Put uploads the content to the swarm with a simple manifest speficying
// its content type
//
// DEPRECATED: Use the HTTP API instead
func (self *Storage) Put(content, contentType string) (string, error) {
key, err := self.api.Put(content, contentType)
if err != nil {
return "", err
}
return key.String(), err
}
// Get retrieves the content from bzzpath and reads the response in full
// It returns the Response object, which serialises containing the
// response body as the value of the Content field
// NOTE: if error is non-nil, sResponse may still have partial content
// the actual size of which is given in len(resp.Content), while the expected
// size is resp.Size
//
// DEPRECATED: Use the HTTP API instead
func (self *Storage) Get(bzzpath string) (*Response, error) {
uri, err := Parse(path.Join("bzz:/", bzzpath))
if err != nil {
return nil, err
}
key, err := self.api.Resolve(uri)
if err != nil {
return nil, err
}
reader, mimeType, status, err := self.api.Get(key, uri.Path)
if err != nil {
return nil, err
}
quitC := make(chan bool)
expsize, err := reader.Size(quitC)
if err != nil {
return nil, err
}
body := make([]byte, expsize)
size, err := reader.Read(body)
if int64(size) == expsize {
err = nil
}
return &Response{mimeType, status, expsize, string(body[:size])}, err
}
// Modify(rootHash, basePath, contentHash, contentType) takes th e manifest trie rooted in rootHash,
// and merge on to it. creating an entry w conentType (mime)
//
// DEPRECATED: Use the HTTP API instead
func (self *Storage) Modify(rootHash, path, contentHash, contentType string) (newRootHash string, err error) {
uri, err := Parse("bzz:/" + rootHash)
if err != nil {
return "", err
}
key, err := self.api.Resolve(uri)
if err != nil {
return "", err
}
key, err = self.api.Modify(key, path, contentHash, contentType)
if err != nil {
return "", err
}
return key.String(), nil
}

49
swarm/api/storage_test.go
View file

@ -1,49 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"testing"
)
func testStorage(t *testing.T, f func(*Storage)) {
testApi(t, func(api *Api) {
f(NewStorage(api))
})
}
func TestStoragePutGet(t *testing.T) {
testStorage(t, func(api *Storage) {
content := "hello"
exp := expResponse(content, "text/plain", 0)
// exp := expResponse([]byte(content), "text/plain", 0)
bzzhash, err := api.Put(content, exp.MimeType)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// to check put against the Api#Get
resp0 := testGet(t, api.api, bzzhash, "")
checkResponse(t, resp0, exp)
// check storage#Get
resp, err := api.Get(bzzhash)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
checkResponse(t, &testResponse{nil, resp}, exp)
})
}

46
swarm/api/testapi.go
View file

@ -1,46 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"github.com/XinFinOrg/XDPoSChain/swarm/network"
)
type Control struct {
api *Api
hive *network.Hive
}
func NewControl(api *Api, hive *network.Hive) *Control {
return &Control{api, hive}
}
func (self *Control) BlockNetworkRead(on bool) {
self.hive.BlockNetworkRead(on)
}
func (self *Control) SyncEnabled(on bool) {
self.hive.SyncEnabled(on)
}
func (self *Control) SwapEnabled(on bool) {
self.hive.SwapEnabled(on)
}
func (self *Control) Hive() string {
return self.hive.String()
}

BIN
swarm/api/testdata/test0/img/logo.png vendored
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 18 KiB

9
swarm/api/testdata/test0/index.css vendored
View file

@ -1,9 +0,0 @@
h1 {
color: black;
font-size: 12px;
background-color: orange;
border: 4px solid black;
}
body {
background-color: orange
}

10
swarm/api/testdata/test0/index.html vendored
View file

@ -1,10 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<link rel="stylesheet" href="index.css">
</head>
<body>
<h1>Swarm Test</h1>
<img src="img/logo.gif" align="center", alt="Ethereum logo">
</body>
</html>

121
swarm/api/uri.go
View file

@ -1,121 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"fmt"
"net/url"
"strings"
)
// URI is a reference to content stored in swarm.
type URI struct {
// Scheme has one of the following values:
//
// * bzz - an entry in a swarm manifest
// * bzz-raw - raw swarm content
// * bzz-immutable - immutable URI of an entry in a swarm manifest
// (address is not resolved)
// * bzz-list - list of all files contained in a swarm manifest
//
// Deprecated Schemes:
// * bzzr - raw swarm content
// * bzzi - immutable URI of an entry in a swarm manifest
// (address is not resolved)
// * bzz-hash - hash of swarm content
//
Scheme string
// Addr is either a hexadecimal storage key or it an address which
// resolves to a storage key
Addr string
// Path is the path to the content within a swarm manifest
Path string
}
// Parse parses rawuri into a URI struct, where rawuri is expected to have one
// of the following formats:
//
// * <scheme>:/
// * <scheme>:/<addr>
// * <scheme>:/<addr>/<path>
// * <scheme>://
// * <scheme>://<addr>
// * <scheme>://<addr>/<path>
//
// with scheme one of bzz, bzz-raw, bzz-immutable, bzz-list or bzz-hash
// or deprecated ones bzzr and bzzi
func Parse(rawuri string) (*URI, error) {
u, err := url.Parse(rawuri)
if err != nil {
return nil, err
}
uri := &URI{Scheme: u.Scheme}
// check the scheme is valid
switch uri.Scheme {
case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzz-hash", "bzzr", "bzzi":
default:
return nil, fmt.Errorf("unknown scheme %q", u.Scheme)
}
// handle URIs like bzz://<addr>/<path> where the addr and path
// have already been split by url.Parse
if u.Host != "" {
uri.Addr = u.Host
uri.Path = strings.TrimLeft(u.Path, "/")
return uri, nil
}
// URI is like bzz:/<addr>/<path> so split the addr and path from
// the raw path (which will be /<addr>/<path>)
parts := strings.SplitN(strings.TrimLeft(u.Path, "/"), "/", 2)
uri.Addr = parts[0]
if len(parts) == 2 {
uri.Path = parts[1]
}
return uri, nil
}
func (u *URI) Raw() bool {
return u.Scheme == "bzz-raw"
}
func (u *URI) Immutable() bool {
return u.Scheme == "bzz-immutable"
}
func (u *URI) List() bool {
return u.Scheme == "bzz-list"
}
func (u *URI) DeprecatedRaw() bool {
return u.Scheme == "bzzr"
}
func (u *URI) DeprecatedImmutable() bool {
return u.Scheme == "bzzi"
}
func (u *URI) Hash() bool {
return u.Scheme == "bzz-hash"
}
func (u *URI) String() string {
return u.Scheme + ":/" + u.Addr + "/" + u.Path
}

176
swarm/api/uri_test.go
View file

@ -1,176 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"reflect"
"testing"
)
func TestParseURI(t *testing.T) {
type test struct {
uri string
expectURI *URI
expectErr bool
expectRaw bool
expectImmutable bool
expectList bool
expectHash bool
expectDeprecatedRaw bool
expectDeprecatedImmutable bool
}
tests := []test{
{
uri: "",
expectErr: true,
},
{
uri: "foo",
expectErr: true,
},
{
uri: "bzz",
expectErr: true,
},
{
uri: "bzz:",
expectURI: &URI{Scheme: "bzz"},
},
{
uri: "bzz-immutable:",
expectURI: &URI{Scheme: "bzz-immutable"},
expectImmutable: true,
},
{
uri: "bzz-raw:",
expectURI: &URI{Scheme: "bzz-raw"},
expectRaw: true,
},
{
uri: "bzz:/",
expectURI: &URI{Scheme: "bzz"},
},
{
uri: "bzz:/abc123",
expectURI: &URI{Scheme: "bzz", Addr: "abc123"},
},
{
uri: "bzz:/abc123/path/to/entry",
expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"},
},
{
uri: "bzz-raw:/",
expectURI: &URI{Scheme: "bzz-raw"},
expectRaw: true,
},
{
uri: "bzz-raw:/abc123",
expectURI: &URI{Scheme: "bzz-raw", Addr: "abc123"},
expectRaw: true,
},
{
uri: "bzz-raw:/abc123/path/to/entry",
expectURI: &URI{Scheme: "bzz-raw", Addr: "abc123", Path: "path/to/entry"},
expectRaw: true,
},
{
uri: "bzz://",
expectURI: &URI{Scheme: "bzz"},
},
{
uri: "bzz://abc123",
expectURI: &URI{Scheme: "bzz", Addr: "abc123"},
},
{
uri: "bzz://abc123/path/to/entry",
expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"},
},
{
uri: "bzz-hash:",
expectURI: &URI{Scheme: "bzz-hash"},
expectHash: true,
},
{
uri: "bzz-hash:/",
expectURI: &URI{Scheme: "bzz-hash"},
expectHash: true,
},
{
uri: "bzz-list:",
expectURI: &URI{Scheme: "bzz-list"},
expectList: true,
},
{
uri: "bzz-list:/",
expectURI: &URI{Scheme: "bzz-list"},
expectList: true,
},
{
uri: "bzzr:",
expectURI: &URI{Scheme: "bzzr"},
expectDeprecatedRaw: true,
},
{
uri: "bzzr:/",
expectURI: &URI{Scheme: "bzzr"},
expectDeprecatedRaw: true,
},
{
uri: "bzzi:",
expectURI: &URI{Scheme: "bzzi"},
expectDeprecatedImmutable: true,
},
{
uri: "bzzi:/",
expectURI: &URI{Scheme: "bzzi"},
expectDeprecatedImmutable: true,
},
}
for _, x := range tests {
actual, err := Parse(x.uri)
if x.expectErr {
if err == nil {
t.Fatalf("expected %s to error", x.uri)
}
continue
}
if err != nil {
t.Fatalf("error parsing %s: %s", x.uri, err)
}
if !reflect.DeepEqual(actual, x.expectURI) {
t.Fatalf("expected %s to return %#v, got %#v", x.uri, x.expectURI, actual)
}
if actual.Raw() != x.expectRaw {
t.Fatalf("expected %s raw to be %t, got %t", x.uri, x.expectRaw, actual.Raw())
}
if actual.Immutable() != x.expectImmutable {
t.Fatalf("expected %s immutable to be %t, got %t", x.uri, x.expectImmutable, actual.Immutable())
}
if actual.List() != x.expectList {
t.Fatalf("expected %s list to be %t, got %t", x.uri, x.expectList, actual.List())
}
if actual.Hash() != x.expectHash {
t.Fatalf("expected %s hash to be %t, got %t", x.uri, x.expectHash, actual.Hash())
}
if actual.DeprecatedRaw() != x.expectDeprecatedRaw {
t.Fatalf("expected %s deprecated raw to be %t, got %t", x.uri, x.expectDeprecatedRaw, actual.DeprecatedRaw())
}
if actual.DeprecatedImmutable() != x.expectDeprecatedImmutable {
t.Fatalf("expected %s deprecated immutable to be %t, got %t", x.uri, x.expectDeprecatedImmutable, actual.DeprecatedImmutable())
}
}
}

2
swarm/dev/.dockerignore
View file

@ -1,2 +0,0 @@
bin/*
cluster/*

2
swarm/dev/.gitignore vendored
View file

@ -1,2 +0,0 @@
bin/*
cluster/*

42
swarm/dev/Dockerfile
View file

@ -1,42 +0,0 @@
FROM ubuntu:xenial
# install build + test dependencies
RUN apt-get update && \
apt-get install --yes --no-install-recommends \
ca-certificates \
curl \
fuse \
g++ \
gcc \
git \
iproute2 \
iputils-ping \
less \
libc6-dev \
make \
pkg-config \
&& \
apt-get clean
# install Go
ENV GO_VERSION 1.8.1
RUN curl -fSLo golang.tar.gz "https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz" && \
tar -xzf golang.tar.gz -C /usr/local && \
rm golang.tar.gz
ENV GOPATH /go
ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
# install docker CLI
RUN curl -fSLo docker.tar.gz https://get.docker.com/builds/Linux/x86_64/docker-17.04.0-ce.tgz && \
tar -xzf docker.tar.gz -C /usr/local/bin --strip-components=1 docker/docker && \
rm docker.tar.gz
# install jq
RUN curl -fSLo /usr/local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 && \
chmod +x /usr/local/bin/jq
# install govendor
RUN go get -u github.com/kardianos/govendor
# add custom bashrc
ADD bashrc /root/.bashrc

14
swarm/dev/Makefile
View file

@ -1,14 +0,0 @@
.PHONY: build cluster test
default: build
build:
go build -o bin/swarm github.com/XinFinOrg/XDPoSChain/cmd/swarm
go build -o bin/XDC github.com/XinFinOrg/XDPoSChain/cmd/XDC
go build -o bin/bootnode github.com/XinFinOrg/XDPoSChain/cmd/bootnode
cluster: build
scripts/boot-cluster.sh
test:
go test -v github.com/XinFinOrg/XDPoSChain/swarm/...

20
swarm/dev/README.md
View file

@ -1,20 +0,0 @@
Swarm development environment
=============================
The Swarm development environment is a Linux bash shell which can be run in a
Docker container and provides a predictable build and test environment.
### Start the Docker container
Run the `run.sh` script to build the Docker image and run it, you will then be
at a bash prompt inside the `swarm/dev` directory.
### Build binaries
Run `make` to build the `swarm`, `geth` and `bootnode` binaries into the
`swarm/dev/bin` directory.
### Boot a cluster
Run `make cluster` to start a 3 node Swarm cluster, or run
`scripts/boot-cluster.sh --size N` to boot a cluster of size N.

21
swarm/dev/bashrc
View file

@ -1,21 +0,0 @@
export ROOT="${GOPATH}/src/github.com/XinFinOrg/XDPoSChain"
export PATH="${ROOT}/swarm/dev/bin:${PATH}"
cd "${ROOT}/swarm/dev"
cat <<WELCOME
=============================================
Welcome to the swarm development environment.
- Run 'make' to build the swarm, geth and bootnode binaries
- Run 'make test' to run the swarm unit tests
- Run 'make cluster' to start a swarm cluster
- Run 'exit' to exit the development environment
See the 'scripts' directory for some useful scripts.
=============================================
WELCOME

90
swarm/dev/run.sh
View file

@ -1,90 +0,0 @@
#!/usr/bin/env bash
#
# A script to build and run the Swarm development environment using Docker.
set -e
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
# DEFAULT_NAME is the default name for the Docker image and container
DEFAULT_NAME="swarm-dev"
usage() {
cat >&2 <<USAGE
usage: $0 [options]
Build and run the Swarm development environment.
Depends on Docker being installed locally.
OPTIONS:
-n, --name NAME Docker image and container name [default: ${DEFAULT_NAME}]
-d, --docker-args ARGS Custom args to pass to 'docker run' (e.g. '-p 8000:8000' to expose a port)
-h, --help Show this message
USAGE
}
main() {
local name="${DEFAULT_NAME}"
local docker_args=""
parse_args "$@"
build_image
run_image
}
parse_args() {
while true; do
case "$1" in
-h | --help)
usage
exit 0
;;
-n | --name)
if [[ -z "$2" ]]; then
echo "ERROR: --name flag requires an argument" >&2
exit 1
fi
name="$2"
shift 2
;;
-d | --docker-args)
if [[ -z "$2" ]]; then
echo "ERROR: --docker-args flag requires an argument" >&2
exit 1
fi
docker_args="$2"
shift 2
;;
*)
break
;;
esac
done
if [[ $# -ne 0 ]]; then
usage
echo "ERROR: invalid arguments" >&2
exit 1
fi
}
build_image() {
docker build --tag "${name}" "${ROOT}/swarm/dev"
}
run_image() {
exec docker run \
--privileged \
--interactive \
--tty \
--rm \
--hostname "${name}" \
--name "${name}" \
--volume "${ROOT}:/go/src/github.com/XinFinOrg/XDPoSChain" \
--volume "/var/run/docker.sock:/var/run/docker.sock" \
${docker_args} \
"${name}" \
/bin/bash
}
main "$@"

288
swarm/dev/scripts/boot-cluster.sh
View file

@ -1,288 +0,0 @@
#!/bin/bash
#
# A script to boot a dev swarm cluster on a Linux host (typically in a Docker
# container started with swarm/dev/run.sh).
#
# The cluster contains a bootnode, a geth node and multiple swarm nodes, with
# each node having its own data directory in a base directory passed with the
# --dir flag (default is swarm/dev/cluster).
#
# To avoid using different ports for each node and to make networking more
# realistic, each node gets its own network namespace with IPs assigned from
# the 192.168.33.0/24 subnet:
#
# bootnode: 192.168.33.2
# geth: 192.168.33.3
# swarm: 192.168.33.10{1,2,...,n}
set -e
ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
source "${ROOT}/swarm/dev/scripts/util.sh"
# DEFAULT_BASE_DIR is the default base directory to store node data
DEFAULT_BASE_DIR="${ROOT}/swarm/dev/cluster"
# DEFAULT_CLUSTER_SIZE is the default swarm cluster size
DEFAULT_CLUSTER_SIZE=3
# Linux bridge configuration for connecting the node network namespaces
BRIDGE_NAME="swarmbr0"
BRIDGE_IP="192.168.33.1"
# static bootnode configuration
BOOTNODE_IP="192.168.33.2"
BOOTNODE_PORT="30301"
BOOTNODE_KEY="32078f313bea771848db70745225c52c00981589ad6b5b49163f0f5ee852617d"
BOOTNODE_PUBKEY="760c4460e5336ac9bbd87952a3c7ec4363fc0a97bd31c86430806e287b437fd1b01abc6e1db640cf3106b520344af1d58b00b57823db3e1407cbc433e1b6d04d"
BOOTNODE_URL="enode://${BOOTNODE_PUBKEY}@${BOOTNODE_IP}:${BOOTNODE_PORT}"
# static geth configuration
GETH_IP="192.168.33.3"
GETH_RPC_PORT="8545"
GETH_RPC_URL="http://${GETH_IP}:${GETH_RPC_PORT}"
usage() {
cat >&2 <<USAGE
usage: $0 [options]
Boot a dev swarm cluster.
OPTIONS:
-d, --dir DIR Base directory to store node data [default: ${DEFAULT_BASE_DIR}]
-s, --size SIZE Size of swarm cluster [default: ${DEFAULT_CLUSTER_SIZE}]
-h, --help Show this message
USAGE
}
main() {
local base_dir="${DEFAULT_BASE_DIR}"
local cluster_size="${DEFAULT_CLUSTER_SIZE}"
parse_args "$@"
local pid_dir="${base_dir}/pids"
local log_dir="${base_dir}/logs"
mkdir -p "${base_dir}" "${pid_dir}" "${log_dir}"
stop_cluster
create_network
start_bootnode
start_geth_node
start_swarm_nodes
}
parse_args() {
while true; do
case "$1" in
-h | --help)
usage
exit 0
;;
-d | --dir)
if [[ -z "$2" ]]; then
fail "--dir flag requires an argument"
fi
base_dir="$2"
shift 2
;;
-s | --size)
if [[ -z "$2" ]]; then
fail "--size flag requires an argument"
fi
cluster_size="$2"
shift 2
;;
*)
break
;;
esac
done
if [[ $# -ne 0 ]]; then
usage
fail "ERROR: invalid arguments: $@"
fi
}
stop_cluster() {
info "stopping existing cluster"
"${ROOT}/swarm/dev/scripts/stop-cluster.sh" --dir "${base_dir}"
}
# create_network creates a Linux bridge which is used to connect the node
# network namespaces together
create_network() {
local subnet="${BRIDGE_IP}/24"
info "creating ${subnet} network on ${BRIDGE_NAME}"
ip link add name "${BRIDGE_NAME}" type bridge
ip link set dev "${BRIDGE_NAME}" up
ip address add "${subnet}" dev "${BRIDGE_NAME}"
}
# start_bootnode starts a bootnode which is used to bootstrap the geth and
# swarm nodes
start_bootnode() {
local key_file="${base_dir}/bootnode.key"
echo -n "${BOOTNODE_KEY}" > "${key_file}"
local args=(
--addr "${BOOTNODE_IP}:${BOOTNODE_PORT}"
--nodekey "${key_file}"
--verbosity "6"
)
start_node "bootnode" "${BOOTNODE_IP}" "$(which bootnode)" ${args[@]}
}
# start_geth_node starts a geth node with --datadir pointing at <base-dir>/geth
# and a single, unlocked account with password "geth"
start_geth_node() {
local dir="${base_dir}/geth"
mkdir -p "${dir}"
local password="geth"
echo "${password}" > "${dir}/password"
# create an account if necessary
if [[ ! -e "${dir}/keystore" ]]; then
info "creating geth account"
create_account "${dir}" "${password}"
fi
# get the account address
local address="$(jq --raw-output '.address' ${dir}/keystore/*)"
if [[ -z "${address}" ]]; then
fail "failed to get geth account address"
fi
local args=(
--datadir "${dir}"
--networkid "321"
--bootnodes "${BOOTNODE_URL}"
--unlock "${address}"
--password "${dir}/password"
--rpc
--rpcaddr "${GETH_IP}"
--rpcport "${GETH_RPC_PORT}"
--verbosity "6"
)
start_node "geth" "${GETH_IP}" "$(which geth)" ${args[@]}
}
start_swarm_nodes() {
for i in $(seq 1 ${cluster_size}); do
start_swarm_node "${i}"
done
}
# start_swarm_node starts a swarm node with a name like "swarmNN" (where NN is
# a zero-padded integer like "07"), --datadir pointing at <base-dir>/<name>
# (e.g. <base-dir>/swarm07) and a single account with <name> as the password
start_swarm_node() {
local num=$1
local name="swarm$(printf '%02d' ${num})"
local ip="192.168.33.1$(printf '%02d' ${num})"
local dir="${base_dir}/${name}"
mkdir -p "${dir}"
local password="${name}"
echo "${password}" > "${dir}/password"
# create an account if necessary
if [[ ! -e "${dir}/keystore" ]]; then
info "creating account for ${name}"
create_account "${dir}" "${password}"
fi
# get the account address
local address="$(jq --raw-output '.address' ${dir}/keystore/*)"
if [[ -z "${address}" ]]; then
fail "failed to get swarm account address"
fi
local args=(
--bootnodes "${BOOTNODE_URL}"
--datadir "${dir}"
--identity "${name}"
--ens-api "${GETH_RPC_URL}"
--bzznetworkid "321"
--bzzaccount "${address}"
--password "${dir}/password"
--verbosity "6"
)
start_node "${name}" "${ip}" "$(which swarm)" ${args[@]}
}
# start_node runs the node command as a daemon in a network namespace
start_node() {
local name="$1"
local ip="$2"
local path="$3"
local cmd_args=${@:4}
info "starting ${name} with IP ${ip}"
create_node_network "${name}" "${ip}"
# add a marker to the log file
cat >> "${log_dir}/${name}.log" <<EOF
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
Starting ${name} node - $(date)
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
EOF
# run the command in the network namespace using start-stop-daemon to
# daemonise the process, sending all output to the log file
local daemon_args=(
--start
--background
--no-close
--make-pidfile
--pidfile "${pid_dir}/${name}.pid"
--exec "${path}"
)
if ! ip netns exec "${name}" start-stop-daemon ${daemon_args[@]} -- $cmd_args &>> "${log_dir}/${name}.log"; then
fail "could not start ${name}, check ${log_dir}/${name}.log"
fi
}
# create_node_network creates a network namespace and connects it to the Linux
# bridge using a veth pair
create_node_network() {
local name="$1"
local ip="$2"
# create the namespace
ip netns add "${name}"
# create the veth pair
local veth0="veth${name}0"
local veth1="veth${name}1"
ip link add name "${veth0}" type veth peer name "${veth1}"
# add one end to the bridge
ip link set dev "${veth0}" master "${BRIDGE_NAME}"
ip link set dev "${veth0}" up
# add the other end to the namespace, rename it eth0 and give it the ip
ip link set dev "${veth1}" netns "${name}"
ip netns exec "${name}" ip link set dev "${veth1}" name "eth0"
ip netns exec "${name}" ip link set dev "eth0" up
ip netns exec "${name}" ip address add "${ip}/24" dev "eth0"
}
create_account() {
local dir=$1
local password=$2
geth --datadir "${dir}" --password /dev/stdin account new <<< "${password}"
}
main "$@"

96
swarm/dev/scripts/random-uploads.sh
View file

@ -1,96 +0,0 @@
#!/bin/bash
#
# A script to upload random data to a swarm cluster.
#
# Example:
#
# random-uploads.sh --addr 192.168.33.101:8500 --size 40k --count 1000
set -e
ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
source "${ROOT}/swarm/dev/scripts/util.sh"
DEFAULT_ADDR="localhost:8500"
DEFAULT_UPLOAD_SIZE="40k"
DEFAULT_UPLOAD_COUNT="1000"
usage() {
cat >&2 <<USAGE
usage: $0 [options]
Upload random data to a Swarm cluster.
OPTIONS:
-a, --addr ADDR Swarm API address [default: ${DEFAULT_ADDR}]
-s, --size SIZE Individual upload size [default: ${DEFAULT_UPLOAD_SIZE}]
-c, --count COUNT Number of uploads [default: ${DEFAULT_UPLOAD_COUNT}]
-h, --help Show this message
USAGE
}
main() {
local addr="${DEFAULT_ADDR}"
local upload_size="${DEFAULT_UPLOAD_SIZE}"
local upload_count="${DEFAULT_UPLOAD_COUNT}"
parse_args "$@"
info "uploading ${upload_count} ${upload_size} random files to ${addr}"
for i in $(seq 1 ${upload_count}); do
info "upload ${i} / ${upload_count}:"
do_random_upload
echo
done
}
do_random_upload() {
curl -fsSL -X POST --data-binary "$(random_data)" "http://${addr}/bzz-raw:/"
}
random_data() {
dd if=/dev/urandom of=/dev/stdout bs="${upload_size}" count=1 2>/dev/null
}
parse_args() {
while true; do
case "$1" in
-h | --help)
usage
exit 0
;;
-a | --addr)
if [[ -z "$2" ]]; then
fail "--addr flag requires an argument"
fi
addr="$2"
shift 2
;;
-s | --size)
if [[ -z "$2" ]]; then
fail "--size flag requires an argument"
fi
upload_size="$2"
shift 2
;;
-c | --count)
if [[ -z "$2" ]]; then
fail "--count flag requires an argument"
fi
upload_count="$2"
shift 2
;;
*)
break
;;
esac
done
if [[ $# -ne 0 ]]; then
usage
fail "ERROR: invalid arguments: $@"
fi
}
main "$@"

98
swarm/dev/scripts/stop-cluster.sh
View file

@ -1,98 +0,0 @@
#!/bin/bash
#
# A script to shutdown a dev swarm cluster.
set -e
ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
source "${ROOT}/swarm/dev/scripts/util.sh"
DEFAULT_BASE_DIR="${ROOT}/swarm/dev/cluster"
usage() {
cat >&2 <<USAGE
usage: $0 [options]
Shutdown a dev swarm cluster.
OPTIONS:
-d, --dir DIR Base directory [default: ${DEFAULT_BASE_DIR}]
-h, --help Show this message
USAGE
}
main() {
local base_dir="${DEFAULT_BASE_DIR}"
parse_args "$@"
local pid_dir="${base_dir}/pids"
stop_swarm_nodes
stop_node "geth"
stop_node "bootnode"
delete_network
}
parse_args() {
while true; do
case "$1" in
-h | --help)
usage
exit 0
;;
-d | --dir)
if [[ -z "$2" ]]; then
fail "--dir flag requires an argument"
fi
base_dir="$2"
shift 2
;;
*)
break
;;
esac
done
if [[ $# -ne 0 ]]; then
usage
fail "ERROR: invalid arguments: $@"
fi
}
stop_swarm_nodes() {
for name in $(ls "${pid_dir}" | grep -oP 'swarm\d+'); do
stop_node "${name}"
done
}
stop_node() {
local name=$1
local pid_file="${pid_dir}/${name}.pid"
if [[ -e "${pid_file}" ]]; then
info "stopping ${name}"
start-stop-daemon \
--stop \
--pidfile "${pid_file}" \
--remove-pidfile \
--oknodo \
--retry 15
fi
if ip netns list | grep -qF "${name}"; then
ip netns delete "${name}"
fi
if ip link show "veth${name}0" &>/dev/null; then
ip link delete dev "veth${name}0"
fi
}
delete_network() {
if ip link show "swarmbr0" &>/dev/null; then
ip link delete dev "swarmbr0"
fi
}
main "$@"

53
swarm/dev/scripts/util.sh
View file

@ -1,53 +0,0 @@
# shared shell functions
info() {
local msg="$@"
local timestamp="$(date +%H:%M:%S)"
say "===> ${timestamp} ${msg}" "green"
}
warn() {
local msg="$@"
local timestamp=$(date +%H:%M:%S)
say "===> ${timestamp} WARN: ${msg}" "yellow" >&2
}
fail() {
local msg="$@"
say "ERROR: ${msg}" "red" >&2
exit 1
}
# say prints the given message to STDOUT, using the optional color if
# STDOUT is a terminal.
#
# usage:
#
# say "foo" - prints "foo"
# say "bar" "red" - prints "bar" in red
# say "baz" "green" - prints "baz" in green
# say "qux" "red" | tee - prints "qux" with no colour
#
say() {
local msg=$1
local color=$2
if [[ -n "${color}" ]] && [[ -t 1 ]]; then
case "${color}" in
red)
echo -e "\033[1;31m${msg}\033[0m"
;;
green)
echo -e "\033[1;32m${msg}\033[0m"
;;
yellow)
echo -e "\033[1;33m${msg}\033[0m"
;;
*)
echo "${msg}"
;;
esac
else
echo "${msg}"
fi
}

155
swarm/fuse/fuse_dir.go
View file

@ -1,155 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// +build linux darwin freebsd
package fuse
import (
"bazil.org/fuse"
"bazil.org/fuse/fs"
"golang.org/x/net/context"
"os"
"path/filepath"
"sync"
)
var (
_ fs.Node = (*SwarmDir)(nil)
_ fs.NodeRequestLookuper = (*SwarmDir)(nil)
_ fs.HandleReadDirAller = (*SwarmDir)(nil)
_ fs.NodeCreater = (*SwarmDir)(nil)
_ fs.NodeRemover = (*SwarmDir)(nil)
_ fs.NodeMkdirer = (*SwarmDir)(nil)
)
type SwarmDir struct {
inode uint64
name string
path string
directories []*SwarmDir
files []*SwarmFile
mountInfo *MountInfo
lock *sync.RWMutex
}
func NewSwarmDir(fullpath string, minfo *MountInfo) *SwarmDir {
newdir := &SwarmDir{
inode: NewInode(),
name: filepath.Base(fullpath),
path: fullpath,
directories: []*SwarmDir{},
files: []*SwarmFile{},
mountInfo: minfo,
lock: &sync.RWMutex{},
}
return newdir
}
func (sd *SwarmDir) Attr(ctx context.Context, a *fuse.Attr) error {
a.Inode = sd.inode
a.Mode = os.ModeDir | 0700
a.Uid = uint32(os.Getuid())
a.Gid = uint32(os.Getegid())
return nil
}
func (sd *SwarmDir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (fs.Node, error) {
for _, n := range sd.files {
if n.name == req.Name {
return n, nil
}
}
for _, n := range sd.directories {
if n.name == req.Name {
return n, nil
}
}
return nil, fuse.ENOENT
}
func (sd *SwarmDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
var children []fuse.Dirent
for _, file := range sd.files {
children = append(children, fuse.Dirent{Inode: file.inode, Type: fuse.DT_File, Name: file.name})
}
for _, dir := range sd.directories {
children = append(children, fuse.Dirent{Inode: dir.inode, Type: fuse.DT_Dir, Name: dir.name})
}
return children, nil
}
func (sd *SwarmDir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) {
newFile := NewSwarmFile(sd.path, req.Name, sd.mountInfo)
newFile.fileSize = 0 // 0 means, file is not in swarm yet and it is just created
sd.lock.Lock()
defer sd.lock.Unlock()
sd.files = append(sd.files, newFile)
return newFile, newFile, nil
}
func (sd *SwarmDir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
if req.Dir && sd.directories != nil {
newDirs := []*SwarmDir{}
for _, dir := range sd.directories {
if dir.name == req.Name {
removeDirectoryFromSwarm(dir)
} else {
newDirs = append(newDirs, dir)
}
}
if len(sd.directories) > len(newDirs) {
sd.lock.Lock()
defer sd.lock.Unlock()
sd.directories = newDirs
}
return nil
} else if !req.Dir && sd.files != nil {
newFiles := []*SwarmFile{}
for _, f := range sd.files {
if f.name == req.Name {
removeFileFromSwarm(f)
} else {
newFiles = append(newFiles, f)
}
}
if len(sd.files) > len(newFiles) {
sd.lock.Lock()
defer sd.lock.Unlock()
sd.files = newFiles
}
return nil
}
return fuse.ENOENT
}
func (sd *SwarmDir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) {
newDir := NewSwarmDir(req.Name, sd.mountInfo)
sd.lock.Lock()
defer sd.lock.Unlock()
sd.directories = append(sd.directories, newDir)
return newDir, nil
}

145
swarm/fuse/fuse_file.go
View file

@ -1,145 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// +build linux darwin freebsd
package fuse
import (
"errors"
"io"
"os"
"sync"
"bazil.org/fuse"
"bazil.org/fuse/fs"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/swarm/storage"
"golang.org/x/net/context"
)
const (
MaxAppendFileSize = 10485760 // 10Mb
)
var (
errInvalidOffset = errors.New("Invalid offset during write")
errFileSizeMaxLimixReached = errors.New("File size exceeded max limit")
)
var (
_ fs.Node = (*SwarmFile)(nil)
_ fs.HandleReader = (*SwarmFile)(nil)
_ fs.HandleWriter = (*SwarmFile)(nil)
)
type SwarmFile struct {
inode uint64
name string
path string
key storage.Key
fileSize int64
reader storage.LazySectionReader
mountInfo *MountInfo
lock *sync.RWMutex
}
func NewSwarmFile(path, fname string, minfo *MountInfo) *SwarmFile {
newFile := &SwarmFile{
inode: NewInode(),
name: fname,
path: path,
key: nil,
fileSize: -1, // -1 means , file already exists in swarm and you need to just get the size from swarm
reader: nil,
mountInfo: minfo,
lock: &sync.RWMutex{},
}
return newFile
}
func (file *SwarmFile) Attr(ctx context.Context, a *fuse.Attr) error {
a.Inode = file.inode
//TODO: need to get permission as argument
a.Mode = 0700
a.Uid = uint32(os.Getuid())
a.Gid = uint32(os.Getegid())
if file.fileSize == -1 {
reader := file.mountInfo.swarmApi.Retrieve(file.key)
quitC := make(chan bool)
size, err := reader.Size(quitC)
if err != nil {
log.Warn("Couldnt get size of file %s : %v", file.path, err)
}
file.fileSize = size
}
a.Size = uint64(file.fileSize)
return nil
}
func (sf *SwarmFile) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {
sf.lock.RLock()
defer sf.lock.RUnlock()
if sf.reader == nil {
sf.reader = sf.mountInfo.swarmApi.Retrieve(sf.key)
}
buf := make([]byte, req.Size)
n, err := sf.reader.ReadAt(buf, req.Offset)
if err == io.ErrUnexpectedEOF || err == io.EOF {
err = nil
}
resp.Data = buf[:n]
sf.reader = nil
return err
}
func (sf *SwarmFile) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error {
if sf.fileSize == 0 && req.Offset == 0 {
// A new file is created
err := addFileToSwarm(sf, req.Data, len(req.Data))
if err != nil {
return err
}
resp.Size = len(req.Data)
} else if req.Offset <= sf.fileSize {
totalSize := sf.fileSize + int64(len(req.Data))
if totalSize > MaxAppendFileSize {
log.Warn("Append file size reached (%v) : (%v)", sf.fileSize, len(req.Data))
return errFileSizeMaxLimixReached
}
err := appendToExistingFileInSwarm(sf, req.Data, req.Offset, int64(len(req.Data)))
if err != nil {
return err
}
resp.Size = len(req.Data)
} else {
log.Warn("Invalid write request size(%v) : off(%v)", sf.fileSize, req.Offset)
return errInvalidOffset
}
return nil
}

35
swarm/fuse/fuse_root.go
View file

@ -1,35 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// +build linux darwin freebsd
package fuse
import (
"bazil.org/fuse/fs"
)
var (
_ fs.Node = (*SwarmDir)(nil)
)
type SwarmRoot struct {
root *SwarmDir
}
func (filesystem *SwarmRoot) Root() (fs.Node, error) {
return filesystem.root, nil
}

64
swarm/fuse/swarmfs.go
View file

@ -1,64 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package fuse
import (
"github.com/XinFinOrg/XDPoSChain/swarm/api"
"sync"
"time"
)
const (
Swarmfs_Version = "0.1"
mountTimeout = time.Second * 5
unmountTimeout = time.Second * 10
maxFuseMounts = 5
)
var (
swarmfs *SwarmFS // Swarm file system singleton
swarmfsLock sync.Once
inode uint64 = 1 // global inode
inodeLock sync.RWMutex
)
type SwarmFS struct {
swarmApi *api.Api
activeMounts map[string]*MountInfo
swarmFsLock *sync.RWMutex
}
func NewSwarmFS(api *api.Api) *SwarmFS {
swarmfsLock.Do(func() {
swarmfs = &SwarmFS{
swarmApi: api,
swarmFsLock: &sync.RWMutex{},
activeMounts: map[string]*MountInfo{},
}
})
return swarmfs
}
// Inode numbers need to be unique, they are used for caching inside fuse
func NewInode() uint64 {
inodeLock.Lock()
defer inodeLock.Unlock()
inode += 1
return inode
}

51
swarm/fuse/swarmfs_fallback.go
View file

@ -1,51 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// +build !linux,!darwin,!freebsd
package fuse
import (
"errors"
)
var errNoFUSE = errors.New("FUSE is not supported on this platform")
func isFUSEUnsupportedError(err error) bool {
return err == errNoFUSE
}
type MountInfo struct {
MountPoint string
StartManifest string
LatestManifest string
}
func (self *SwarmFS) Mount(mhash, mountpoint string) (*MountInfo, error) {
return nil, errNoFUSE
}
func (self *SwarmFS) Unmount(mountpoint string) (bool, error) {
return false, errNoFUSE
}
func (self *SwarmFS) Listmounts() ([]*MountInfo, error) {
return nil, errNoFUSE
}
func (self *SwarmFS) Stop() error {
return nil
}

836
swarm/fuse/swarmfs_test.go
View file

@ -1,836 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
//go:build linux || darwin || freebsd
// +build linux darwin freebsd
package fuse
import (
"bytes"
"crypto/rand"
"io"
"os"
"path/filepath"
"testing"
"github.com/XinFinOrg/XDPoSChain/swarm/api"
"github.com/XinFinOrg/XDPoSChain/swarm/storage"
)
type fileInfo struct {
perm uint64
uid int
gid int
contents []byte
}
func createTestFilesAndUploadToSwarm(t *testing.T, api *api.Api, files map[string]fileInfo, uploadDir string) string {
os.RemoveAll(uploadDir)
for fname, finfo := range files {
actualPath := filepath.Join(uploadDir, fname)
filePath := filepath.Dir(actualPath)
err := os.MkdirAll(filePath, 0777)
if err != nil {
t.Fatalf("Error creating directory '%v' : %v", filePath, err)
}
fd, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(finfo.perm))
if err1 != nil {
t.Fatalf("Error creating file %v: %v", actualPath, err1)
}
fd.Write(finfo.contents)
fd.Chown(finfo.uid, finfo.gid)
fd.Chmod(os.FileMode(finfo.perm))
fd.Sync()
fd.Close()
}
bzzhash, err := api.Upload(uploadDir, "")
if err != nil {
t.Fatalf("Error uploading directory %v: %v", uploadDir, err)
}
return bzzhash
}
func mountDir(t *testing.T, api *api.Api, files map[string]fileInfo, bzzHash string, mountDir string) *SwarmFS {
os.RemoveAll(mountDir)
os.MkdirAll(mountDir, 0777)
swarmfs := NewSwarmFS(api)
_, err := swarmfs.Mount(bzzHash, mountDir)
if isFUSEUnsupportedError(err) {
t.Skip("FUSE not supported:", err)
} else if err != nil {
t.Fatalf("Error mounting hash %v: %v", bzzHash, err)
}
found := false
mi := swarmfs.Listmounts()
for _, minfo := range mi {
if minfo.MountPoint == mountDir {
if minfo.StartManifest != bzzHash ||
minfo.LatestManifest != bzzHash ||
minfo.fuseConnection == nil {
t.Fatalf("Error mounting: exp(%s): act(%s)", bzzHash, minfo.StartManifest)
}
found = true
}
}
// Test listMounts
if !found {
t.Fatalf("Error getting mounts information for %v: %v", mountDir, err)
}
// Check if file and their attributes are as expected
compareGeneratedFileWithFileInMount(t, files, mountDir)
return swarmfs
}
func compareGeneratedFileWithFileInMount(t *testing.T, files map[string]fileInfo, mountDir string) {
err := filepath.Walk(mountDir, func(path string, f os.FileInfo, err error) error {
if f.IsDir() {
return nil
}
fname := path[len(mountDir)+1:]
if _, ok := files[fname]; !ok {
t.Fatalf(" file %v present in mount dir and is not expected", fname)
}
return nil
})
if err != nil {
t.Fatalf("Error walking dir %v", mountDir)
}
for fname, finfo := range files {
destinationFile := filepath.Join(mountDir, fname)
dfinfo, err := os.Stat(destinationFile)
if err != nil {
t.Fatalf("Destination file %v missing in mount: %v", fname, err)
}
if int64(len(finfo.contents)) != dfinfo.Size() {
t.Fatalf("file %v Size mismatch source (%v) vs destination(%v)", fname, int64(len(finfo.contents)), dfinfo.Size())
}
if dfinfo.Mode().Perm().String() != "-rwx------" {
t.Fatalf("file %v Permission mismatch source (-rwx------) vs destination(%v)", fname, dfinfo.Mode().Perm())
}
fileContents, err := os.ReadFile(filepath.Join(mountDir, fname))
if err != nil {
t.Fatalf("Could not readfile %v : %v", fname, err)
}
if !bytes.Equal(fileContents, finfo.contents) {
t.Fatalf("File %v contents mismatch: %v , %v", fname, fileContents, finfo.contents)
}
// TODO: check uid and gid
}
}
func checkFile(t *testing.T, testMountDir, fname string, contents []byte) {
destinationFile := filepath.Join(testMountDir, fname)
dfinfo, err1 := os.Stat(destinationFile)
if err1 != nil {
t.Fatalf("Could not stat file %v", destinationFile)
}
if dfinfo.Size() != int64(len(contents)) {
t.Fatalf("Mismatch in size actual(%v) vs expected(%v)", dfinfo.Size(), int64(len(contents)))
}
fd, err2 := os.OpenFile(destinationFile, os.O_RDONLY, os.FileMode(0665))
if err2 != nil {
t.Fatalf("Could not open file %v", destinationFile)
}
newcontent := make([]byte, len(contents))
fd.Read(newcontent)
fd.Close()
if !bytes.Equal(contents, newcontent) {
t.Fatalf("File content mismatch expected (%v): received (%v) ", contents, newcontent)
}
}
func getRandomBtes(size int) []byte {
contents := make([]byte, size)
rand.Read(contents)
return contents
}
func isDirEmpty(name string) bool {
f, err := os.Open(name)
if err != nil {
return false
}
defer f.Close()
_, err = f.Readdirnames(1)
return err == io.EOF
}
type testAPI struct {
api *api.Api
}
func (ta *testAPI) mountListAndUnmount(t *testing.T) {
files := make(map[string]fileInfo)
testUploadDir, _ := os.MkdirTemp(os.TempDir(), "fuse-source")
testMountDir, _ := os.MkdirTemp(os.TempDir(), "fuse-dest")
files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
files["2.txt"] = fileInfo{0711, 333, 444, getRandomBtes(10)}
files["3.txt"] = fileInfo{0622, 333, 444, getRandomBtes(100)}
files["4.txt"] = fileInfo{0533, 333, 444, getRandomBtes(1024)}
files["5.txt"] = fileInfo{0544, 333, 444, getRandomBtes(10)}
files["6.txt"] = fileInfo{0555, 333, 444, getRandomBtes(10)}
files["7.txt"] = fileInfo{0666, 333, 444, getRandomBtes(10)}
files["8.txt"] = fileInfo{0777, 333, 333, getRandomBtes(10)}
files["11.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10)}
files["111.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10)}
files["two/2.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10)}
files["two/2/2.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10)}
files["two/2./2.txt"] = fileInfo{0777, 444, 444, getRandomBtes(10)}
files["twice/2.txt"] = fileInfo{0777, 444, 333, getRandomBtes(200)}
files["one/two/three/four/five/six/seven/eight/nine/10.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10240)}
files["one/two/three/four/five/six/six"] = fileInfo{0777, 333, 444, getRandomBtes(10)}
bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
swarmfs := mountDir(t, ta.api, files, bzzHash, testMountDir)
defer swarmfs.Stop()
// Check unmount
_, err := swarmfs.Unmount(testMountDir)
if err != nil {
t.Fatalf("could not unmount %v", bzzHash)
}
if !isDirEmpty(testMountDir) {
t.Fatalf("unmount didnt work for %v", testMountDir)
}
}
func (ta *testAPI) maxMounts(t *testing.T) {
files := make(map[string]fileInfo)
files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
uploadDir1, _ := os.MkdirTemp(os.TempDir(), "max-upload1")
bzzHash1 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir1)
mount1, _ := os.MkdirTemp(os.TempDir(), "max-mount1")
swarmfs1 := mountDir(t, ta.api, files, bzzHash1, mount1)
defer swarmfs1.Stop()
files["2.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
uploadDir2, _ := os.MkdirTemp(os.TempDir(), "max-upload2")
bzzHash2 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir2)
mount2, _ := os.MkdirTemp(os.TempDir(), "max-mount2")
swarmfs2 := mountDir(t, ta.api, files, bzzHash2, mount2)
defer swarmfs2.Stop()
files["3.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
uploadDir3, _ := os.MkdirTemp(os.TempDir(), "max-upload3")
bzzHash3 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir3)
mount3, _ := os.MkdirTemp(os.TempDir(), "max-mount3")
swarmfs3 := mountDir(t, ta.api, files, bzzHash3, mount3)
defer swarmfs3.Stop()
files["4.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
uploadDir4, _ := os.MkdirTemp(os.TempDir(), "max-upload4")
bzzHash4 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir4)
mount4, _ := os.MkdirTemp(os.TempDir(), "max-mount4")
swarmfs4 := mountDir(t, ta.api, files, bzzHash4, mount4)
defer swarmfs4.Stop()
files["5.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
uploadDir5, _ := os.MkdirTemp(os.TempDir(), "max-upload5")
bzzHash5 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir5)
mount5, _ := os.MkdirTemp(os.TempDir(), "max-mount5")
swarmfs5 := mountDir(t, ta.api, files, bzzHash5, mount5)
defer swarmfs5.Stop()
files["6.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
uploadDir6, _ := os.MkdirTemp(os.TempDir(), "max-upload6")
bzzHash6 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir6)
mount6, _ := os.MkdirTemp(os.TempDir(), "max-mount6")
os.RemoveAll(mount6)
os.MkdirAll(mount6, 0777)
_, err := swarmfs.Mount(bzzHash6, mount6)
if err == nil {
t.Fatalf("Error: Going beyond max mounts %v", bzzHash6)
}
}
func (ta *testAPI) remount(t *testing.T) {
files := make(map[string]fileInfo)
files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
uploadDir1, _ := os.MkdirTemp(os.TempDir(), "re-upload1")
bzzHash1 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir1)
testMountDir1, _ := os.MkdirTemp(os.TempDir(), "re-mount1")
swarmfs := mountDir(t, ta.api, files, bzzHash1, testMountDir1)
defer swarmfs.Stop()
uploadDir2, _ := os.MkdirTemp(os.TempDir(), "re-upload2")
bzzHash2 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir2)
testMountDir2, _ := os.MkdirTemp(os.TempDir(), "re-mount2")
// try mounting the same hash second time
os.RemoveAll(testMountDir2)
os.MkdirAll(testMountDir2, 0777)
_, err := swarmfs.Mount(bzzHash1, testMountDir2)
if err != nil {
t.Fatalf("Error mounting hash %v", bzzHash1)
}
// mount a different hash in already mounted point
_, err = swarmfs.Mount(bzzHash2, testMountDir1)
if err == nil {
t.Fatalf("Error mounting hash %v", bzzHash2)
}
// mount nonexistent hash
_, err = swarmfs.Mount("0xfea11223344", testMountDir1)
if err == nil {
t.Fatalf("Error mounting hash %v", bzzHash2)
}
}
func (ta *testAPI) unmount(t *testing.T) {
files := make(map[string]fileInfo)
uploadDir, _ := os.MkdirTemp(os.TempDir(), "ex-upload")
testMountDir, _ := os.MkdirTemp(os.TempDir(), "ex-mount")
files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir)
swarmfs := mountDir(t, ta.api, files, bzzHash, testMountDir)
defer swarmfs.Stop()
swarmfs.Unmount(testMountDir)
mi := swarmfs.Listmounts()
for _, minfo := range mi {
if minfo.MountPoint == testMountDir {
t.Fatalf("mount state not cleaned up in unmount case %v", testMountDir)
}
}
}
func (ta *testAPI) unmountWhenResourceBusy(t *testing.T) {
files := make(map[string]fileInfo)
testUploadDir, _ := os.MkdirTemp(os.TempDir(), "ex-upload")
testMountDir, _ := os.MkdirTemp(os.TempDir(), "ex-mount")
files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
swarmfs := mountDir(t, ta.api, files, bzzHash, testMountDir)
defer swarmfs.Stop()
actualPath := filepath.Join(testMountDir, "2.txt")
d, err := os.OpenFile(actualPath, os.O_RDWR, os.FileMode(0700))
d.Write(getRandomBtes(10))
_, err = swarmfs.Unmount(testMountDir)
if err != nil {
t.Fatalf("could not unmount %v", bzzHash)
}
d.Close()
mi := swarmfs.Listmounts()
for _, minfo := range mi {
if minfo.MountPoint == testMountDir {
t.Fatalf("mount state not cleaned up in unmount case %v", testMountDir)
}
}
}
func (ta *testAPI) seekInMultiChunkFile(t *testing.T) {
files := make(map[string]fileInfo)
testUploadDir, _ := os.MkdirTemp(os.TempDir(), "seek-upload")
testMountDir, _ := os.MkdirTemp(os.TempDir(), "seek-mount")
files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10240)}
bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
swarmfs := mountDir(t, ta.api, files, bzzHash, testMountDir)
defer swarmfs.Stop()
// Create a new file seek the second chunk
actualPath := filepath.Join(testMountDir, "1.txt")
d, _ := os.OpenFile(actualPath, os.O_RDONLY, os.FileMode(0700))
d.Seek(5000, 0)
contents := make([]byte, 1024)
d.Read(contents)
finfo := files["1.txt"]
if !bytes.Equal(finfo.contents[:6024][5000:], contents) {
t.Fatalf("File seek contents mismatch")
}
d.Close()
}
func (ta *testAPI) createNewFile(t *testing.T) {
files := make(map[string]fileInfo)
testUploadDir, _ := os.MkdirTemp(os.TempDir(), "create-upload")
testMountDir, _ := os.MkdirTemp(os.TempDir(), "create-mount")
files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir)
defer swarmfs1.Stop()
// Create a new file in the root dir and check
actualPath := filepath.Join(testMountDir, "2.txt")
d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665))
if err1 != nil {
t.Fatalf("Could not create file %s : %v", actualPath, err1)
}
contents := make([]byte, 11)
rand.Read(contents)
d.Write(contents)
d.Close()
mi, err2 := swarmfs1.Unmount(testMountDir)
if err2 != nil {
t.Fatalf("Could not unmount %v", err2)
}
// mount again and see if things are okay
files["2.txt"] = fileInfo{0700, 333, 444, contents}
swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir)
defer swarmfs2.Stop()
checkFile(t, testMountDir, "2.txt", contents)
}
func (ta *testAPI) createNewFileInsideDirectory(t *testing.T) {
files := make(map[string]fileInfo)
testUploadDir, _ := os.MkdirTemp(os.TempDir(), "createinsidedir-upload")
testMountDir, _ := os.MkdirTemp(os.TempDir(), "createinsidedir-mount")
files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir)
defer swarmfs1.Stop()
// Create a new file inside a existing dir and check
dirToCreate := filepath.Join(testMountDir, "one")
actualPath := filepath.Join(dirToCreate, "2.txt")
d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665))
if err1 != nil {
t.Fatalf("Could not create file %s : %v", actualPath, err1)
}
contents := make([]byte, 11)
rand.Read(contents)
d.Write(contents)
d.Close()
mi, err2 := swarmfs1.Unmount(testMountDir)
if err2 != nil {
t.Fatalf("Could not unmount %v", err2)
}
// mount again and see if things are okay
files["one/2.txt"] = fileInfo{0700, 333, 444, contents}
swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir)
defer swarmfs2.Stop()
checkFile(t, testMountDir, "one/2.txt", contents)
}
func (ta *testAPI) createNewFileInsideNewDirectory(t *testing.T) {
files := make(map[string]fileInfo)
testUploadDir, _ := os.MkdirTemp(os.TempDir(), "createinsidenewdir-upload")
testMountDir, _ := os.MkdirTemp(os.TempDir(), "createinsidenewdir-mount")
files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir)
defer swarmfs1.Stop()
// Create a new file inside a existing dir and check
dirToCreate := filepath.Join(testMountDir, "one")
os.MkdirAll(dirToCreate, 0777)
actualPath := filepath.Join(dirToCreate, "2.txt")
d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665))
if err1 != nil {
t.Fatalf("Could not create file %s : %v", actualPath, err1)
}
contents := make([]byte, 11)
rand.Read(contents)
d.Write(contents)
d.Close()
mi, err2 := swarmfs1.Unmount(testMountDir)
if err2 != nil {
t.Fatalf("Could not unmount %v", err2)
}
// mount again and see if things are okay
files["one/2.txt"] = fileInfo{0700, 333, 444, contents}
swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir)
defer swarmfs2.Stop()
checkFile(t, testMountDir, "one/2.txt", contents)
}
func (ta *testAPI) removeExistingFile(t *testing.T) {
files := make(map[string]fileInfo)
testUploadDir, _ := os.MkdirTemp(os.TempDir(), "remove-upload")
testMountDir, _ := os.MkdirTemp(os.TempDir(), "remove-mount")
files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir)
defer swarmfs1.Stop()
// Remove a file in the root dir and check
actualPath := filepath.Join(testMountDir, "five.txt")
os.Remove(actualPath)
mi, err2 := swarmfs1.Unmount(testMountDir)
if err2 != nil {
t.Fatalf("Could not unmount %v", err2)
}
// mount again and see if things are okay
delete(files, "five.txt")
swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir)
defer swarmfs2.Stop()
}
func (ta *testAPI) removeExistingFileInsideDir(t *testing.T) {
files := make(map[string]fileInfo)
testUploadDir, _ := os.MkdirTemp(os.TempDir(), "remove-upload")
testMountDir, _ := os.MkdirTemp(os.TempDir(), "remove-mount")
files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
files["one/five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
files["one/six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir)
defer swarmfs1.Stop()
// Remove a file in the root dir and check
actualPath := filepath.Join(testMountDir, "one/five.txt")
os.Remove(actualPath)
mi, err2 := swarmfs1.Unmount(testMountDir)
if err2 != nil {
t.Fatalf("Could not unmount %v", err2)
}
// mount again and see if things are okay
delete(files, "one/five.txt")
swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir)
defer swarmfs2.Stop()
}
func (ta *testAPI) removeNewlyAddedFile(t *testing.T) {
files := make(map[string]fileInfo)
testUploadDir, _ := os.MkdirTemp(os.TempDir(), "removenew-upload")
testMountDir, _ := os.MkdirTemp(os.TempDir(), "removenew-mount")
files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir)
defer swarmfs1.Stop()
// Adda a new file and remove it
dirToCreate := filepath.Join(testMountDir, "one")
os.MkdirAll(dirToCreate, os.FileMode(0665))
actualPath := filepath.Join(dirToCreate, "2.txt")
d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665))
if err1 != nil {
t.Fatalf("Could not create file %s : %v", actualPath, err1)
}
contents := make([]byte, 11)
rand.Read(contents)
d.Write(contents)
d.Close()
checkFile(t, testMountDir, "one/2.txt", contents)
os.Remove(actualPath)
mi, err2 := swarmfs1.Unmount(testMountDir)
if err2 != nil {
t.Fatalf("Could not unmount %v", err2)
}
// mount again and see if things are okay
swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir)
defer swarmfs2.Stop()
if bzzHash != mi.LatestManifest {
t.Fatalf("same contents different hash orig(%v): new(%v)", bzzHash, mi.LatestManifest)
}
}
func (ta *testAPI) addNewFileAndModifyContents(t *testing.T) {
files := make(map[string]fileInfo)
testUploadDir, _ := os.MkdirTemp(os.TempDir(), "modifyfile-upload")
testMountDir, _ := os.MkdirTemp(os.TempDir(), "modifyfile-mount")
files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir)
defer swarmfs1.Stop()
// Create a new file in the root dir and check
actualPath := filepath.Join(testMountDir, "2.txt")
d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665))
if err1 != nil {
t.Fatalf("Could not create file %s : %v", actualPath, err1)
}
line1 := []byte("Line 1")
rand.Read(line1)
d.Write(line1)
d.Close()
mi1, err2 := swarmfs1.Unmount(testMountDir)
if err2 != nil {
t.Fatalf("Could not unmount %v", err2)
}
// mount again and see if things are okay
files["2.txt"] = fileInfo{0700, 333, 444, line1}
swarmfs2 := mountDir(t, ta.api, files, mi1.LatestManifest, testMountDir)
defer swarmfs2.Stop()
checkFile(t, testMountDir, "2.txt", line1)
mi2, err3 := swarmfs2.Unmount(testMountDir)
if err3 != nil {
t.Fatalf("Could not unmount %v", err3)
}
// mount again and modify
swarmfs3 := mountDir(t, ta.api, files, mi2.LatestManifest, testMountDir)
defer swarmfs3.Stop()
fd, err4 := os.OpenFile(actualPath, os.O_RDWR|os.O_APPEND, os.FileMode(0665))
if err4 != nil {
t.Fatalf("Could not create file %s : %v", actualPath, err4)
}
line2 := []byte("Line 2")
rand.Read(line2)
fd.Seek(int64(len(line1)), 0)
fd.Write(line2)
fd.Close()
mi3, err5 := swarmfs3.Unmount(testMountDir)
if err5 != nil {
t.Fatalf("Could not unmount %v", err5)
}
// mount again and see if things are okay
b := [][]byte{line1, line2}
line1and2 := bytes.Join(b, []byte(""))
files["2.txt"] = fileInfo{0700, 333, 444, line1and2}
swarmfs4 := mountDir(t, ta.api, files, mi3.LatestManifest, testMountDir)
defer swarmfs4.Stop()
checkFile(t, testMountDir, "2.txt", line1and2)
}
func (ta *testAPI) removeEmptyDir(t *testing.T) {
files := make(map[string]fileInfo)
testUploadDir, _ := os.MkdirTemp(os.TempDir(), "rmdir-upload")
testMountDir, _ := os.MkdirTemp(os.TempDir(), "rmdir-mount")
files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir)
defer swarmfs1.Stop()
os.MkdirAll(filepath.Join(testMountDir, "newdir"), 0777)
mi, err3 := swarmfs1.Unmount(testMountDir)
if err3 != nil {
t.Fatalf("Could not unmount %v", err3)
}
if bzzHash != mi.LatestManifest {
t.Fatalf("same contents different hash orig(%v): new(%v)", bzzHash, mi.LatestManifest)
}
}
func (ta *testAPI) removeDirWhichHasFiles(t *testing.T) {
files := make(map[string]fileInfo)
testUploadDir, _ := os.MkdirTemp(os.TempDir(), "rmdir-upload")
testMountDir, _ := os.MkdirTemp(os.TempDir(), "rmdir-mount")
files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
files["two/five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
files["two/six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir)
defer swarmfs1.Stop()
dirPath := filepath.Join(testMountDir, "two")
os.RemoveAll(dirPath)
mi, err2 := swarmfs1.Unmount(testMountDir)
if err2 != nil {
t.Fatalf("Could not unmount %v ", err2)
}
// mount again and see if things are okay
delete(files, "two/five.txt")
delete(files, "two/six.txt")
swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir)
defer swarmfs2.Stop()
}
func (ta *testAPI) removeDirWhichHasSubDirs(t *testing.T) {
files := make(map[string]fileInfo)
testUploadDir, _ := os.MkdirTemp(os.TempDir(), "rmsubdir-upload")
testMountDir, _ := os.MkdirTemp(os.TempDir(), "rmsubdir-mount")
files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
files["two/three/2.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
files["two/three/3.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
files["two/four/5.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
files["two/four/6.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
files["two/four/six/7.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir)
defer swarmfs1.Stop()
dirPath := filepath.Join(testMountDir, "two")
os.RemoveAll(dirPath)
mi, err2 := swarmfs1.Unmount(testMountDir)
if err2 != nil {
t.Fatalf("Could not unmount %v ", err2)
}
// mount again and see if things are okay
delete(files, "two/three/2.txt")
delete(files, "two/three/3.txt")
delete(files, "two/four/5.txt")
delete(files, "two/four/6.txt")
delete(files, "two/four/six/7.txt")
swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir)
defer swarmfs2.Stop()
}
func (ta *testAPI) appendFileContentsToEnd(t *testing.T) {
files := make(map[string]fileInfo)
testUploadDir, _ := os.MkdirTemp(os.TempDir(), "appendlargefile-upload")
testMountDir, _ := os.MkdirTemp(os.TempDir(), "appendlargefile-mount")
line1 := make([]byte, 10)
rand.Read(line1)
files["1.txt"] = fileInfo{0700, 333, 444, line1}
bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir)
defer swarmfs1.Stop()
actualPath := filepath.Join(testMountDir, "1.txt")
fd, err4 := os.OpenFile(actualPath, os.O_RDWR|os.O_APPEND, os.FileMode(0665))
if err4 != nil {
t.Fatalf("Could not create file %s : %v", actualPath, err4)
}
line2 := make([]byte, 5)
rand.Read(line2)
fd.Seek(int64(len(line1)), 0)
fd.Write(line2)
fd.Close()
mi1, err5 := swarmfs1.Unmount(testMountDir)
if err5 != nil {
t.Fatalf("Could not unmount %v ", err5)
}
// mount again and see if things are okay
b := [][]byte{line1, line2}
line1and2 := bytes.Join(b, []byte(""))
files["1.txt"] = fileInfo{0700, 333, 444, line1and2}
swarmfs2 := mountDir(t, ta.api, files, mi1.LatestManifest, testMountDir)
defer swarmfs2.Stop()
checkFile(t, testMountDir, "1.txt", line1and2)
}
func TestFUSE(t *testing.T) {
datadir, err := os.MkdirTemp("", "fuse")
if err != nil {
t.Fatalf("unable to create temp dir: %v", err)
}
os.RemoveAll(datadir)
dpa, err := storage.NewLocalDPA(datadir)
if err != nil {
t.Fatal(err)
}
ta := &testAPI{api: api.NewApi(dpa, nil)}
dpa.Start()
defer dpa.Stop()
t.Run("mountListAndUmount", ta.mountListAndUnmount)
t.Run("maxMounts", ta.maxMounts)
t.Run("remount", ta.remount)
t.Run("unmount", ta.unmount)
t.Run("unmountWhenResourceBusy", ta.unmountWhenResourceBusy)
t.Run("seekInMultiChunkFile", ta.seekInMultiChunkFile)
t.Run("createNewFile", ta.createNewFile)
t.Run("createNewFileInsideDirectory", ta.createNewFileInsideDirectory)
t.Run("createNewFileInsideNewDirectory", ta.createNewFileInsideNewDirectory)
t.Run("removeExistingFile", ta.removeExistingFile)
t.Run("removeExistingFileInsideDir", ta.removeExistingFileInsideDir)
t.Run("removeNewlyAddedFile", ta.removeNewlyAddedFile)
t.Run("addNewFileAndModifyContents", ta.addNewFileAndModifyContents)
t.Run("removeEmptyDir", ta.removeEmptyDir)
t.Run("removeDirWhichHasFiles", ta.removeDirWhichHasFiles)
t.Run("removeDirWhichHasSubDirs", ta.removeDirWhichHasSubDirs)
t.Run("appendFileContentsToEnd", ta.appendFileContentsToEnd)
}

232
swarm/fuse/swarmfs_unix.go
View file

@ -1,232 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// +build linux darwin freebsd
package fuse
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"time"
"bazil.org/fuse"
"bazil.org/fuse/fs"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/swarm/api"
)
var (
errEmptyMountPoint = errors.New("need non-empty mount point")
errMaxMountCount = errors.New("max FUSE mount count reached")
errMountTimeout = errors.New("mount timeout")
errAlreadyMounted = errors.New("mount point is already serving")
)
func isFUSEUnsupportedError(err error) bool {
if perr, ok := err.(*os.PathError); ok {
return perr.Op == "open" && perr.Path == "/dev/fuse"
}
return err == fuse.ErrOSXFUSENotFound
}
// information about every active mount
type MountInfo struct {
MountPoint string
StartManifest string
LatestManifest string
rootDir *SwarmDir
fuseConnection *fuse.Conn
swarmApi *api.Api
lock *sync.RWMutex
}
func NewMountInfo(mhash, mpoint string, sapi *api.Api) *MountInfo {
newMountInfo := &MountInfo{
MountPoint: mpoint,
StartManifest: mhash,
LatestManifest: mhash,
rootDir: nil,
fuseConnection: nil,
swarmApi: sapi,
lock: &sync.RWMutex{},
}
return newMountInfo
}
func (self *SwarmFS) Mount(mhash, mountpoint string) (*MountInfo, error) {
if mountpoint == "" {
return nil, errEmptyMountPoint
}
cleanedMountPoint, err := filepath.Abs(filepath.Clean(mountpoint))
if err != nil {
return nil, err
}
self.swarmFsLock.Lock()
defer self.swarmFsLock.Unlock()
noOfActiveMounts := len(self.activeMounts)
if noOfActiveMounts >= maxFuseMounts {
return nil, errMaxMountCount
}
if _, ok := self.activeMounts[cleanedMountPoint]; ok {
return nil, errAlreadyMounted
}
log.Info(fmt.Sprintf("Attempting to mount %s ", cleanedMountPoint))
_, manifestEntryMap, err := self.swarmApi.BuildDirectoryTree(mhash, true)
if err != nil {
return nil, err
}
mi := NewMountInfo(mhash, cleanedMountPoint, self.swarmApi)
dirTree := map[string]*SwarmDir{}
rootDir := NewSwarmDir("/", mi)
dirTree["/"] = rootDir
mi.rootDir = rootDir
for suffix, entry := range manifestEntryMap {
key := common.Hex2Bytes(entry.Hash)
fullpath := "/" + suffix
basepath := filepath.Dir(fullpath)
parentDir := rootDir
dirUntilNow := ""
paths := strings.Split(basepath, "/")
for i := range paths {
if paths[i] != "" {
thisDir := paths[i]
dirUntilNow = dirUntilNow + "/" + thisDir
if _, ok := dirTree[dirUntilNow]; !ok {
dirTree[dirUntilNow] = NewSwarmDir(dirUntilNow, mi)
parentDir.directories = append(parentDir.directories, dirTree[dirUntilNow])
parentDir = dirTree[dirUntilNow]
} else {
parentDir = dirTree[dirUntilNow]
}
}
}
thisFile := NewSwarmFile(basepath, filepath.Base(fullpath), mi)
thisFile.key = key
parentDir.files = append(parentDir.files, thisFile)
}
fconn, err := fuse.Mount(cleanedMountPoint, fuse.FSName("swarmfs"), fuse.VolumeName(mhash))
if isFUSEUnsupportedError(err) {
log.Warn("Fuse not installed", "mountpoint", cleanedMountPoint, "err", err)
return nil, err
} else if err != nil {
fuse.Unmount(cleanedMountPoint)
log.Warn("Error mounting swarm manifest", "mountpoint", cleanedMountPoint, "err", err)
return nil, err
}
mi.fuseConnection = fconn
serverr := make(chan error, 1)
go func() {
log.Info(fmt.Sprintf("Serving %s at %s", mhash, cleanedMountPoint))
filesys := &SwarmRoot{root: rootDir}
if err := fs.Serve(fconn, filesys); err != nil {
log.Warn(fmt.Sprintf("Could not Serve SwarmFileSystem error: %v", err))
serverr <- err
}
}()
// Check if the mount process has an error to report.
select {
case <-time.After(mountTimeout):
fuse.Unmount(cleanedMountPoint)
return nil, errMountTimeout
case err := <-serverr:
fuse.Unmount(cleanedMountPoint)
log.Warn("Error serving swarm FUSE FS", "mountpoint", cleanedMountPoint, "err", err)
return nil, err
case <-fconn.Ready:
log.Info("Now serving swarm FUSE FS", "manifest", mhash, "mountpoint", cleanedMountPoint)
}
self.activeMounts[cleanedMountPoint] = mi
return mi, nil
}
func (self *SwarmFS) Unmount(mountpoint string) (*MountInfo, error) {
self.swarmFsLock.Lock()
defer self.swarmFsLock.Unlock()
cleanedMountPoint, err := filepath.Abs(filepath.Clean(mountpoint))
if err != nil {
return nil, err
}
mountInfo := self.activeMounts[cleanedMountPoint]
if mountInfo == nil || mountInfo.MountPoint != cleanedMountPoint {
return nil, fmt.Errorf("%s is not mounted", cleanedMountPoint)
}
err = fuse.Unmount(cleanedMountPoint)
if err != nil {
err1 := externalUnmount(cleanedMountPoint)
if err1 != nil {
errStr := fmt.Sprintf("UnMount error: %v", err)
log.Warn(errStr)
return nil, err1
}
}
mountInfo.fuseConnection.Close()
delete(self.activeMounts, cleanedMountPoint)
succString := fmt.Sprintf("UnMounting %v succeeded", cleanedMountPoint)
log.Info(succString)
return mountInfo, nil
}
func (self *SwarmFS) Listmounts() []*MountInfo {
self.swarmFsLock.RLock()
defer self.swarmFsLock.RUnlock()
rows := make([]*MountInfo, 0, len(self.activeMounts))
for _, mi := range self.activeMounts {
rows = append(rows, mi)
}
return rows
}
func (self *SwarmFS) Stop() bool {
for mp := range self.activeMounts {
mountInfo := self.activeMounts[mp]
self.Unmount(mountInfo.MountPoint)
}
return true
}

121
swarm/fuse/swarmfs_util.go
View file

@ -1,121 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// +build linux darwin freebsd
package fuse
import (
"context"
"fmt"
"os/exec"
"runtime"
"github.com/XinFinOrg/XDPoSChain/log"
)
func externalUnmount(mountPoint string) error {
ctx, cancel := context.WithTimeout(context.Background(), unmountTimeout)
defer cancel()
// Try generic umount.
if err := exec.CommandContext(ctx, "umount", mountPoint).Run(); err == nil {
return nil
}
// Try FUSE-specific commands if umount didn't work.
switch runtime.GOOS {
case "darwin":
return exec.CommandContext(ctx, "diskutil", "umount", "force", mountPoint).Run()
case "linux":
return exec.CommandContext(ctx, "fusermount", "-u", mountPoint).Run()
default:
return fmt.Errorf("unmount: unimplemented")
}
}
func addFileToSwarm(sf *SwarmFile, content []byte, size int) error {
fkey, mhash, err := sf.mountInfo.swarmApi.AddFile(sf.mountInfo.LatestManifest, sf.path, sf.name, content, true)
if err != nil {
return err
}
sf.lock.Lock()
defer sf.lock.Unlock()
sf.key = fkey
sf.fileSize = int64(size)
sf.mountInfo.lock.Lock()
defer sf.mountInfo.lock.Unlock()
sf.mountInfo.LatestManifest = mhash
log.Info("Added new file:", "fname", sf.name, "New Manifest hash", mhash)
return nil
}
func removeFileFromSwarm(sf *SwarmFile) error {
mkey, err := sf.mountInfo.swarmApi.RemoveFile(sf.mountInfo.LatestManifest, sf.path, sf.name, true)
if err != nil {
return err
}
sf.mountInfo.lock.Lock()
defer sf.mountInfo.lock.Unlock()
sf.mountInfo.LatestManifest = mkey
log.Info("Removed file:", "fname", sf.name, "New Manifest hash", mkey)
return nil
}
func removeDirectoryFromSwarm(sd *SwarmDir) error {
if len(sd.directories) == 0 && len(sd.files) == 0 {
return nil
}
for _, d := range sd.directories {
err := removeDirectoryFromSwarm(d)
if err != nil {
return err
}
}
for _, f := range sd.files {
err := removeFileFromSwarm(f)
if err != nil {
return err
}
}
return nil
}
func appendToExistingFileInSwarm(sf *SwarmFile, content []byte, offset int64, length int64) error {
fkey, mhash, err := sf.mountInfo.swarmApi.AppendFile(sf.mountInfo.LatestManifest, sf.path, sf.name, sf.fileSize, content, sf.key, offset, length, true)
if err != nil {
return err
}
sf.lock.Lock()
defer sf.lock.Unlock()
sf.key = fkey
sf.fileSize = sf.fileSize + int64(len(content))
sf.mountInfo.lock.Lock()
defer sf.mountInfo.lock.Unlock()
sf.mountInfo.LatestManifest = mhash
log.Info("Appended file:", "fname", sf.name, "New Manifest hash", mhash)
return nil
}

91
swarm/metrics/flags.go
View file

@ -1,91 +0,0 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package metrics
import (
"time"
"github.com/XinFinOrg/XDPoSChain/cmd/utils"
"github.com/XinFinOrg/XDPoSChain/log"
gethmetrics "github.com/XinFinOrg/XDPoSChain/metrics"
"github.com/XinFinOrg/XDPoSChain/metrics/influxdb"
"gopkg.in/urfave/cli.v1"
)
var (
metricsEnableInfluxDBExportFlag = cli.BoolFlag{
Name: "metrics.influxdb.export",
Usage: "Enable metrics export/push to an external InfluxDB database",
}
metricsInfluxDBEndpointFlag = cli.StringFlag{
Name: "metrics.influxdb.endpoint",
Usage: "Metrics InfluxDB endpoint",
Value: "http://127.0.0.1:8086",
}
metricsInfluxDBDatabaseFlag = cli.StringFlag{
Name: "metrics.influxdb.database",
Usage: "Metrics InfluxDB database",
Value: "metrics",
}
metricsInfluxDBUsernameFlag = cli.StringFlag{
Name: "metrics.influxdb.username",
Usage: "Metrics InfluxDB username",
Value: "",
}
metricsInfluxDBPasswordFlag = cli.StringFlag{
Name: "metrics.influxdb.password",
Usage: "Metrics InfluxDB password",
Value: "",
}
// The `host` tag is part of every measurement sent to InfluxDB. Queries on tags are faster in InfluxDB.
// It is used so that we can group all nodes and average a measurement across all of them, but also so
// that we can select a specific node and inspect its measurements.
// https://docs.influxdata.com/influxdb/v1.4/concepts/key_concepts/#tag-key
metricsInfluxDBHostTagFlag = cli.StringFlag{
Name: "metrics.influxdb.host.tag",
Usage: "Metrics InfluxDB `host` tag attached to all measurements",
Value: "localhost",
}
)
// Flags holds all command-line flags required for metrics collection.
var Flags = []cli.Flag{
utils.MetricsEnabledFlag,
metricsEnableInfluxDBExportFlag,
metricsInfluxDBEndpointFlag, metricsInfluxDBDatabaseFlag, metricsInfluxDBUsernameFlag, metricsInfluxDBPasswordFlag, metricsInfluxDBHostTagFlag,
}
func Setup(ctx *cli.Context) {
if gethmetrics.Enabled {
log.Info("Enabling swarm metrics collection")
var (
enableExport = ctx.GlobalBool(metricsEnableInfluxDBExportFlag.Name)
endpoint = ctx.GlobalString(metricsInfluxDBEndpointFlag.Name)
database = ctx.GlobalString(metricsInfluxDBDatabaseFlag.Name)
username = ctx.GlobalString(metricsInfluxDBUsernameFlag.Name)
password = ctx.GlobalString(metricsInfluxDBPasswordFlag.Name)
hosttag = ctx.GlobalString(metricsInfluxDBHostTagFlag.Name)
)
if enableExport {
log.Info("Enabling swarm metrics export to InfluxDB")
go influxdb.InfluxDBWithTags(gethmetrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "swarm.", map[string]string{
"host": hosttag,
})
}
}
}

232
swarm/network/depo.go
View file

@ -1,232 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package network
import (
"bytes"
"encoding/binary"
"fmt"
"time"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/metrics"
"github.com/XinFinOrg/XDPoSChain/swarm/storage"
)
//metrics variables
var (
syncReceiveCount = metrics.NewRegisteredCounter("network.sync.recv.count", nil)
syncReceiveIgnore = metrics.NewRegisteredCounter("network.sync.recv.ignore", nil)
syncSendCount = metrics.NewRegisteredCounter("network.sync.send.count", nil)
syncSendRefused = metrics.NewRegisteredCounter("network.sync.send.refused", nil)
syncSendNotFound = metrics.NewRegisteredCounter("network.sync.send.notfound", nil)
)
// Handler for storage/retrieval related protocol requests
// implements the StorageHandler interface used by the bzz protocol
type Depo struct {
hashfunc storage.SwarmHasher
localStore storage.ChunkStore
netStore storage.ChunkStore
}
func NewDepo(hash storage.SwarmHasher, localStore, remoteStore storage.ChunkStore) *Depo {
return &Depo{
hashfunc: hash,
localStore: localStore,
netStore: remoteStore, // entrypoint internal
}
}
// Handles UnsyncedKeysMsg after msg decoding - unsynced hashes upto sync state
// * the remote sync state is just stored and handled in protocol
// * filters through the new syncRequests and send the ones missing
// * back immediately as a deliveryRequest message
// * empty message just pings back for more (is this needed?)
// * strict signed sync states may be needed.
func (self *Depo) HandleUnsyncedKeysMsg(req *unsyncedKeysMsgData, p *peer) error {
unsynced := req.Unsynced
var missing []*syncRequest
var chunk *storage.Chunk
var err error
for _, req := range unsynced {
// skip keys that are found,
chunk, err = self.localStore.Get(req.Key[:])
if err != nil || chunk.SData == nil {
missing = append(missing, req)
}
}
log.Debug(fmt.Sprintf("Depo.HandleUnsyncedKeysMsg: received %v unsynced keys: %v missing. new state: %v", len(unsynced), len(missing), req.State))
log.Trace(fmt.Sprintf("Depo.HandleUnsyncedKeysMsg: received %v", unsynced))
// send delivery request with missing keys
err = p.deliveryRequest(missing)
if err != nil {
return err
}
// set peers state to persist
p.syncState = req.State
return nil
}
// Handles deliveryRequestMsg
// * serves actual chunks asked by the remote peer
// by pushing to the delivery queue (sync db) of the correct priority
// (remote peer is free to reprioritize)
// * the message implies remote peer wants more, so trigger for
// * new outgoing unsynced keys message is fired
func (self *Depo) HandleDeliveryRequestMsg(req *deliveryRequestMsgData, p *peer) error {
deliver := req.Deliver
// queue the actual delivery of a chunk ()
log.Trace(fmt.Sprintf("Depo.HandleDeliveryRequestMsg: received %v delivery requests: %v", len(deliver), deliver))
for _, sreq := range deliver {
// TODO: look up in cache here or in deliveries
// priorities are taken from the message so the remote party can
// reprioritise to at their leisure
// r = self.pullCached(sreq.Key) // pulls and deletes from cache
Push(p, sreq.Key, sreq.Priority)
}
// sends it out as unsyncedKeysMsg
p.syncer.sendUnsyncedKeys()
return nil
}
// the entrypoint for store requests coming from the bzz wire protocol
// if key found locally, return. otherwise
// remote is untrusted, so hash is verified and chunk passed on to NetStore
func (self *Depo) HandleStoreRequestMsg(req *storeRequestMsgData, p *peer) {
var islocal bool
req.from = p
chunk, err := self.localStore.Get(req.Key)
switch {
case err != nil:
log.Trace(fmt.Sprintf("Depo.handleStoreRequest: %v not found locally. create new chunk/request", req.Key))
// not found in memory cache, ie., a genuine store request
// create chunk
syncReceiveCount.Inc(1)
chunk = storage.NewChunk(req.Key, nil)
case chunk.SData == nil:
// found chunk in memory store, needs the data, validate now
log.Trace(fmt.Sprintf("Depo.HandleStoreRequest: %v. request entry found", req))
default:
// data is found, store request ignored
// this should update access count?
syncReceiveIgnore.Inc(1)
log.Trace(fmt.Sprintf("Depo.HandleStoreRequest: %v found locally. ignore.", req))
islocal = true
//return
}
hasher := self.hashfunc()
hasher.Write(req.SData)
if !bytes.Equal(hasher.Sum(nil), req.Key) {
// data does not validate, ignore
// TODO: peer should be penalised/dropped?
log.Warn(fmt.Sprintf("Depo.HandleStoreRequest: chunk invalid. store request ignored: %v", req))
return
}
if islocal {
return
}
// update chunk with size and data
chunk.SData = req.SData // protocol validates that SData is minimum 9 bytes long (int64 size + at least one byte of data)
chunk.Size = int64(binary.LittleEndian.Uint64(req.SData[0:8]))
log.Trace(fmt.Sprintf("delivery of %v from %v", chunk, p))
chunk.Source = p
self.netStore.Put(chunk)
}
// entrypoint for retrieve requests coming from the bzz wire protocol
// checks swap balance - return if peer has no credit
func (self *Depo) HandleRetrieveRequestMsg(req *retrieveRequestMsgData, p *peer) {
req.from = p
// swap - record credit for 1 request
// note that only charge actual reqsearches
var err error
if p.swap != nil {
err = p.swap.Add(1)
}
if err != nil {
log.Warn(fmt.Sprintf("Depo.HandleRetrieveRequest: %v - cannot process request: %v", req.Key.Log(), err))
return
}
// call storage.NetStore#Get which
// blocks until local retrieval finished
// launches cloud retrieval
chunk, _ := self.netStore.Get(req.Key)
req = self.strategyUpdateRequest(chunk.Req, req)
// check if we can immediately deliver
if chunk.SData != nil {
log.Trace(fmt.Sprintf("Depo.HandleRetrieveRequest: %v - content found, delivering...", req.Key.Log()))
if req.MaxSize == 0 || int64(req.MaxSize) >= chunk.Size {
sreq := &storeRequestMsgData{
Id: req.Id,
Key: chunk.Key,
SData: chunk.SData,
requestTimeout: req.timeout, //
}
syncSendCount.Inc(1)
p.syncer.addRequest(sreq, DeliverReq)
} else {
syncSendRefused.Inc(1)
log.Trace(fmt.Sprintf("Depo.HandleRetrieveRequest: %v - content found, not wanted", req.Key.Log()))
}
} else {
syncSendNotFound.Inc(1)
log.Trace(fmt.Sprintf("Depo.HandleRetrieveRequest: %v - content not found locally. asked swarm for help. will get back", req.Key.Log()))
}
}
// add peer request the chunk and decides the timeout for the response if still searching
func (self *Depo) strategyUpdateRequest(rs *storage.RequestStatus, origReq *retrieveRequestMsgData) (req *retrieveRequestMsgData) {
log.Trace(fmt.Sprintf("Depo.strategyUpdateRequest: key %v", origReq.Key.Log()))
// we do not create an alternative one
req = origReq
if rs != nil {
self.addRequester(rs, req)
req.setTimeout(self.searchTimeout(rs, req))
}
return
}
// decides the timeout promise sent with the immediate peers response to a retrieve request
// if timeout is explicitly set and expired
func (self *Depo) searchTimeout(rs *storage.RequestStatus, req *retrieveRequestMsgData) (timeout *time.Time) {
reqt := req.getTimeout()
t := time.Now().Add(searchTimeout)
if reqt != nil && reqt.Before(t) {
return reqt
} else {
return &t
}
}
/*
adds a new peer to an existing open request
only add if less than requesterCount peers forwarded the same request id so far
note this is done irrespective of status (searching or found)
*/
func (self *Depo) addRequester(rs *storage.RequestStatus, req *retrieveRequestMsgData) {
log.Trace(fmt.Sprintf("Depo.addRequester: key %v - add peer to req.Id %v", req.Key.Log(), req.Id))
list := rs.Requesters[req.Id]
rs.Requesters[req.Id] = append(list, req)
}

150
swarm/network/forwarding.go
View file

@ -1,150 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package network
import (
"fmt"
"math/rand"
"time"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/swarm/storage"
)
const requesterCount = 3
/*
forwarder implements the CloudStore interface (use by storage.NetStore)
and serves as the cloud store backend orchestrating storage/retrieval/delivery
via the native bzz protocol
which uses an MSB logarithmic distance-based semi-permanent Kademlia table for
* recursive forwarding style routing for retrieval
* smart syncronisation
*/
type forwarder struct {
hive *Hive
}
func NewForwarder(hive *Hive) *forwarder {
return &forwarder{hive: hive}
}
// generate a unique id uint64
func generateId() uint64 {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
return uint64(r.Int63())
}
var searchTimeout = 3 * time.Second
// forwarding logic
// logic propagating retrieve requests to peers given by the kademlia hive
func (self *forwarder) Retrieve(chunk *storage.Chunk) {
peers := self.hive.getPeers(chunk.Key, 0)
log.Trace(fmt.Sprintf("forwarder.Retrieve: %v - received %d peers from KΛÐΞMLIΛ...", chunk.Key.Log(), len(peers)))
OUT:
for _, p := range peers {
log.Trace(fmt.Sprintf("forwarder.Retrieve: sending retrieveRequest %v to peer [%v]", chunk.Key.Log(), p))
for _, recipients := range chunk.Req.Requesters {
for _, recipient := range recipients {
req := recipient.(*retrieveRequestMsgData)
if req.from.Addr() == p.Addr() {
continue OUT
}
}
}
req := &retrieveRequestMsgData{
Key: chunk.Key,
Id: generateId(),
}
var err error
if p.swap != nil {
err = p.swap.Add(-1)
}
if err == nil {
p.retrieve(req)
break OUT
}
log.Warn(fmt.Sprintf("forwarder.Retrieve: unable to send retrieveRequest to peer [%v]: %v", chunk.Key.Log(), err))
}
}
// requests to specific peers given by the kademlia hive
// except for peers that the store request came from (if any)
// delivery queueing taken care of by syncer
func (self *forwarder) Store(chunk *storage.Chunk) {
var n int
msg := &storeRequestMsgData{
Key: chunk.Key,
SData: chunk.SData,
}
var source *peer
if chunk.Source != nil {
source = chunk.Source.(*peer)
}
for _, p := range self.hive.getPeers(chunk.Key, 0) {
log.Trace(fmt.Sprintf("forwarder.Store: %v %v", p, chunk))
if p.syncer != nil && (source == nil || p.Addr() != source.Addr()) {
n++
Deliver(p, msg, PropagateReq)
}
}
log.Trace(fmt.Sprintf("forwarder.Store: sent to %v peers (chunk = %v)", n, chunk))
}
// once a chunk is found deliver it to its requesters unless timed out
func (self *forwarder) Deliver(chunk *storage.Chunk) {
// iterate over request entries
for id, requesters := range chunk.Req.Requesters {
counter := requesterCount
msg := &storeRequestMsgData{
Key: chunk.Key,
SData: chunk.SData,
}
var n int
var req *retrieveRequestMsgData
// iterate over requesters with the same id
for id, r := range requesters {
req = r.(*retrieveRequestMsgData)
if req.timeout == nil || req.timeout.After(time.Now()) {
log.Trace(fmt.Sprintf("forwarder.Deliver: %v -> %v", req.Id, req.from))
msg.Id = uint64(id)
Deliver(req.from, msg, DeliverReq)
n++
counter--
if counter <= 0 {
break
}
}
}
log.Trace(fmt.Sprintf("forwarder.Deliver: submit chunk %v (request id %v) for delivery to %v peers", chunk.Key.Log(), id, n))
}
}
// initiate delivery of a chunk to a particular peer via syncer#addRequest
// depending on syncer mode and priority settings and sync request type
// this either goes via confirmation roundtrip or queued or pushed directly
func Deliver(p *peer, req interface{}, ty int) {
p.syncer.addRequest(req, ty)
}
// push chunk over to peer
func Push(p *peer, key storage.Key, priority uint) {
p.syncer.doDelivery(key, priority, p.syncer.quit)
}

403
swarm/network/hive.go
View file

@ -1,403 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package network
import (
"fmt"
"math/rand"
"path/filepath"
"time"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/metrics"
"github.com/XinFinOrg/XDPoSChain/p2p/discover"
"github.com/XinFinOrg/XDPoSChain/p2p/netutil"
"github.com/XinFinOrg/XDPoSChain/swarm/network/kademlia"
"github.com/XinFinOrg/XDPoSChain/swarm/storage"
)
// Hive is the logistic manager of the swarm
// it uses a generic kademlia nodetable to find best peer list
// for any target
// this is used by the netstore to search for content in the swarm
// the bzz protocol peersMsgData exchange is relayed to Kademlia
// for db storage and filtering
// connections and disconnections are reported and relayed
// to keep the nodetable uptodate
var (
peersNumGauge = metrics.NewRegisteredGauge("network.peers.num", nil)
addPeerCounter = metrics.NewRegisteredCounter("network.addpeer.count", nil)
removePeerCounter = metrics.NewRegisteredCounter("network.removepeer.count", nil)
)
type Hive struct {
listenAddr func() string
callInterval uint64
id discover.NodeID
addr kademlia.Address
kad *kademlia.Kademlia
path string
quit chan bool
toggle chan bool
more chan bool
// for testing only
swapEnabled bool
syncEnabled bool
blockRead bool
blockWrite bool
}
const (
callInterval = 3000000000
// bucketSize = 3
// maxProx = 8
// proxBinSize = 4
)
type HiveParams struct {
CallInterval uint64
KadDbPath string
*kademlia.KadParams
}
//create default params
func NewDefaultHiveParams() *HiveParams {
kad := kademlia.NewDefaultKadParams()
// kad.BucketSize = bucketSize
// kad.MaxProx = maxProx
// kad.ProxBinSize = proxBinSize
return &HiveParams{
CallInterval: callInterval,
KadParams: kad,
}
}
//this can only finally be set after all config options (file, cmd line, env vars)
//have been evaluated
func (self *HiveParams) Init(path string) {
self.KadDbPath = filepath.Join(path, "bzz-peers.json")
}
func NewHive(addr common.Hash, params *HiveParams, swapEnabled, syncEnabled bool) *Hive {
kad := kademlia.New(kademlia.Address(addr), params.KadParams)
return &Hive{
callInterval: params.CallInterval,
kad: kad,
addr: kad.Addr(),
path: params.KadDbPath,
swapEnabled: swapEnabled,
syncEnabled: syncEnabled,
}
}
func (self *Hive) SyncEnabled(on bool) {
self.syncEnabled = on
}
func (self *Hive) SwapEnabled(on bool) {
self.swapEnabled = on
}
func (self *Hive) BlockNetworkRead(on bool) {
self.blockRead = on
}
func (self *Hive) BlockNetworkWrite(on bool) {
self.blockWrite = on
}
// public accessor to the hive base address
func (self *Hive) Addr() kademlia.Address {
return self.addr
}
// Start receives network info only at startup
// listedAddr is a function to retrieve listening address to advertise to peers
// connectPeer is a function to connect to a peer based on its NodeID or enode URL
// there are called on the p2p.Server which runs on the node
func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPeer func(string) error) (err error) {
self.toggle = make(chan bool)
self.more = make(chan bool)
self.quit = make(chan bool)
self.id = id
self.listenAddr = listenAddr
err = self.kad.Load(self.path, nil)
if err != nil {
log.Warn(fmt.Sprintf("Warning: error reading kaddb '%s' (skipping): %v", self.path, err))
err = nil
}
// this loop is doing bootstrapping and maintains a healthy table
go self.keepAlive()
go func() {
// whenever toggled ask kademlia about most preferred peer
for alive := range self.more {
if !alive {
// receiving false closes the loop while allowing parallel routines
// to attempt to write to more (remove Peer when shutting down)
return
}
node, need, proxLimit := self.kad.Suggest()
if node != nil && len(node.Url) > 0 {
log.Trace(fmt.Sprintf("call known bee %v", node.Url))
// enode or any lower level connection address is unnecessary in future
// discovery table is used to look it up.
connectPeer(node.Url)
}
if need {
// a random peer is taken from the table
peers := self.kad.FindClosest(kademlia.RandomAddressAt(self.addr, rand.Intn(self.kad.MaxProx)), 1)
if len(peers) > 0 {
// a random address at prox bin 0 is sent for lookup
randAddr := kademlia.RandomAddressAt(self.addr, proxLimit)
req := &retrieveRequestMsgData{
Key: storage.Key(randAddr[:]),
}
log.Trace(fmt.Sprintf("call any bee near %v (PO%03d) - messenger bee: %v", randAddr, proxLimit, peers[0]))
peers[0].(*peer).retrieve(req)
} else {
log.Warn(fmt.Sprintf("no peer"))
}
log.Trace(fmt.Sprintf("buzz kept alive"))
} else {
log.Info(fmt.Sprintf("no need for more bees"))
}
select {
case self.toggle <- need:
case <-self.quit:
return
}
log.Debug(fmt.Sprintf("queen's address: %v, population: %d (%d)", self.addr, self.kad.Count(), self.kad.DBCount()))
}
}()
return
}
// keepAlive is a forever loop
// in its awake state it periodically triggers connection attempts
// by writing to self.more until Kademlia Table is saturated
// wake state is toggled by writing to self.toggle
// it restarts if the table becomes non-full again due to disconnections
func (self *Hive) keepAlive() {
alarm := time.NewTicker(time.Duration(self.callInterval)).C
for {
peersNumGauge.Update(int64(self.kad.Count()))
select {
case <-alarm:
if self.kad.DBCount() > 0 {
select {
case self.more <- true:
log.Debug(fmt.Sprintf("buzz wakeup"))
default:
}
}
case need := <-self.toggle:
if alarm == nil && need {
alarm = time.NewTicker(time.Duration(self.callInterval)).C
}
if alarm != nil && !need {
alarm = nil
}
case <-self.quit:
return
}
}
}
func (self *Hive) Stop() error {
// closing toggle channel quits the updateloop
close(self.quit)
return self.kad.Save(self.path, saveSync)
}
// called at the end of a successful protocol handshake
func (self *Hive) addPeer(p *peer) error {
addPeerCounter.Inc(1)
defer func() {
select {
case self.more <- true:
default:
}
}()
log.Trace(fmt.Sprintf("hi new bee %v", p))
err := self.kad.On(p, loadSync)
if err != nil {
return err
}
// self lookup (can be encoded as nil/zero key since peers addr known) + no id ()
// the most common way of saying hi in bzz is initiation of gossip
// let me know about anyone new from my hood , here is the storageradius
// to send the 6 byte self lookup
// we do not record as request or forward it, just reply with peers
p.retrieve(&retrieveRequestMsgData{})
log.Trace(fmt.Sprintf("'whatsup wheresdaparty' sent to %v", p))
return nil
}
// called after peer disconnected
func (self *Hive) removePeer(p *peer) {
removePeerCounter.Inc(1)
log.Debug(fmt.Sprintf("bee %v removed", p))
self.kad.Off(p, saveSync)
select {
case self.more <- true:
default:
}
if self.kad.Count() == 0 {
log.Debug(fmt.Sprintf("empty, all bees gone"))
}
}
// Retrieve a list of live peers that are closer to target than us
func (self *Hive) getPeers(target storage.Key, max int) (peers []*peer) {
var addr kademlia.Address
copy(addr[:], target[:])
for _, node := range self.kad.FindClosest(addr, max) {
peers = append(peers, node.(*peer))
}
return
}
// disconnects all the peers
func (self *Hive) DropAll() {
log.Info(fmt.Sprintf("dropping all bees"))
for _, node := range self.kad.FindClosest(kademlia.Address{}, 0) {
node.Drop()
}
}
// contructor for kademlia.NodeRecord based on peer address alone
// TODO: should go away and only addr passed to kademlia
func newNodeRecord(addr *peerAddr) *kademlia.NodeRecord {
now := time.Now()
return &kademlia.NodeRecord{
Addr: addr.Addr,
Url: addr.String(),
Seen: now,
After: now,
}
}
// called by the protocol when receiving peerset (for target address)
// peersMsgData is converted to a slice of NodeRecords for Kademlia
// this is to store all thats needed
func (self *Hive) HandlePeersMsg(req *peersMsgData, from *peer) {
var nrs []*kademlia.NodeRecord
for _, p := range req.Peers {
if err := netutil.CheckRelayIP(from.remoteAddr.IP, p.IP); err != nil {
log.Trace(fmt.Sprintf("invalid peer IP %v from %v: %v", from.remoteAddr.IP, p.IP, err))
continue
}
nrs = append(nrs, newNodeRecord(p))
}
self.kad.Add(nrs)
}
// peer wraps the protocol instance to represent a connected peer
// it implements kademlia.Node interface
type peer struct {
*bzz // protocol instance running on peer connection
}
// protocol instance implements kademlia.Node interface (embedded peer)
func (self *peer) Addr() kademlia.Address {
return self.remoteAddr.Addr
}
func (self *peer) Url() string {
return self.remoteAddr.String()
}
// TODO take into account traffic
func (self *peer) LastActive() time.Time {
return self.lastActive
}
// reads the serialised form of sync state persisted as the 'Meta' attribute
// and sets the decoded syncState on the online node
func loadSync(record *kademlia.NodeRecord, node kademlia.Node) error {
p, ok := node.(*peer)
if !ok {
return fmt.Errorf("invalid type")
}
if record.Meta == nil {
log.Debug(fmt.Sprintf("no sync state for node record %v setting default", record))
p.syncState = &syncState{DbSyncState: &storage.DbSyncState{}}
return nil
}
state, err := decodeSync(record.Meta)
if err != nil {
return fmt.Errorf("error decoding kddb record meta info into a sync state: %v", err)
}
log.Trace(fmt.Sprintf("sync state for node record %v read from Meta: %s", record, string(*(record.Meta))))
p.syncState = state
return err
}
// callback when saving a sync state
func saveSync(record *kademlia.NodeRecord, node kademlia.Node) {
if p, ok := node.(*peer); ok {
meta, err := encodeSync(p.syncState)
if err != nil {
log.Warn(fmt.Sprintf("error saving sync state for %v: %v", node, err))
return
}
log.Trace(fmt.Sprintf("saved sync state for %v: %s", node, string(*meta)))
record.Meta = meta
}
}
// the immediate response to a retrieve request,
// sends relevant peer data given by the kademlia hive to the requester
// TODO: remember peers sent for duration of the session, only new peers sent
func (self *Hive) peers(req *retrieveRequestMsgData) {
if req != nil {
var addrs []*peerAddr
if req.timeout == nil || time.Now().Before(*(req.timeout)) {
key := req.Key
// self lookup from remote peer
if storage.IsZeroKey(key) {
addr := req.from.Addr()
key = storage.Key(addr[:])
req.Key = nil
}
// get peer addresses from hive
for _, peer := range self.getPeers(key, int(req.MaxPeers)) {
addrs = append(addrs, peer.remoteAddr)
}
log.Debug(fmt.Sprintf("Hive sending %d peer addresses to %v. req.Id: %v, req.Key: %v", len(addrs), req.from, req.Id, req.Key.Log()))
peersData := &peersMsgData{
Peers: addrs,
Key: req.Key,
Id: req.Id,
}
peersData.setTimeout(req.timeout)
req.from.peers(peersData)
}
}
}
func (self *Hive) String() string {
return self.kad.String()
}

173
swarm/network/kademlia/address.go
View file

@ -1,173 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package kademlia
import (
"fmt"
"math/rand"
"strings"
"github.com/XinFinOrg/XDPoSChain/common"
)
type Address common.Hash
func (a Address) String() string {
return fmt.Sprintf("%x", a[:])
}
func (a *Address) MarshalJSON() (out []byte, err error) {
return []byte(`"` + a.String() + `"`), nil
}
func (a *Address) UnmarshalJSON(value []byte) error {
*a = Address(common.HexToHash(string(value[1 : len(value)-1])))
return nil
}
// the string form of the binary representation of an address (only first 8 bits)
func (a Address) Bin() string {
var bs []string
for _, b := range a[:] {
bs = append(bs, fmt.Sprintf("%08b", b))
}
return strings.Join(bs, "")
}
/*
Proximity(x, y) returns the proximity order of the MSB distance between x and y
The distance metric MSB(x, y) of two equal length byte sequences x an y is the
value of the binary integer cast of the x^y, ie., x and y bitwise xor-ed.
the binary cast is big endian: most significant bit first (=MSB).
Proximity(x, y) is a discrete logarithmic scaling of the MSB distance.
It is defined as the reverse rank of the integer part of the base 2
logarithm of the distance.
It is calculated by counting the number of common leading zeros in the (MSB)
binary representation of the x^y.
(0 farthest, 255 closest, 256 self)
*/
func proximity(one, other Address) (ret int) {
for i := 0; i < len(one); i++ {
oxo := one[i] ^ other[i]
for j := 0; j < 8; j++ {
if (oxo>>uint8(7-j))&0x01 != 0 {
return i*8 + j
}
}
}
return len(one) * 8
}
// Address.ProxCmp compares the distances a->target and b->target.
// Returns -1 if a is closer to target, 1 if b is closer to target
// and 0 if they are equal.
func (target Address) ProxCmp(a, b Address) int {
for i := range target {
da := a[i] ^ target[i]
db := b[i] ^ target[i]
if da > db {
return 1
} else if da < db {
return -1
}
}
return 0
}
// randomAddressAt(address, prox) generates a random address
// at proximity order prox relative to address
// if prox is negative a random address is generated
func RandomAddressAt(self Address, prox int) (addr Address) {
addr = self
var pos int
if prox >= 0 {
pos = prox / 8
trans := prox % 8
transbytea := byte(0)
for j := 0; j <= trans; j++ {
transbytea |= 1 << uint8(7-j)
}
flipbyte := byte(1 << uint8(7-trans))
transbyteb := transbytea ^ byte(255)
randbyte := byte(rand.Intn(255))
addr[pos] = ((addr[pos] & transbytea) ^ flipbyte) | randbyte&transbyteb
}
for i := pos + 1; i < len(addr); i++ {
addr[i] = byte(rand.Intn(255))
}
return
}
// KeyRange(a0, a1, proxLimit) returns the address inclusive address
// range that contain addresses closer to one than other
func KeyRange(one, other Address, proxLimit int) (start, stop Address) {
prox := proximity(one, other)
if prox >= proxLimit {
prox = proxLimit
}
start = CommonBitsAddrByte(one, other, byte(0x00), prox)
stop = CommonBitsAddrByte(one, other, byte(0xff), prox)
return
}
func CommonBitsAddrF(self, other Address, f func() byte, p int) (addr Address) {
prox := proximity(self, other)
var pos int
if p <= prox {
prox = p
}
pos = prox / 8
addr = self
trans := byte(prox % 8)
var transbytea byte
if p > prox {
transbytea = byte(0x7f)
} else {
transbytea = byte(0xff)
}
transbytea >>= trans
transbyteb := transbytea ^ byte(0xff)
addrpos := addr[pos]
addrpos &= transbyteb
if p > prox {
addrpos ^= byte(0x80 >> trans)
}
addrpos |= transbytea & f()
addr[pos] = addrpos
for i := pos + 1; i < len(addr); i++ {
addr[i] = f()
}
return
}
func CommonBitsAddr(self, other Address, prox int) (addr Address) {
return CommonBitsAddrF(self, other, func() byte { return byte(rand.Intn(255)) }, prox)
}
func CommonBitsAddrByte(self, other Address, b byte, prox int) (addr Address) {
return CommonBitsAddrF(self, other, func() byte { return b }, prox)
}
// randomAddressAt() generates a random address
func RandomAddress() Address {
return RandomAddressAt(Address{}, -1)
}

96
swarm/network/kademlia/address_test.go
View file

@ -1,96 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package kademlia
import (
"math/rand"
"reflect"
"testing"
"github.com/XinFinOrg/XDPoSChain/common"
)
func (Address) Generate(rand *rand.Rand, size int) reflect.Value {
var id Address
for i := 0; i < len(id); i++ {
id[i] = byte(uint8(rand.Intn(255)))
}
return reflect.ValueOf(id)
}
func TestCommonBitsAddrF(t *testing.T) {
a := Address(common.HexToHash("0x0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"))
b := Address(common.HexToHash("0x8123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"))
c := Address(common.HexToHash("0x4123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"))
d := Address(common.HexToHash("0x0023456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"))
e := Address(common.HexToHash("0x01A3456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"))
ab := CommonBitsAddrF(a, b, func() byte { return byte(0x00) }, 10)
expab := Address(common.HexToHash("0x8000000000000000000000000000000000000000000000000000000000000000"))
if ab != expab {
t.Fatalf("%v != %v", ab, expab)
}
ac := CommonBitsAddrF(a, c, func() byte { return byte(0x00) }, 10)
expac := Address(common.HexToHash("0x4000000000000000000000000000000000000000000000000000000000000000"))
if ac != expac {
t.Fatalf("%v != %v", ac, expac)
}
ad := CommonBitsAddrF(a, d, func() byte { return byte(0x00) }, 10)
expad := Address(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"))
if ad != expad {
t.Fatalf("%v != %v", ad, expad)
}
ae := CommonBitsAddrF(a, e, func() byte { return byte(0x00) }, 10)
expae := Address(common.HexToHash("0x0180000000000000000000000000000000000000000000000000000000000000"))
if ae != expae {
t.Fatalf("%v != %v", ae, expae)
}
acf := CommonBitsAddrF(a, c, func() byte { return byte(0xff) }, 10)
expacf := Address(common.HexToHash("0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"))
if acf != expacf {
t.Fatalf("%v != %v", acf, expacf)
}
aeo := CommonBitsAddrF(a, e, func() byte { return byte(0x00) }, 2)
expaeo := Address(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"))
if aeo != expaeo {
t.Fatalf("%v != %v", aeo, expaeo)
}
aep := CommonBitsAddrF(a, e, func() byte { return byte(0xff) }, 2)
expaep := Address(common.HexToHash("0x3fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"))
if aep != expaep {
t.Fatalf("%v != %v", aep, expaep)
}
}
func TestRandomAddressAt(t *testing.T) {
var a Address
for i := 0; i < 100; i++ {
a = RandomAddress()
prox := rand.Intn(255)
b := RandomAddressAt(a, prox)
if proximity(a, b) != prox {
t.Fatalf("incorrect address prox(%v, %v) == %v (expected %v)", a, b, proximity(a, b), prox)
}
}
}

348
swarm/network/kademlia/kaddb.go
View file

@ -1,348 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package kademlia
import (
"encoding/json"
"fmt"
"os"
"sync"
"time"
"github.com/XinFinOrg/XDPoSChain/log"
)
type NodeData interface {
json.Marshaler
json.Unmarshaler
}
// allow inactive peers under
type NodeRecord struct {
Addr Address // address of node
Url string // Url, used to connect to node
After time.Time // next call after time
Seen time.Time // last connected at time
Meta *json.RawMessage // arbitrary metadata saved for a peer
node Node
}
func (self *NodeRecord) setSeen() {
t := time.Now()
self.Seen = t
self.After = t
}
func (self *NodeRecord) String() string {
return fmt.Sprintf("<%v>", self.Addr)
}
// persisted node record database ()
type KadDb struct {
Address Address
Nodes [][]*NodeRecord
index map[Address]*NodeRecord
cursors []int
lock sync.RWMutex
purgeInterval time.Duration
initialRetryInterval time.Duration
connRetryExp int
}
func newKadDb(addr Address, params *KadParams) *KadDb {
return &KadDb{
Address: addr,
Nodes: make([][]*NodeRecord, params.MaxProx+1), // overwritten by load
cursors: make([]int, params.MaxProx+1),
index: make(map[Address]*NodeRecord),
purgeInterval: params.PurgeInterval,
initialRetryInterval: params.InitialRetryInterval,
connRetryExp: params.ConnRetryExp,
}
}
func (self *KadDb) findOrCreate(index int, a Address, url string) *NodeRecord {
defer self.lock.Unlock()
self.lock.Lock()
record, found := self.index[a]
if !found {
record = &NodeRecord{
Addr: a,
Url: url,
}
log.Info(fmt.Sprintf("add new record %v to kaddb", record))
// insert in kaddb
self.index[a] = record
self.Nodes[index] = append(self.Nodes[index], record)
} else {
log.Info(fmt.Sprintf("found record %v in kaddb", record))
}
// update last seen time
record.setSeen()
// update with url in case IP/port changes
record.Url = url
return record
}
// add adds node records to kaddb (persisted node record db)
func (self *KadDb) add(nrs []*NodeRecord, proximityBin func(Address) int) {
defer self.lock.Unlock()
self.lock.Lock()
var n int
var nodes []*NodeRecord
for _, node := range nrs {
_, found := self.index[node.Addr]
if !found && node.Addr != self.Address {
node.setSeen()
self.index[node.Addr] = node
index := proximityBin(node.Addr)
dbcursor := self.cursors[index]
nodes = self.Nodes[index]
// this is inefficient for allocation, need to just append then shift
newnodes := make([]*NodeRecord, len(nodes)+1)
copy(newnodes[:], nodes[:dbcursor])
newnodes[dbcursor] = node
copy(newnodes[dbcursor+1:], nodes[dbcursor:])
log.Trace(fmt.Sprintf("new nodes: %v, nodes: %v", newnodes, nodes))
self.Nodes[index] = newnodes
n++
}
}
if n > 0 {
log.Debug(fmt.Sprintf("%d/%d node records (new/known)", n, len(nrs)))
}
}
/*
next return one node record with the highest priority for desired
connection.
This is used to pick candidates for live nodes that are most wanted for
a higly connected low centrality network structure for Swarm which best suits
for a Kademlia-style routing.
* Starting as naive node with empty db, this implements Kademlia bootstrapping
* As a mature node, it fills short lines. All on demand.
The candidate is chosen using the following strategy:
We check for missing online nodes in the buckets for 1 upto Max BucketSize rounds.
On each round we proceed from the low to high proximity order buckets.
If the number of active nodes (=connected peers) is < rounds, then start looking
for a known candidate. To determine if there is a candidate to recommend the
kaddb node record database row corresponding to the bucket is checked.
If the row cursor is on position i, the ith element in the row is chosen.
If the record is scheduled not to be retried before NOW, the next element is taken.
If the record is scheduled to be retried, it is set as checked, scheduled for
checking and is returned. The time of the next check is in X (duration) such that
X = ConnRetryExp * delta where delta is the time past since the last check and
ConnRetryExp is constant obsoletion factor. (Note that when node records are added
from peer messages, they are marked as checked and placed at the cursor, ie.
given priority over older entries). Entries which were checked more than
purgeInterval ago are deleted from the kaddb row. If no candidate is found after
a full round of checking the next bucket up is considered. If no candidate is
found when we reach the maximum-proximity bucket, the next round starts.
node record a is more favoured to b a > b iff a is a passive node (record of
offline past peer)
|proxBin(a)| < |proxBin(b)|
|| (proxBin(a) < proxBin(b) && |proxBin(a)| == |proxBin(b)|)
|| (proxBin(a) == proxBin(b) && lastChecked(a) < lastChecked(b))
The second argument returned names the first missing slot found
*/
func (self *KadDb) findBest(maxBinSize int, binSize func(int) int) (node *NodeRecord, need bool, proxLimit int) {
// return nil, proxLimit indicates that all buckets are filled
defer self.lock.Unlock()
self.lock.Lock()
var interval time.Duration
var found bool
var purge []bool
var delta time.Duration
var cursor int
var count int
var after time.Time
// iterate over columns maximum bucketsize times
for rounds := 1; rounds <= maxBinSize; rounds++ {
ROUND:
// iterate over rows from PO 0 upto MaxProx
for po, dbrow := range self.Nodes {
// if row has rounds connected peers, then take the next
if binSize(po) >= rounds {
continue ROUND
}
if !need {
// set proxlimit to the PO where the first missing slot is found
proxLimit = po
need = true
}
purge = make([]bool, len(dbrow))
// there is a missing slot - finding a node to connect to
// select a node record from the relavant kaddb row (of identical prox order)
ROW:
for cursor = self.cursors[po]; !found && count < len(dbrow); cursor = (cursor + 1) % len(dbrow) {
count++
node = dbrow[cursor]
// skip already connected nodes
if node.node != nil {
log.Debug(fmt.Sprintf("kaddb record %v (PO%03d:%d/%d) already connected", node.Addr, po, cursor, len(dbrow)))
continue ROW
}
// if node is scheduled to connect
if node.After.After(time.Now()) {
log.Debug(fmt.Sprintf("kaddb record %v (PO%03d:%d) skipped. seen at %v (%v ago), scheduled at %v", node.Addr, po, cursor, node.Seen, delta, node.After))
continue ROW
}
delta = time.Since(node.Seen)
if delta < self.initialRetryInterval {
delta = self.initialRetryInterval
}
if delta > self.purgeInterval {
// remove node
purge[cursor] = true
log.Debug(fmt.Sprintf("kaddb record %v (PO%03d:%d) unreachable since %v. Removed", node.Addr, po, cursor, node.Seen))
continue ROW
}
log.Debug(fmt.Sprintf("kaddb record %v (PO%03d:%d) ready to be tried. seen at %v (%v ago), scheduled at %v", node.Addr, po, cursor, node.Seen, delta, node.After))
// scheduling next check
interval = delta * time.Duration(self.connRetryExp)
after = time.Now().Add(interval)
log.Debug(fmt.Sprintf("kaddb record %v (PO%03d:%d) selected as candidate connection %v. seen at %v (%v ago), selectable since %v, retry after %v (in %v)", node.Addr, po, cursor, rounds, node.Seen, delta, node.After, after, interval))
node.After = after
found = true
} // ROW
self.cursors[po] = cursor
self.delete(po, purge)
if found {
return node, need, proxLimit
}
} // ROUND
} // ROUNDS
return nil, need, proxLimit
}
// deletes the noderecords of a kaddb row corresponding to the indexes
// caller must hold the dblock
// the call is unsafe, no index checks
func (self *KadDb) delete(row int, purge []bool) {
var nodes []*NodeRecord
dbrow := self.Nodes[row]
for i, del := range purge {
if i == self.cursors[row] {
//reset cursor
self.cursors[row] = len(nodes)
}
// delete the entry to be purged
if del {
delete(self.index, dbrow[i].Addr)
continue
}
// otherwise append to new list
nodes = append(nodes, dbrow[i])
}
self.Nodes[row] = nodes
}
// save persists kaddb on disk (written to file on path in json format.
func (self *KadDb) save(path string, cb func(*NodeRecord, Node)) error {
defer self.lock.Unlock()
self.lock.Lock()
var n int
for _, b := range self.Nodes {
for _, node := range b {
n++
node.After = time.Now()
node.Seen = time.Now()
if cb != nil {
cb(node, node.node)
}
}
}
data, err := json.MarshalIndent(self, "", " ")
if err != nil {
return err
}
err = os.WriteFile(path, data, os.ModePerm)
if err != nil {
log.Warn(fmt.Sprintf("unable to save kaddb with %v nodes to %v: %v", n, path, err))
} else {
log.Info(fmt.Sprintf("saved kaddb with %v nodes to %v", n, path))
}
return err
}
// Load(path) loads the node record database (kaddb) from file on path.
func (self *KadDb) load(path string, cb func(*NodeRecord, Node) error) (err error) {
defer self.lock.Unlock()
self.lock.Lock()
var data []byte
data, err = os.ReadFile(path)
if err != nil {
return
}
err = json.Unmarshal(data, self)
if err != nil {
return
}
var n int
var purge []bool
for po, b := range self.Nodes {
purge = make([]bool, len(b))
ROW:
for i, node := range b {
if cb != nil {
err = cb(node, node.node)
if err != nil {
purge[i] = true
continue ROW
}
}
n++
if node.After.IsZero() {
node.After = time.Now()
}
self.index[node.Addr] = node
}
self.delete(po, purge)
}
log.Info(fmt.Sprintf("loaded kaddb with %v nodes from %v", n, path))
return
}
// accessor for KAD offline db count
func (self *KadDb) count() int {
defer self.lock.Unlock()
self.lock.Lock()
return len(self.index)
}

454
swarm/network/kademlia/kademlia.go
View file

@ -1,454 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package kademlia
import (
"fmt"
"sort"
"strings"
"sync"
"time"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/metrics"
)
//metrics variables
//For metrics, we want to count how many times peers are added/removed
//at a certain index. Thus we do that with an array of counters with
//entry for each index
var (
bucketAddIndexCount []metrics.Counter
bucketRmIndexCount []metrics.Counter
)
const (
bucketSize = 4
proxBinSize = 2
maxProx = 8
connRetryExp = 2
maxPeers = 100
)
var (
purgeInterval = 42 * time.Hour
initialRetryInterval = 42 * time.Millisecond
maxIdleInterval = 42 * 1000 * time.Millisecond
// maxIdleInterval = 42 * 10 0 * time.Millisecond
)
type KadParams struct {
// adjustable parameters
MaxProx int
ProxBinSize int
BucketSize int
PurgeInterval time.Duration
InitialRetryInterval time.Duration
MaxIdleInterval time.Duration
ConnRetryExp int
}
func NewDefaultKadParams() *KadParams {
return &KadParams{
MaxProx: maxProx,
ProxBinSize: proxBinSize,
BucketSize: bucketSize,
PurgeInterval: purgeInterval,
InitialRetryInterval: initialRetryInterval,
MaxIdleInterval: maxIdleInterval,
ConnRetryExp: connRetryExp,
}
}
// Kademlia is a table of active nodes
type Kademlia struct {
addr Address // immutable baseaddress of the table
*KadParams // Kademlia configuration parameters
proxLimit int // state, the PO of the first row of the most proximate bin
proxSize int // state, the number of peers in the most proximate bin
count int // number of active peers (w live connection)
buckets [][]Node // the actual bins
db *KadDb // kaddb, node record database
lock sync.RWMutex // mutex to access buckets
}
type Node interface {
Addr() Address
Url() string
LastActive() time.Time
Drop()
}
// public constructor
// add is the base address of the table
// params is KadParams configuration
func New(addr Address, params *KadParams) *Kademlia {
buckets := make([][]Node, params.MaxProx+1)
kad := &Kademlia{
addr: addr,
KadParams: params,
buckets: buckets,
db: newKadDb(addr, params),
}
kad.initMetricsVariables()
return kad
}
// accessor for KAD base address
func (self *Kademlia) Addr() Address {
return self.addr
}
// accessor for KAD active node count
func (self *Kademlia) Count() int {
defer self.lock.Unlock()
self.lock.Lock()
return self.count
}
// accessor for KAD active node count
func (self *Kademlia) DBCount() int {
return self.db.count()
}
// On is the entry point called when a new nodes is added
// unsafe in that node is not checked to be already active node (to be called once)
func (self *Kademlia) On(node Node, cb func(*NodeRecord, Node) error) (err error) {
log.Debug(fmt.Sprintf("%v", self))
defer self.lock.Unlock()
self.lock.Lock()
index := self.proximityBin(node.Addr())
record := self.db.findOrCreate(index, node.Addr(), node.Url())
if cb != nil {
err = cb(record, node)
log.Trace(fmt.Sprintf("cb(%v, %v) ->%v", record, node, err))
if err != nil {
return fmt.Errorf("unable to add node %v, callback error: %v", node.Addr(), err)
}
log.Debug(fmt.Sprintf("add node record %v with node %v", record, node))
}
// insert in kademlia table of active nodes
bucket := self.buckets[index]
// if bucket is full insertion replaces the worst node
// TODO: give priority to peers with active traffic
if len(bucket) < self.BucketSize { // >= allows us to add peers beyond the bucketsize limitation
self.buckets[index] = append(bucket, node)
bucketAddIndexCount[index].Inc(1)
log.Debug(fmt.Sprintf("add node %v to table", node))
self.setProxLimit(index, true)
record.node = node
self.count++
return nil
}
// always rotate peers
idle := self.MaxIdleInterval
var pos int
var replaced Node
for i, p := range bucket {
idleInt := time.Since(p.LastActive())
if idleInt > idle {
idle = idleInt
pos = i
replaced = p
}
}
if replaced == nil {
log.Debug(fmt.Sprintf("all peers wanted, PO%03d bucket full", index))
return fmt.Errorf("bucket full")
}
log.Debug(fmt.Sprintf("node %v replaced by %v (idle for %v > %v)", replaced, node, idle, self.MaxIdleInterval))
replaced.Drop()
// actually replace in the row. When off(node) is called, the peer is no longer in the row
bucket[pos] = node
// there is no change in bucket cardinalities so no prox limit adjustment is needed
record.node = node
self.count++
return nil
}
// Off is the called when a node is taken offline (from the protocol main loop exit)
func (self *Kademlia) Off(node Node, cb func(*NodeRecord, Node)) (err error) {
self.lock.Lock()
defer self.lock.Unlock()
index := self.proximityBin(node.Addr())
bucketRmIndexCount[index].Inc(1)
bucket := self.buckets[index]
for i := 0; i < len(bucket); i++ {
if node.Addr() == bucket[i].Addr() {
self.buckets[index] = append(bucket[:i], bucket[(i+1):]...)
self.setProxLimit(index, false)
break
}
}
record := self.db.index[node.Addr()]
// callback on remove
if cb != nil {
cb(record, record.node)
}
record.node = nil
self.count--
log.Debug(fmt.Sprintf("remove node %v from table, population now is %v", node, self.count))
return
}
// proxLimit is dynamically adjusted so that
// 1) there is no empty buckets in bin < proxLimit and
// 2) the sum of all items are the minimum possible but higher than ProxBinSize
// adjust Prox (proxLimit and proxSize after an insertion/removal of nodes)
// caller holds the lock
func (self *Kademlia) setProxLimit(r int, on bool) {
// if the change is outside the core (PO lower)
// and the change does not leave a bucket empty then
// no adjustment needed
if r < self.proxLimit && len(self.buckets[r]) > 0 {
return
}
// if on=a node was added, then r must be within prox limit so increment cardinality
if on {
self.proxSize++
curr := len(self.buckets[self.proxLimit])
// if now core is big enough without the furthest bucket, then contract
// this can result in more than one bucket change
for self.proxSize >= self.ProxBinSize+curr && curr > 0 {
self.proxSize -= curr
self.proxLimit++
curr = len(self.buckets[self.proxLimit])
log.Trace(fmt.Sprintf("proxbin contraction (size: %v, limit: %v, bin: %v)", self.proxSize, self.proxLimit, r))
}
return
}
// otherwise
if r >= self.proxLimit {
self.proxSize--
}
// expand core by lowering prox limit until hit zero or cover the empty bucket or reached target cardinality
for (self.proxSize < self.ProxBinSize || r < self.proxLimit) &&
self.proxLimit > 0 {
//
self.proxLimit--
self.proxSize += len(self.buckets[self.proxLimit])
log.Trace(fmt.Sprintf("proxbin expansion (size: %v, limit: %v, bin: %v)", self.proxSize, self.proxLimit, r))
}
}
/*
returns the list of nodes belonging to the same proximity bin
as the target. The most proximate bin will be the union of the bins between
proxLimit and MaxProx.
*/
func (self *Kademlia) FindClosest(target Address, max int) []Node {
self.lock.Lock()
defer self.lock.Unlock()
r := nodesByDistance{
target: target,
}
po := self.proximityBin(target)
index := po
step := 1
log.Trace(fmt.Sprintf("serving %v nodes at %v (PO%02d)", max, index, po))
// if max is set to 0, just want a full bucket, dynamic number
min := max
// set limit to max
limit := max
if max == 0 {
min = 1
limit = maxPeers
}
var n int
for index >= 0 {
// add entire bucket
for _, p := range self.buckets[index] {
r.push(p, limit)
n++
}
// terminate if index reached the bottom or enough peers > min
log.Trace(fmt.Sprintf("add %v -> %v (PO%02d, PO%03d)", len(self.buckets[index]), n, index, po))
if n >= min && (step < 0 || max == 0) {
break
}
// reach top most non-empty PO bucket, turn around
if index == self.MaxProx {
index = po
step = -1
}
index += step
}
log.Trace(fmt.Sprintf("serve %d (<=%d) nodes for target lookup %v (PO%03d)", n, max, target, po))
return r.nodes
}
func (self *Kademlia) Suggest() (*NodeRecord, bool, int) {
defer self.lock.RUnlock()
self.lock.RLock()
return self.db.findBest(self.BucketSize, func(i int) int { return len(self.buckets[i]) })
}
// adds node records to kaddb (persisted node record db)
func (self *Kademlia) Add(nrs []*NodeRecord) {
self.db.add(nrs, self.proximityBin)
}
// nodesByDistance is a list of nodes, ordered by distance to target.
type nodesByDistance struct {
nodes []Node
target Address
}
func sortedByDistanceTo(target Address, slice []Node) bool {
var last Address
for i, node := range slice {
if i > 0 {
if target.ProxCmp(node.Addr(), last) < 0 {
return false
}
}
last = node.Addr()
}
return true
}
// push(node, max) adds the given node to the list, keeping the total size
// below max elements.
func (h *nodesByDistance) push(node Node, max int) {
// returns the firt index ix such that func(i) returns true
ix := sort.Search(len(h.nodes), func(i int) bool {
return h.target.ProxCmp(h.nodes[i].Addr(), node.Addr()) >= 0
})
if len(h.nodes) < max {
h.nodes = append(h.nodes, node)
}
if ix < len(h.nodes) {
copy(h.nodes[ix+1:], h.nodes[ix:])
h.nodes[ix] = node
}
}
/*
Taking the proximity order relative to a fix point x classifies the points in
the space (n byte long byte sequences) into bins. Items in each are at
most half as distant from x as items in the previous bin. Given a sample of
uniformly distributed items (a hash function over arbitrary sequence) the
proximity scale maps onto series of subsets with cardinalities on a negative
exponential scale.
It also has the property that any two item belonging to the same bin are at
most half as distant from each other as they are from x.
If we think of random sample of items in the bins as connections in a network of interconnected nodes than relative proximity can serve as the basis for local
decisions for graph traversal where the task is to find a route between two
points. Since in every hop, the finite distance halves, there is
a guaranteed constant maximum limit on the number of hops needed to reach one
node from the other.
*/
func (self *Kademlia) proximityBin(other Address) (ret int) {
ret = proximity(self.addr, other)
if ret > self.MaxProx {
ret = self.MaxProx
}
return
}
// provides keyrange for chunk db iteration
func (self *Kademlia) KeyRange(other Address) (start, stop Address) {
defer self.lock.RUnlock()
self.lock.RLock()
return KeyRange(self.addr, other, self.proxLimit)
}
// save persists kaddb on disk (written to file on path in json format.
func (self *Kademlia) Save(path string, cb func(*NodeRecord, Node)) error {
return self.db.save(path, cb)
}
// Load(path) loads the node record database (kaddb) from file on path.
func (self *Kademlia) Load(path string, cb func(*NodeRecord, Node) error) (err error) {
return self.db.load(path, cb)
}
// kademlia table + kaddb table displayed with ascii
func (self *Kademlia) String() string {
defer self.lock.RUnlock()
self.lock.RLock()
defer self.db.lock.RUnlock()
self.db.lock.RLock()
var rows []string
rows = append(rows, "=========================================================================")
rows = append(rows, fmt.Sprintf("%v KΛÐΞMLIΛ hive: queen's address: %v", time.Now().UTC().Format(time.UnixDate), self.addr.String()[:6]))
rows = append(rows, fmt.Sprintf("population: %d (%d), proxLimit: %d, proxSize: %d", self.count, len(self.db.index), self.proxLimit, self.proxSize))
rows = append(rows, fmt.Sprintf("MaxProx: %d, ProxBinSize: %d, BucketSize: %d", self.MaxProx, self.ProxBinSize, self.BucketSize))
for i, bucket := range self.buckets {
if i == self.proxLimit {
rows = append(rows, fmt.Sprintf("============ PROX LIMIT: %d ==========================================", i))
}
row := []string{fmt.Sprintf("%03d", i), fmt.Sprintf("%2d", len(bucket))}
var k int
c := self.db.cursors[i]
for ; k < len(bucket); k++ {
p := bucket[(c+k)%len(bucket)]
row = append(row, p.Addr().String()[:6])
if k == 4 {
break
}
}
for ; k < 4; k++ {
row = append(row, " ")
}
row = append(row, fmt.Sprintf("| %2d %2d", len(self.db.Nodes[i]), self.db.cursors[i]))
for j, p := range self.db.Nodes[i] {
row = append(row, p.Addr.String()[:6])
if j == 3 {
break
}
}
rows = append(rows, strings.Join(row, " "))
if i == self.MaxProx {
}
}
rows = append(rows, "=========================================================================")
return strings.Join(rows, "\n")
}
//We have to build up the array of counters for each index
func (self *Kademlia) initMetricsVariables() {
//create the arrays
bucketAddIndexCount = make([]metrics.Counter, self.MaxProx+1)
bucketRmIndexCount = make([]metrics.Counter, self.MaxProx+1)
//at each index create a metrics counter
for i := 0; i < (self.KadParams.MaxProx + 1); i++ {
bucketAddIndexCount[i] = metrics.NewRegisteredCounter(fmt.Sprintf("network.kademlia.bucket.add.%d.index", i), nil)
bucketRmIndexCount[i] = metrics.NewRegisteredCounter(fmt.Sprintf("network.kademlia.bucket.rm.%d.index", i), nil)
}
}

392
swarm/network/kademlia/kademlia_test.go
View file

@ -1,392 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package kademlia
import (
"fmt"
"math"
"math/rand"
"os"
"path/filepath"
"reflect"
"testing"
"testing/quick"
"time"
)
var (
quickrand = rand.New(rand.NewSource(time.Now().Unix()))
quickcfgFindClosest = &quick.Config{MaxCount: 50, Rand: quickrand}
quickcfgBootStrap = &quick.Config{MaxCount: 100, Rand: quickrand}
)
type testNode struct {
addr Address
}
func (n *testNode) String() string {
return fmt.Sprintf("%x", n.addr[:])
}
func (n *testNode) Addr() Address {
return n.addr
}
func (n *testNode) Drop() {
}
func (n *testNode) Url() string {
return ""
}
func (n *testNode) LastActive() time.Time {
return time.Now()
}
func TestOn(t *testing.T) {
addr, ok1 := gen(Address{}, quickrand).(Address)
other, ok2 := gen(Address{}, quickrand).(Address)
if !ok1 || !ok2 {
t.Errorf("oops")
}
kad := New(addr, NewDefaultKadParams())
err := kad.On(&testNode{addr: other}, nil)
_ = err
}
func TestBootstrap(t *testing.T) {
test := func(test *bootstrapTest) bool {
// for any node kad.le, Target and N
params := NewDefaultKadParams()
params.MaxProx = test.MaxProx
params.BucketSize = test.BucketSize
params.ProxBinSize = test.BucketSize
kad := New(test.Self, params)
var err error
for p := 0; p < 9; p++ {
var nrs []*NodeRecord
n := math.Pow(float64(2), float64(7-p))
for i := 0; i < int(n); i++ {
addr := RandomAddressAt(test.Self, p)
nrs = append(nrs, &NodeRecord{
Addr: addr,
})
}
kad.Add(nrs)
}
node := &testNode{test.Self}
n := 0
for n < 100 {
err = kad.On(node, nil)
if err != nil {
t.Fatalf("backend not accepting node: %v", err)
}
record, need, _ := kad.Suggest()
if !need {
break
}
n++
if record == nil {
continue
}
node = &testNode{record.Addr}
}
exp := test.BucketSize * (test.MaxProx + 1)
if kad.Count() != exp {
t.Errorf("incorrect number of peers, expected %d, got %d\n%v", exp, kad.Count(), kad)
return false
}
return true
}
if err := quick.Check(test, quickcfgBootStrap); err != nil {
t.Error(err)
}
}
func TestFindClosest(t *testing.T) {
test := func(test *FindClosestTest) bool {
// for any node kad.le, Target and N
params := NewDefaultKadParams()
params.MaxProx = 7
kad := New(test.Self, params)
var err error
for _, node := range test.All {
err = kad.On(node, nil)
if err != nil && err.Error() != "bucket full" {
t.Fatalf("backend not accepting node: %v", err)
}
}
if len(test.All) == 0 || test.N == 0 {
return true
}
nodes := kad.FindClosest(test.Target, test.N)
// check that the number of results is min(N, kad.len)
wantN := test.N
if tlen := kad.Count(); tlen < test.N {
wantN = tlen
}
if len(nodes) != wantN {
t.Errorf("wrong number of nodes: got %d, want %d", len(nodes), wantN)
return false
}
if hasDuplicates(nodes) {
t.Errorf("result contains duplicates")
return false
}
if !sortedByDistanceTo(test.Target, nodes) {
t.Errorf("result is not sorted by distance to target")
return false
}
// check that the result nodes have minimum distance to target.
farthestResult := nodes[len(nodes)-1].Addr()
for i, b := range kad.buckets {
for j, n := range b {
if contains(nodes, n.Addr()) {
continue // don't run the check below for nodes in result
}
if test.Target.ProxCmp(n.Addr(), farthestResult) < 0 {
_ = i * j
t.Errorf("kad.le contains node that is closer to target but it's not in result")
return false
}
}
}
return true
}
if err := quick.Check(test, quickcfgFindClosest); err != nil {
t.Error(err)
}
}
type proxTest struct {
add bool
index int
addr Address
}
var (
addresses []Address
)
func TestProxAdjust(t *testing.T) {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
self := gen(Address{}, r).(Address)
params := NewDefaultKadParams()
params.MaxProx = 7
kad := New(self, params)
var err error
for i := 0; i < 100; i++ {
a := gen(Address{}, r).(Address)
addresses = append(addresses, a)
err = kad.On(&testNode{addr: a}, nil)
if err != nil && err.Error() != "bucket full" {
t.Fatalf("backend not accepting node: %v", err)
}
if !kad.proxCheck(t) {
return
}
}
test := func(test *proxTest) bool {
node := &testNode{test.addr}
if test.add {
kad.On(node, nil)
} else {
kad.Off(node, nil)
}
return kad.proxCheck(t)
}
if err := quick.Check(test, quickcfgFindClosest); err != nil {
t.Error(err)
}
}
func TestSaveLoad(t *testing.T) {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
addresses := gen([]Address{}, r).([]Address)
self := RandomAddress()
params := NewDefaultKadParams()
params.MaxProx = 7
kad := New(self, params)
var err error
for _, a := range addresses {
err = kad.On(&testNode{addr: a}, nil)
if err != nil && err.Error() != "bucket full" {
t.Fatalf("backend not accepting node: %v", err)
}
}
nodes := kad.FindClosest(self, 100)
path := filepath.Join(os.TempDir(), "bzz-kad-test-save-load.peers")
err = kad.Save(path, nil)
if err != nil && err.Error() != "bucket full" {
t.Fatalf("unepected error saving kaddb: %v", err)
}
kad = New(self, params)
err = kad.Load(path, nil)
if err != nil && err.Error() != "bucket full" {
t.Fatalf("unepected error loading kaddb: %v", err)
}
for _, b := range kad.db.Nodes {
for _, node := range b {
err = kad.On(&testNode{node.Addr}, nil)
if err != nil && err.Error() != "bucket full" {
t.Fatalf("backend not accepting node: %v", err)
}
}
}
loadednodes := kad.FindClosest(self, 100)
for i, node := range loadednodes {
if nodes[i].Addr() != node.Addr() {
t.Errorf("node mismatch at %d/%d: %v != %v", i, len(nodes), nodes[i].Addr(), node.Addr())
}
}
}
func (self *Kademlia) proxCheck(t *testing.T) bool {
var sum int
for i, b := range self.buckets {
l := len(b)
// if we are in the high prox multibucket
if i >= self.proxLimit {
sum += l
} else if l == 0 {
t.Errorf("bucket %d empty, yet proxLimit is %d\n%v", len(b), self.proxLimit, self)
return false
}
}
// check if merged high prox bucket does not exceed size
if sum > 0 {
if sum != self.proxSize {
t.Errorf("proxSize incorrect, expected %v, got %v", sum, self.proxSize)
return false
}
last := len(self.buckets[self.proxLimit])
if last > 0 && sum >= self.ProxBinSize+last {
t.Errorf("proxLimit %v incorrect, redundant non-empty bucket %d added to proxBin with %v (target %v)\n%v", self.proxLimit, last, sum-last, self.ProxBinSize, self)
return false
}
if self.proxLimit > 0 && sum < self.ProxBinSize {
t.Errorf("proxLimit %v incorrect. proxSize %v is less than target %v, yet there is more peers", self.proxLimit, sum, self.ProxBinSize)
return false
}
}
return true
}
type bootstrapTest struct {
MaxProx int
BucketSize int
Self Address
}
func (*bootstrapTest) Generate(rand *rand.Rand, size int) reflect.Value {
t := &bootstrapTest{
Self: gen(Address{}, rand).(Address),
MaxProx: 5 + rand.Intn(2),
BucketSize: rand.Intn(3) + 1,
}
return reflect.ValueOf(t)
}
type FindClosestTest struct {
Self Address
Target Address
All []Node
N int
}
func (c FindClosestTest) String() string {
return fmt.Sprintf("A: %064x\nT: %064x\n(%d)\n", c.Self[:], c.Target[:], c.N)
}
func (*FindClosestTest) Generate(rand *rand.Rand, size int) reflect.Value {
t := &FindClosestTest{
Self: gen(Address{}, rand).(Address),
Target: gen(Address{}, rand).(Address),
N: rand.Intn(bucketSize),
}
for _, a := range gen([]Address{}, rand).([]Address) {
t.All = append(t.All, &testNode{addr: a})
}
return reflect.ValueOf(t)
}
func (*proxTest) Generate(rand *rand.Rand, size int) reflect.Value {
var add bool
if rand.Intn(1) == 0 {
add = true
}
var t *proxTest
if add {
t = &proxTest{
addr: gen(Address{}, rand).(Address),
add: add,
}
} else {
t = &proxTest{
index: rand.Intn(len(addresses)),
add: add,
}
}
return reflect.ValueOf(t)
}
func hasDuplicates(slice []Node) bool {
seen := make(map[Address]bool)
for _, node := range slice {
if seen[node.Addr()] {
return true
}
seen[node.Addr()] = true
}
return false
}
func contains(nodes []Node, addr Address) bool {
for _, n := range nodes {
if n.Addr() == addr {
return true
}
}
return false
}
// gen wraps quick.Value so it's easier to use.
// it generates a random value of the given value's type.
func gen(typ interface{}, rand *rand.Rand) interface{} {
v, ok := quick.Value(reflect.TypeOf(typ), rand)
if !ok {
panic(fmt.Sprintf("couldn't generate random value of type %T", typ))
}
return v.Interface()
}

308
swarm/network/messages.go
View file

@ -1,308 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package network
import (
"fmt"
"net"
"time"
"github.com/XinFinOrg/XDPoSChain/contracts/chequebook"
"github.com/XinFinOrg/XDPoSChain/p2p/discover"
"github.com/XinFinOrg/XDPoSChain/swarm/network/kademlia"
"github.com/XinFinOrg/XDPoSChain/swarm/services/swap"
"github.com/XinFinOrg/XDPoSChain/swarm/storage"
)
/*
BZZ protocol Message Types and Message Data Types
*/
// bzz protocol message codes
const (
statusMsg = iota // 0x01
storeRequestMsg // 0x02
retrieveRequestMsg // 0x03
peersMsg // 0x04
syncRequestMsg // 0x05
deliveryRequestMsg // 0x06
unsyncedKeysMsg // 0x07
paymentMsg // 0x08
)
/*
Handshake
* Version: 8 byte integer version of the protocol
* ID: arbitrary byte sequence client identifier human readable
* Addr: the address advertised by the node, format similar to DEVp2p wire protocol
* Swap: info for the swarm accounting protocol
* NetworkID: 8 byte integer network identifier
* Caps: swarm-specific capabilities, format identical to devp2p
* SyncState: syncronisation state (db iterator key and address space etc) persisted about the peer
*/
type statusMsgData struct {
Version uint64
ID string
Addr *peerAddr
Swap *swap.SwapProfile
NetworkId uint64
}
func (self *statusMsgData) String() string {
return fmt.Sprintf("Status: Version: %v, ID: %v, Addr: %v, Swap: %v, NetworkId: %v", self.Version, self.ID, self.Addr, self.Swap, self.NetworkId)
}
/*
store requests are forwarded to the peers in their kademlia proximity bin
if they are distant
if they are within our storage radius or have any incentive to store it
then attach your nodeID to the metadata
if the storage request is sufficiently close (within our proxLimit, i. e., the
last row of the routing table)
*/
type storeRequestMsgData struct {
Key storage.Key // hash of datasize | data
SData []byte // the actual chunk Data
// optional
Id uint64 // request ID. if delivery, the ID is retrieve request ID
requestTimeout *time.Time // expiry for forwarding - [not serialised][not currently used]
storageTimeout *time.Time // expiry of content - [not serialised][not currently used]
from *peer // [not serialised] protocol registers the requester
}
func (self storeRequestMsgData) String() string {
var from string
if self.from == nil {
from = "self"
} else {
from = self.from.Addr().String()
}
end := len(self.SData)
if len(self.SData) > 10 {
end = 10
}
return fmt.Sprintf("from: %v, Key: %v; ID: %v, requestTimeout: %v, storageTimeout: %v, SData %x", from, self.Key, self.Id, self.requestTimeout, self.storageTimeout, self.SData[:end])
}
/*
Retrieve request
Timeout in milliseconds. Note that zero timeout retrieval requests do not request forwarding, but prompt for a peers message response. therefore they serve also
as messages to retrieve peers.
MaxSize specifies the maximum size that the peer will accept. This is useful in
particular if we allow storage and delivery of multichunk payload representing
the entire or partial subtree unfolding from the requested root key.
So when only interested in limited part of a stream (infinite trees) or only
testing chunk availability etc etc, we can indicate it by limiting the size here.
Request ID can be newly generated or kept from the request originator.
If request ID Is missing or zero, the request is handled as a lookup only
prompting a peers response but not launching a search. Lookup requests are meant
to be used to bootstrap kademlia tables.
In the special case that the key is the zero value as well, the remote peer's
address is assumed (the message is to be handled as a self lookup request).
The response is a PeersMsg with the peers in the kademlia proximity bin
corresponding to the address.
*/
type retrieveRequestMsgData struct {
Key storage.Key // target Key address of chunk to be retrieved
Id uint64 // request id, request is a lookup if missing or zero
MaxSize uint64 // maximum size of delivery accepted
MaxPeers uint64 // maximum number of peers returned
Timeout uint64 // the longest time we are expecting a response
timeout *time.Time // [not serialied]
from *peer //
}
func (self *retrieveRequestMsgData) String() string {
var from string
if self.from == nil {
from = "ourselves"
} else {
from = self.from.Addr().String()
}
var target []byte
if len(self.Key) > 3 {
target = self.Key[:4]
}
return fmt.Sprintf("from: %v, Key: %x; ID: %v, MaxSize: %v, MaxPeers: %d", from, target, self.Id, self.MaxSize, self.MaxPeers)
}
// lookups are encoded by missing request ID
func (self *retrieveRequestMsgData) isLookup() bool {
return self.Id == 0
}
// sets timeout fields
func (self *retrieveRequestMsgData) setTimeout(t *time.Time) {
self.timeout = t
if t != nil {
self.Timeout = uint64(t.UnixNano())
} else {
self.Timeout = 0
}
}
func (self *retrieveRequestMsgData) getTimeout() (t *time.Time) {
if self.Timeout > 0 && self.timeout == nil {
timeout := time.Unix(int64(self.Timeout), 0)
t = &timeout
self.timeout = t
}
return
}
// peerAddr is sent in StatusMsg as part of the handshake
type peerAddr struct {
IP net.IP
Port uint16
ID []byte // the 64 byte NodeID (ECDSA Public Key)
Addr kademlia.Address
}
// peerAddr pretty prints as enode
func (self *peerAddr) String() string {
var nodeid discover.NodeID
copy(nodeid[:], self.ID)
return discover.NewNode(nodeid, self.IP, 0, self.Port).String()
}
/*
peers Msg is one response to retrieval; it is always encouraged after a retrieval
request to respond with a list of peers in the same kademlia proximity bin.
The encoding of a peer is identical to that in the devp2p base protocol peers
messages: [IP, Port, NodeID]
note that a node's DPA address is not the NodeID but the hash of the NodeID.
Timeout serves to indicate whether the responder is forwarding the query within
the timeout or not.
NodeID serves as the owner of payment contracts and signer of proofs of transfer.
The Key is the target (if response to a retrieval request) or missing (zero value)
peers address (hash of NodeID) if retrieval request was a self lookup.
Peers message is requested by retrieval requests with a missing or zero value request ID
*/
type peersMsgData struct {
Peers []*peerAddr //
Timeout uint64 //
timeout *time.Time // indicate whether responder is expected to deliver content
Key storage.Key // present if a response to a retrieval request
Id uint64 // present if a response to a retrieval request
from *peer
}
// peers msg pretty printer
func (self *peersMsgData) String() string {
var from string
if self.from == nil {
from = "ourselves"
} else {
from = self.from.Addr().String()
}
var target []byte
if len(self.Key) > 3 {
target = self.Key[:4]
}
return fmt.Sprintf("from: %v, Key: %x; ID: %v, Peers: %v", from, target, self.Id, self.Peers)
}
func (self *peersMsgData) setTimeout(t *time.Time) {
self.timeout = t
if t != nil {
self.Timeout = uint64(t.UnixNano())
} else {
self.Timeout = 0
}
}
/*
syncRequest
is sent after the handshake to initiate syncing
the syncState of the remote node is persisted in kaddb and set on the
peer/protocol instance when the node is registered by hive as online{
*/
type syncRequestMsgData struct {
SyncState *syncState `rlp:"nil"`
}
func (self *syncRequestMsgData) String() string {
return fmt.Sprintf("%v", self.SyncState)
}
/*
deliveryRequest
is sent once a batch of sync keys is filtered. The ones not found are
sent as a list of syncReuest (hash, priority) in the Deliver field.
When the source receives the sync request it continues to iterate
and fetch at most N items as yet unsynced.
At the same time responds with deliveries of the items.
*/
type deliveryRequestMsgData struct {
Deliver []*syncRequest
}
func (self *deliveryRequestMsgData) String() string {
return fmt.Sprintf("sync request for new chunks\ndelivery request for %v chunks", len(self.Deliver))
}
/*
unsyncedKeys
is sent first after the handshake if SyncState iterator brings up hundreds, thousands?
and subsequently sent as a response to deliveryRequestMsgData.
Syncing is the iterative process of exchanging unsyncedKeys and deliveryRequestMsgs
both ways.
State contains the sync state sent by the source. When the source receives the
sync state it continues to iterate and fetch at most N items as yet unsynced.
At the same time responds with deliveries of the items.
*/
type unsyncedKeysMsgData struct {
Unsynced []*syncRequest
State *syncState
}
func (self *unsyncedKeysMsgData) String() string {
return fmt.Sprintf("sync: keys of %d new chunks (state %v) => synced: %v", len(self.Unsynced), self.State, self.State.Synced)
}
/*
payment
is sent when the swap balance is tilted in favour of the remote peer
and in absolute units exceeds the PayAt parameter in the remote peer's profile
*/
type paymentMsgData struct {
Units uint // units actually paid for (checked against amount by swap)
Promise *chequebook.Cheque // payment with cheque
}
func (self *paymentMsgData) String() string {
return fmt.Sprintf("payment for %d units: %v", self.Units, self.Promise)
}

534
swarm/network/protocol.go
View file

@ -1,534 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package network
/*
bzz implements the swarm wire protocol [bzz] (sister of eth and shh)
the protocol instance is launched on each peer by the network layer if the
bzz protocol handler is registered on the p2p server.
The bzz protocol component speaks the bzz protocol
* handle the protocol handshake
* register peers in the KΛÐΞMLIΛ table via the hive logistic manager
* dispatch to hive for handling the DHT logic
* encode and decode requests for storage and retrieval
* handle sync protocol messages via the syncer
* talks the SWAP payment protocol (swap accounting is done within NetStore)
*/
import (
"errors"
"fmt"
"net"
"strconv"
"time"
"github.com/XinFinOrg/XDPoSChain/contracts/chequebook"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/metrics"
"github.com/XinFinOrg/XDPoSChain/p2p"
bzzswap "github.com/XinFinOrg/XDPoSChain/swarm/services/swap"
"github.com/XinFinOrg/XDPoSChain/swarm/services/swap/swap"
"github.com/XinFinOrg/XDPoSChain/swarm/storage"
)
//metrics variables
var (
storeRequestMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.storerequest.count", nil)
retrieveRequestMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.retrieverequest.count", nil)
peersMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.peers.count", nil)
syncRequestMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.syncrequest.count", nil)
unsyncedKeysMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.unsyncedkeys.count", nil)
deliverRequestMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.deliverrequest.count", nil)
paymentMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.payment.count", nil)
invalidMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.invalid.count", nil)
handleStatusMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.handlestatus.count", nil)
)
const (
Version = 0
ProtocolLength = uint64(8)
ProtocolMaxMsgSize = 10 * 1024 * 1024
NetworkId = 3
)
// bzz represents the swarm wire protocol
// an instance is running on each peer
type bzz struct {
storage StorageHandler // handler storage/retrieval related requests coming via the bzz wire protocol
hive *Hive // the logistic manager, peerPool, routing service and peer handler
dbAccess *DbAccess // access to db storage counter and iterator for syncing
requestDb *storage.LDBDatabase // db to persist backlog of deliveries to aid syncing
remoteAddr *peerAddr // remote peers address
peer *p2p.Peer // the p2p peer object
rw p2p.MsgReadWriter // messageReadWriter to send messages to
backend chequebook.Backend
lastActive time.Time
NetworkId uint64
swap *swap.Swap // swap instance for the peer connection
swapParams *bzzswap.SwapParams // swap settings both local and remote
swapEnabled bool // flag to enable SWAP (will be set via Caps in handshake)
syncEnabled bool // flag to enable SYNC (will be set via Caps in handshake)
syncer *syncer // syncer instance for the peer connection
syncParams *SyncParams // syncer params
syncState *syncState // outgoing syncronisation state (contains reference to remote peers db counter)
}
// interface type for handler of storage/retrieval related requests coming
// via the bzz wire protocol
// messages: UnsyncedKeys, DeliveryRequest, StoreRequest, RetrieveRequest
type StorageHandler interface {
HandleUnsyncedKeysMsg(req *unsyncedKeysMsgData, p *peer) error
HandleDeliveryRequestMsg(req *deliveryRequestMsgData, p *peer) error
HandleStoreRequestMsg(req *storeRequestMsgData, p *peer)
HandleRetrieveRequestMsg(req *retrieveRequestMsgData, p *peer)
}
/*
main entrypoint, wrappers starting a server that will run the bzz protocol
use this constructor to attach the protocol ("class") to server caps
This is done by node.Node#Register(func(node.ServiceContext) (Service, error))
Service implements Protocols() which is an array of protocol constructors
at node startup the protocols are initialised
the Dev p2p layer then calls Run(p *p2p.Peer, rw p2p.MsgReadWriter) error
on each peer connection
The Run function of the Bzz protocol class creates a bzz instance
which will represent the peer for the swarm hive and all peer-aware components
*/
func Bzz(cloud StorageHandler, backend chequebook.Backend, hive *Hive, dbaccess *DbAccess, sp *bzzswap.SwapParams, sy *SyncParams, networkId uint64) (p2p.Protocol, error) {
// a single global request db is created for all peer connections
// this is to persist delivery backlog and aid syncronisation
requestDb, err := storage.NewLDBDatabase(sy.RequestDbPath)
if err != nil {
return p2p.Protocol{}, fmt.Errorf("error setting up request db: %v", err)
}
if networkId == 0 {
networkId = NetworkId
}
return p2p.Protocol{
Name: "bzz",
Version: Version,
Length: ProtocolLength,
Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error {
return run(requestDb, cloud, backend, hive, dbaccess, sp, sy, networkId, p, rw)
},
}, nil
}
/*
the main protocol loop that
* does the handshake by exchanging statusMsg
* if peer is valid and accepted, registers with the hive
* then enters into a forever loop handling incoming messages
* storage and retrieval related queries coming via bzz are dispatched to StorageHandler
* peer-related messages are dispatched to the hive
* payment related messages are relayed to SWAP service
* on disconnect, unregister the peer in the hive (note RemovePeer in the post-disconnect hook)
* whenever the loop terminates, the peer will disconnect with Subprotocol error
* whenever handlers return an error the loop terminates
*/
func run(requestDb *storage.LDBDatabase, depo StorageHandler, backend chequebook.Backend, hive *Hive, dbaccess *DbAccess, sp *bzzswap.SwapParams, sy *SyncParams, networkId uint64, p *p2p.Peer, rw p2p.MsgReadWriter) (err error) {
self := &bzz{
storage: depo,
backend: backend,
hive: hive,
dbAccess: dbaccess,
requestDb: requestDb,
peer: p,
rw: rw,
swapParams: sp,
syncParams: sy,
swapEnabled: hive.swapEnabled,
syncEnabled: true,
NetworkId: networkId,
}
// handle handshake
err = self.handleStatus()
if err != nil {
return err
}
defer func() {
// if the handler loop exits, the peer is disconnecting
// deregister the peer in the hive
self.hive.removePeer(&peer{bzz: self})
if self.syncer != nil {
self.syncer.stop() // quits request db and delivery loops, save requests
}
if self.swap != nil {
self.swap.Stop() // quits chequebox autocash etc
}
}()
// the main forever loop that handles incoming requests
for {
if self.hive.blockRead {
log.Warn(fmt.Sprintf("Cannot read network"))
time.Sleep(100 * time.Millisecond)
continue
}
err = self.handle()
if err != nil {
return
}
}
}
// TODO: may need to implement protocol drop only? don't want to kick off the peer
// if they are useful for other protocols
func (self *bzz) Drop() {
self.peer.Disconnect(p2p.DiscSubprotocolError)
}
// one cycle of the main forever loop that handles and dispatches incoming messages
func (self *bzz) handle() error {
msg, err := self.rw.ReadMsg()
log.Debug(fmt.Sprintf("<- %v", msg))
if err != nil {
return err
}
if msg.Size > ProtocolMaxMsgSize {
return fmt.Errorf("message too long: %v > %v", msg.Size, ProtocolMaxMsgSize)
}
// make sure that the payload has been fully consumed
defer msg.Discard()
switch msg.Code {
case statusMsg:
// no extra status message allowed. The one needed already handled by
// handleStatus
log.Debug(fmt.Sprintf("Status message: %v", msg))
return errors.New("extra status message")
case storeRequestMsg:
// store requests are dispatched to netStore
storeRequestMsgCounter.Inc(1)
var req storeRequestMsgData
if err := msg.Decode(&req); err != nil {
return fmt.Errorf("<- %v: %v", msg, err)
}
if n := len(req.SData); n < 9 {
return fmt.Errorf("<- %v: Data too short (%v)", msg, n)
}
// last Active time is set only when receiving chunks
self.lastActive = time.Now()
log.Trace(fmt.Sprintf("incoming store request: %s", req.String()))
// swap accounting is done within forwarding
self.storage.HandleStoreRequestMsg(&req, &peer{bzz: self})
case retrieveRequestMsg:
// retrieve Requests are dispatched to netStore
retrieveRequestMsgCounter.Inc(1)
var req retrieveRequestMsgData
if err := msg.Decode(&req); err != nil {
return fmt.Errorf("<- %v: %v", msg, err)
}
req.from = &peer{bzz: self}
// if request is lookup and not to be delivered
if req.isLookup() {
log.Trace(fmt.Sprintf("self lookup for %v: responding with peers only...", req.from))
} else if req.Key == nil {
return fmt.Errorf("protocol handler: req.Key == nil || req.Timeout == nil")
} else {
// swap accounting is done within netStore
self.storage.HandleRetrieveRequestMsg(&req, &peer{bzz: self})
}
// direct response with peers, TODO: sort this out
self.hive.peers(&req)
case peersMsg:
// response to lookups and immediate response to retrieve requests
// dispatches new peer data to the hive that adds them to KADDB
peersMsgCounter.Inc(1)
var req peersMsgData
if err := msg.Decode(&req); err != nil {
return fmt.Errorf("<- %v: %v", msg, err)
}
req.from = &peer{bzz: self}
log.Trace(fmt.Sprintf("<- peer addresses: %v", req))
self.hive.HandlePeersMsg(&req, &peer{bzz: self})
case syncRequestMsg:
syncRequestMsgCounter.Inc(1)
var req syncRequestMsgData
if err := msg.Decode(&req); err != nil {
return fmt.Errorf("<- %v: %v", msg, err)
}
log.Debug(fmt.Sprintf("<- sync request: %v", req))
self.lastActive = time.Now()
self.sync(req.SyncState)
case unsyncedKeysMsg:
// coming from parent node offering
unsyncedKeysMsgCounter.Inc(1)
var req unsyncedKeysMsgData
if err := msg.Decode(&req); err != nil {
return fmt.Errorf("<- %v: %v", msg, err)
}
log.Debug(fmt.Sprintf("<- unsynced keys : %s", req.String()))
err := self.storage.HandleUnsyncedKeysMsg(&req, &peer{bzz: self})
self.lastActive = time.Now()
if err != nil {
return fmt.Errorf("<- %v: %v", msg, err)
}
case deliveryRequestMsg:
// response to syncKeysMsg hashes filtered not existing in db
// also relays the last synced state to the source
deliverRequestMsgCounter.Inc(1)
var req deliveryRequestMsgData
if err := msg.Decode(&req); err != nil {
return fmt.Errorf("<-msg %v: %v", msg, err)
}
log.Debug(fmt.Sprintf("<- delivery request: %s", req.String()))
err := self.storage.HandleDeliveryRequestMsg(&req, &peer{bzz: self})
self.lastActive = time.Now()
if err != nil {
return fmt.Errorf("<- %v: %v", msg, err)
}
case paymentMsg:
// swap protocol message for payment, Units paid for, Cheque paid with
paymentMsgCounter.Inc(1)
if self.swapEnabled {
var req paymentMsgData
if err := msg.Decode(&req); err != nil {
return fmt.Errorf("<- %v: %v", msg, err)
}
log.Debug(fmt.Sprintf("<- payment: %s", req.String()))
self.swap.Receive(int(req.Units), req.Promise)
}
default:
// no other message is allowed
invalidMsgCounter.Inc(1)
return fmt.Errorf("invalid message code: %v", msg.Code)
}
return nil
}
func (self *bzz) handleStatus() (err error) {
handshake := &statusMsgData{
Version: uint64(Version),
ID: "honey",
Addr: self.selfAddr(),
NetworkId: self.NetworkId,
Swap: &bzzswap.SwapProfile{
Profile: self.swapParams.Profile,
PayProfile: self.swapParams.PayProfile,
},
}
err = p2p.Send(self.rw, statusMsg, handshake)
if err != nil {
return err
}
// read and handle remote status
var msg p2p.Msg
msg, err = self.rw.ReadMsg()
if err != nil {
return err
}
if msg.Code != statusMsg {
return fmt.Errorf("first msg has code %x (!= %x)", msg.Code, statusMsg)
}
handleStatusMsgCounter.Inc(1)
if msg.Size > ProtocolMaxMsgSize {
return fmt.Errorf("message too long: %v > %v", msg.Size, ProtocolMaxMsgSize)
}
var status statusMsgData
if err := msg.Decode(&status); err != nil {
return fmt.Errorf("<- %v: %v", msg, err)
}
if status.NetworkId != self.NetworkId {
return fmt.Errorf("network id mismatch: %d (!= %d)", status.NetworkId, self.NetworkId)
}
if Version != status.Version {
return fmt.Errorf("protocol version mismatch: %d (!= %d)", status.Version, Version)
}
self.remoteAddr = self.peerAddr(status.Addr)
log.Trace(fmt.Sprintf("self: advertised IP: %v, peer advertised: %v, local address: %v\npeer: advertised IP: %v, remote address: %v\n", self.selfAddr(), self.remoteAddr, self.peer.LocalAddr(), status.Addr.IP, self.peer.RemoteAddr()))
if self.swapEnabled {
// set remote profile for accounting
self.swap, err = bzzswap.NewSwap(self.swapParams, status.Swap, self.backend, self)
if err != nil {
return err
}
}
log.Info(fmt.Sprintf("Peer %08x is capable (%d/%d)", self.remoteAddr.Addr[:4], status.Version, status.NetworkId))
err = self.hive.addPeer(&peer{bzz: self})
if err != nil {
return err
}
// hive sets syncstate so sync should start after node added
log.Info(fmt.Sprintf("syncronisation request sent with %v", self.syncState))
self.syncRequest()
return nil
}
func (self *bzz) sync(state *syncState) error {
// syncer setup
if self.syncer != nil {
return errors.New("sync request can only be sent once")
}
cnt := self.dbAccess.counter()
remoteaddr := self.remoteAddr.Addr
start, stop := self.hive.kad.KeyRange(remoteaddr)
// an explicitly received nil syncstate disables syncronisation
if state == nil {
self.syncEnabled = false
log.Warn(fmt.Sprintf("syncronisation disabled for peer %v", self))
state = &syncState{DbSyncState: &storage.DbSyncState{}, Synced: true}
} else {
state.synced = make(chan bool)
state.SessionAt = cnt
if storage.IsZeroKey(state.Stop) && state.Synced {
state.Start = storage.Key(start[:])
state.Stop = storage.Key(stop[:])
}
log.Debug(fmt.Sprintf("syncronisation requested by peer %v at state %v", self, state))
}
var err error
self.syncer, err = newSyncer(
self.requestDb,
storage.Key(remoteaddr[:]),
self.dbAccess,
self.unsyncedKeys, self.store,
self.syncParams, state, func() bool { return self.syncEnabled },
)
if err != nil {
return nil
}
log.Trace(fmt.Sprintf("syncer set for peer %v", self))
return nil
}
func (self *bzz) String() string {
return self.remoteAddr.String()
}
// repair reported address if IP missing
func (self *bzz) peerAddr(base *peerAddr) *peerAddr {
if base.IP.IsUnspecified() {
host, _, _ := net.SplitHostPort(self.peer.RemoteAddr().String())
base.IP = net.ParseIP(host)
}
return base
}
// returns self advertised node connection info (listening address w enodes)
// IP will get repaired on the other end if missing
// or resolved via ID by discovery at dialout
func (self *bzz) selfAddr() *peerAddr {
id := self.hive.id
host, port, _ := net.SplitHostPort(self.hive.listenAddr())
intport, _ := strconv.Atoi(port)
addr := &peerAddr{
Addr: self.hive.addr,
ID: id[:],
IP: net.ParseIP(host),
Port: uint16(intport),
}
return addr
}
// outgoing messages
// send retrieveRequestMsg
func (self *bzz) retrieve(req *retrieveRequestMsgData) error {
return self.send(retrieveRequestMsg, req)
}
// send storeRequestMsg
func (self *bzz) store(req *storeRequestMsgData) error {
return self.send(storeRequestMsg, req)
}
func (self *bzz) syncRequest() error {
req := &syncRequestMsgData{}
if self.hive.syncEnabled {
log.Debug(fmt.Sprintf("syncronisation request to peer %v at state %v", self, self.syncState))
req.SyncState = self.syncState
}
if self.syncState == nil {
log.Warn(fmt.Sprintf("syncronisation disabled for peer %v at state %v", self, self.syncState))
}
return self.send(syncRequestMsg, req)
}
// queue storeRequestMsg in request db
func (self *bzz) deliveryRequest(reqs []*syncRequest) error {
req := &deliveryRequestMsgData{
Deliver: reqs,
}
return self.send(deliveryRequestMsg, req)
}
// batch of syncRequests to send off
func (self *bzz) unsyncedKeys(reqs []*syncRequest, state *syncState) error {
req := &unsyncedKeysMsgData{
Unsynced: reqs,
State: state,
}
return self.send(unsyncedKeysMsg, req)
}
// send paymentMsg
func (self *bzz) Pay(units int, promise swap.Promise) {
req := &paymentMsgData{uint(units), promise.(*chequebook.Cheque)}
self.payment(req)
}
// send paymentMsg
func (self *bzz) payment(req *paymentMsgData) error {
return self.send(paymentMsg, req)
}
// sends peersMsg
func (self *bzz) peers(req *peersMsgData) error {
return self.send(peersMsg, req)
}
func (self *bzz) send(msg uint64, data interface{}) error {
if self.hive.blockWrite {
return fmt.Errorf("network write blocked")
}
log.Trace(fmt.Sprintf("-> %v: %v (%T) to %v", msg, data, data, self))
err := p2p.Send(self.rw, msg, data)
if err != nil {
self.Drop()
}
return err
}

17
swarm/network/protocol_test.go
View file

@ -1,17 +0,0 @@
// Copyright 2014 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package network

389
swarm/network/syncdb.go
View file

@ -1,389 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package network
import (
"encoding/binary"
"fmt"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/swarm/storage"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/iterator"
)
const counterKeyPrefix = 0x01
/*
syncDb is a queueing service for outgoing deliveries.
One instance per priority queue for each peer
a syncDb instance maintains an in-memory buffer (of capacity bufferSize)
once its in-memory buffer is full it switches to persisting in db
and dbRead iterator iterates through the items keeping their order
once the db read catches up (there is no more items in the db) then
it switches back to in-memory buffer.
when syncdb is stopped all items in the buffer are saved to the db
*/
type syncDb struct {
start []byte // this syncdb starting index in requestdb
key storage.Key // remote peers address key
counterKey []byte // db key to persist counter
priority uint // priotity High|Medium|Low
buffer chan interface{} // incoming request channel
db *storage.LDBDatabase // underlying db (TODO should be interface)
done chan bool // chan to signal goroutines finished quitting
quit chan bool // chan to signal quitting to goroutines
total, dbTotal int // counts for one session
batch chan chan int // channel for batch requests
dbBatchSize uint // number of items before batch is saved
}
// constructor needs a shared request db (leveldb)
// priority is used in the index key
// uses a buffer and a leveldb for persistent storage
// bufferSize, dbBatchSize are config parameters
func newSyncDb(db *storage.LDBDatabase, key storage.Key, priority uint, bufferSize, dbBatchSize uint, deliver func(interface{}, chan bool) bool) *syncDb {
start := make([]byte, 42)
start[1] = byte(priorities - priority)
copy(start[2:34], key)
counterKey := make([]byte, 34)
counterKey[0] = counterKeyPrefix
copy(counterKey[1:], start[1:34])
syncdb := &syncDb{
start: start,
key: key,
counterKey: counterKey,
priority: priority,
buffer: make(chan interface{}, bufferSize),
db: db,
done: make(chan bool),
quit: make(chan bool),
batch: make(chan chan int),
dbBatchSize: dbBatchSize,
}
log.Trace(fmt.Sprintf("syncDb[peer: %v, priority: %v] - initialised", key.Log(), priority))
// starts the main forever loop reading from buffer
go syncdb.bufferRead(deliver)
return syncdb
}
/*
bufferRead is a forever iterator loop that takes care of delivering
outgoing store requests reads from incoming buffer
its argument is the deliver function taking the item as first argument
and a quit channel as second.
Closing of this channel is supposed to abort all waiting for delivery
(typically network write)
The iteration switches between 2 modes,
* buffer mode reads the in-memory buffer and delivers the items directly
* db mode reads from the buffer and writes to the db, parallelly another
routine is started that reads from the db and delivers items
If there is buffer contention in buffer mode (slow network, high upload volume)
syncdb switches to db mode and starts dbRead
Once db backlog is delivered, it reverts back to in-memory buffer
It is automatically started when syncdb is initialised.
It saves the buffer to db upon receiving quit signal. syncDb#stop()
*/
func (self *syncDb) bufferRead(deliver func(interface{}, chan bool) bool) {
var buffer, db chan interface{} // channels representing the two read modes
var more bool
var req interface{}
var entry *syncDbEntry
var inBatch, inDb int
batch := new(leveldb.Batch)
var dbSize chan int
quit := self.quit
counterValue := make([]byte, 8)
// counter is used for keeping the items in order, persisted to db
// start counter where db was at, 0 if not found
data, err := self.db.Get(self.counterKey)
var counter uint64
if err == nil {
counter = binary.BigEndian.Uint64(data)
log.Trace(fmt.Sprintf("syncDb[%v/%v] - counter read from db at %v", self.key.Log(), self.priority, counter))
} else {
log.Trace(fmt.Sprintf("syncDb[%v/%v] - counter starts at %v", self.key.Log(), self.priority, counter))
}
LOOP:
for {
// waiting for item next in the buffer, or quit signal or batch request
select {
// buffer only closes when writing to db
case req = <-buffer:
// deliver request : this is blocking on network write so
// it is passed the quit channel as argument, so that it returns
// if syncdb is stopped. In this case we need to save the item to the db
more = deliver(req, self.quit)
if !more {
log.Debug(fmt.Sprintf("syncDb[%v/%v] quit: switching to db. session tally (db/total): %v/%v", self.key.Log(), self.priority, self.dbTotal, self.total))
// received quit signal, save request currently waiting delivery
// by switching to db mode and closing the buffer
buffer = nil
db = self.buffer
close(db)
quit = nil // needs to block the quit case in select
break // break from select, this item will be written to the db
}
self.total++
log.Trace(fmt.Sprintf("syncDb[%v/%v] deliver (db/total): %v/%v", self.key.Log(), self.priority, self.dbTotal, self.total))
// by the time deliver returns, there were new writes to the buffer
// if buffer contention is detected, switch to db mode which drains
// the buffer so no process will block on pushing store requests
if len(buffer) == cap(buffer) {
log.Debug(fmt.Sprintf("syncDb[%v/%v] buffer full %v: switching to db. session tally (db/total): %v/%v", self.key.Log(), self.priority, cap(buffer), self.dbTotal, self.total))
buffer = nil
db = self.buffer
}
continue LOOP
// incoming entry to put into db
case req, more = <-db:
if !more {
// only if quit is called, saved all the buffer
binary.BigEndian.PutUint64(counterValue, counter)
batch.Put(self.counterKey, counterValue) // persist counter in batch
self.writeSyncBatch(batch) // save batch
log.Trace(fmt.Sprintf("syncDb[%v/%v] quitting: save current batch to db", self.key.Log(), self.priority))
break LOOP
}
self.dbTotal++
self.total++
// otherwise break after select
case dbSize = <-self.batch:
// explicit request for batch
if inBatch == 0 && quit != nil {
// there was no writes since the last batch so db depleted
// switch to buffer mode
log.Debug(fmt.Sprintf("syncDb[%v/%v] empty db: switching to buffer", self.key.Log(), self.priority))
db = nil
buffer = self.buffer
dbSize <- 0 // indicates to 'caller' that batch has been written
inDb = 0
continue LOOP
}
binary.BigEndian.PutUint64(counterValue, counter)
batch.Put(self.counterKey, counterValue)
log.Debug(fmt.Sprintf("syncDb[%v/%v] write batch %v/%v - %x - %x", self.key.Log(), self.priority, inBatch, counter, self.counterKey, counterValue))
batch = self.writeSyncBatch(batch)
dbSize <- inBatch // indicates to 'caller' that batch has been written
inBatch = 0
continue LOOP
// closing syncDb#quit channel is used to signal to all goroutines to quit
case <-quit:
// need to save backlog, so switch to db mode
db = self.buffer
buffer = nil
quit = nil
log.Trace(fmt.Sprintf("syncDb[%v/%v] quitting: save buffer to db", self.key.Log(), self.priority))
close(db)
continue LOOP
}
// only get here if we put req into db
entry, err = self.newSyncDbEntry(req, counter)
if err != nil {
log.Warn(fmt.Sprintf("syncDb[%v/%v] saving request %v (#%v/%v) failed: %v", self.key.Log(), self.priority, req, inBatch, inDb, err))
continue LOOP
}
batch.Put(entry.key, entry.val)
log.Trace(fmt.Sprintf("syncDb[%v/%v] to batch %v '%v' (#%v/%v/%v)", self.key.Log(), self.priority, req, entry, inBatch, inDb, counter))
// if just switched to db mode and not quitting, then launch dbRead
// in a parallel go routine to send deliveries from db
if inDb == 0 && quit != nil {
log.Trace(fmt.Sprintf("syncDb[%v/%v] start dbRead", self.key.Log(), self.priority))
go self.dbRead(true, counter, deliver)
}
inDb++
inBatch++
counter++
// need to save the batch if it gets too large (== dbBatchSize)
if inBatch%int(self.dbBatchSize) == 0 {
batch = self.writeSyncBatch(batch)
}
}
log.Info(fmt.Sprintf("syncDb[%v:%v]: saved %v keys (saved counter at %v)", self.key.Log(), self.priority, inBatch, counter))
close(self.done)
}
// writes the batch to the db and returns a new batch object
func (self *syncDb) writeSyncBatch(batch *leveldb.Batch) *leveldb.Batch {
err := self.db.Write(batch)
if err != nil {
log.Warn(fmt.Sprintf("syncDb[%v/%v] saving batch to db failed: %v", self.key.Log(), self.priority, err))
return batch
}
return new(leveldb.Batch)
}
// abstract type for db entries (TODO could be a feature of Receipts)
type syncDbEntry struct {
key, val []byte
}
func (self syncDbEntry) String() string {
return fmt.Sprintf("key: %x, value: %x", self.key, self.val)
}
/*
dbRead is iterating over store requests to be sent over to the peer
this is mainly to prevent crashes due to network output buffer contention (???)
as well as to make syncronisation resilient to disconnects
the messages are supposed to be sent in the p2p priority queue.
the request DB is shared between peers, but domains for each syncdb
are disjoint. dbkeys (42 bytes) are structured:
* 0: 0x00 (0x01 reserved for counter key)
* 1: priorities - priority (so that high priority can be replayed first)
* 2-33: peers address
* 34-41: syncdb counter to preserve order (this field is missing for the counter key)
values (40 bytes) are:
* 0-31: key
* 32-39: request id
dbRead needs a boolean to indicate if on first round all the historical
record is synced. Second argument to indicate current db counter
The third is the function to apply
*/
func (self *syncDb) dbRead(useBatches bool, counter uint64, fun func(interface{}, chan bool) bool) {
key := make([]byte, 42)
copy(key, self.start)
binary.BigEndian.PutUint64(key[34:], counter)
var batches, n, cnt, total int
var more bool
var entry *syncDbEntry
var it iterator.Iterator
var del *leveldb.Batch
batchSizes := make(chan int)
for {
// if useBatches is false, cnt is not set
if useBatches {
// this could be called before all cnt items sent out
// so that loop is not blocking while delivering
// only relevant if cnt is large
select {
case self.batch <- batchSizes:
case <-self.quit:
return
}
// wait for the write to finish and get the item count in the next batch
cnt = <-batchSizes
batches++
if cnt == 0 {
// empty
return
}
}
it = self.db.NewIterator()
it.Seek(key)
if !it.Valid() {
copy(key, self.start)
useBatches = true
continue
}
del = new(leveldb.Batch)
log.Trace(fmt.Sprintf("syncDb[%v/%v]: new iterator: %x (batch %v, count %v)", self.key.Log(), self.priority, key, batches, cnt))
for n = 0; !useBatches || n < cnt; it.Next() {
copy(key, it.Key())
if len(key) == 0 || key[0] != 0 {
copy(key, self.start)
useBatches = true
break
}
val := make([]byte, 40)
copy(val, it.Value())
entry = &syncDbEntry{key, val}
// log.Trace(fmt.Sprintf("syncDb[%v/%v] - %v, batches: %v, total: %v, session total from db: %v/%v", self.key.Log(), self.priority, self.key.Log(), batches, total, self.dbTotal, self.total))
more = fun(entry, self.quit)
if !more {
// quit received when waiting to deliver entry, the entry will not be deleted
log.Trace(fmt.Sprintf("syncDb[%v/%v] batch %v quit after %v/%v items", self.key.Log(), self.priority, batches, n, cnt))
break
}
// since subsequent batches of the same db session are indexed incrementally
// deleting earlier batches can be delayed and parallelised
// this could be batch delete when db is idle (but added complexity esp when quitting)
del.Delete(key)
n++
total++
}
log.Debug(fmt.Sprintf("syncDb[%v/%v] - db session closed, batches: %v, total: %v, session total from db: %v/%v", self.key.Log(), self.priority, batches, total, self.dbTotal, self.total))
self.db.Write(del) // this could be async called only when db is idle
it.Release()
}
}
//
func (self *syncDb) stop() {
close(self.quit)
<-self.done
}
// calculate a dbkey for the request, for the db to work
// see syncdb for db key structure
// polimorphic: accepted types, see syncer#addRequest
func (self *syncDb) newSyncDbEntry(req interface{}, counter uint64) (entry *syncDbEntry, err error) {
var key storage.Key
var chunk *storage.Chunk
var id uint64
var ok bool
var sreq *storeRequestMsgData
if key, ok = req.(storage.Key); ok {
id = generateId()
} else if chunk, ok = req.(*storage.Chunk); ok {
key = chunk.Key
id = generateId()
} else if sreq, ok = req.(*storeRequestMsgData); ok {
key = sreq.Key
id = sreq.Id
} else if entry, ok = req.(*syncDbEntry); !ok {
return nil, fmt.Errorf("type not allowed: %v (%T)", req, req)
}
// order by peer > priority > seqid
// value is request id if exists
if entry == nil {
dbkey := make([]byte, 42)
dbval := make([]byte, 40)
// encode key
copy(dbkey[:], self.start[:34]) // db peer
binary.BigEndian.PutUint64(dbkey[34:], counter)
// encode value
copy(dbval, key[:])
binary.BigEndian.PutUint64(dbval[32:], id)
entry = &syncDbEntry{dbkey, dbval}
}
return
}

221
swarm/network/syncdb_test.go
View file

@ -1,221 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package network
import (
"bytes"
"fmt"
"os"
"path/filepath"
"testing"
"time"
"github.com/XinFinOrg/XDPoSChain/crypto"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/swarm/storage"
)
func init() {
log.Root().SetHandler(log.LvlFilterHandler(log.LvlCrit, log.StreamHandler(os.Stderr, log.TerminalFormat(false))))
}
type testSyncDb struct {
*syncDb
c int
t *testing.T
fromDb chan bool
delivered [][]byte
sent []int
dbdir string
at int
}
func newTestSyncDb(priority, bufferSize, batchSize int, dbdir string, t *testing.T) *testSyncDb {
if len(dbdir) == 0 {
tmp, err := os.MkdirTemp(os.TempDir(), "syncdb-test")
if err != nil {
t.Fatalf("unable to create temporary direcory %v: %v", tmp, err)
}
dbdir = tmp
}
db, err := storage.NewLDBDatabase(filepath.Join(dbdir, "requestdb"))
if err != nil {
t.Fatalf("unable to create db: %v", err)
}
self := &testSyncDb{
fromDb: make(chan bool),
dbdir: dbdir,
t: t,
}
h := crypto.Keccak256Hash([]byte{0})
key := storage.Key(h[:])
self.syncDb = newSyncDb(db, key, uint(priority), uint(bufferSize), uint(batchSize), self.deliver)
// kick off db iterator right away, if no items on db this will allow
// reading from the buffer
return self
}
func (self *testSyncDb) close() {
self.db.Close()
os.RemoveAll(self.dbdir)
}
func (self *testSyncDb) push(n int) {
for i := 0; i < n; i++ {
self.buffer <- storage.Key(crypto.Keccak256([]byte{byte(self.c)}))
self.sent = append(self.sent, self.c)
self.c++
}
log.Debug(fmt.Sprintf("pushed %v requests", n))
}
func (self *testSyncDb) draindb() {
it := self.db.NewIterator()
defer it.Release()
for {
it.Seek(self.start)
if !it.Valid() {
return
}
k := it.Key()
if len(k) == 0 || k[0] == 1 {
return
}
it.Release()
it = self.db.NewIterator()
}
}
func (self *testSyncDb) deliver(req interface{}, quit chan bool) bool {
_, db := req.(*syncDbEntry)
key, _, _, _, err := parseRequest(req)
if err != nil {
self.t.Fatalf("unexpected error of key %v: %v", key, err)
}
self.delivered = append(self.delivered, key)
select {
case self.fromDb <- db:
return true
case <-quit:
return false
}
}
func (self *testSyncDb) expect(n int, db bool) {
var ok bool
// for n items
for i := 0; i < n; i++ {
ok = <-self.fromDb
if self.at+1 > len(self.delivered) {
self.t.Fatalf("expected %v, got %v", self.at+1, len(self.delivered))
}
if len(self.sent) > self.at && !bytes.Equal(crypto.Keccak256([]byte{byte(self.sent[self.at])}), self.delivered[self.at]) {
self.t.Fatalf("expected delivery %v/%v/%v to be hash of %v, from db: %v = %v", i, n, self.at, self.sent[self.at], ok, db)
log.Debug(fmt.Sprintf("%v/%v/%v to be hash of %v, from db: %v = %v", i, n, self.at, self.sent[self.at], ok, db))
}
if !ok && db {
self.t.Fatalf("expected delivery %v/%v/%v from db", i, n, self.at)
}
if ok && !db {
self.t.Fatalf("expected delivery %v/%v/%v from cache", i, n, self.at)
}
self.at++
}
}
func TestSyncDb(t *testing.T) {
t.Skip("fails randomly on all platforms")
priority := High
bufferSize := 5
batchSize := 2 * bufferSize
s := newTestSyncDb(priority, bufferSize, batchSize, "", t)
defer s.close()
defer s.stop()
s.dbRead(false, 0, s.deliver)
s.draindb()
s.push(4)
s.expect(1, false)
// 3 in buffer
time.Sleep(100 * time.Millisecond)
s.push(3)
// push over limit
s.expect(1, false)
// one popped from the buffer, then contention detected
s.expect(4, true)
s.push(4)
s.expect(5, true)
// depleted db, switch back to buffer
s.draindb()
s.push(5)
s.expect(4, false)
s.push(3)
s.expect(4, false)
// buffer depleted
time.Sleep(100 * time.Millisecond)
s.push(6)
s.expect(1, false)
// push into buffer full, switch to db
s.expect(5, true)
s.draindb()
s.push(1)
s.expect(1, false)
}
func TestSaveSyncDb(t *testing.T) {
amount := 30
priority := High
bufferSize := amount
batchSize := 10
s := newTestSyncDb(priority, bufferSize, batchSize, "", t)
go s.dbRead(false, 0, s.deliver)
s.push(amount)
s.stop()
s.db.Close()
s = newTestSyncDb(priority, bufferSize, batchSize, s.dbdir, t)
go s.dbRead(false, 0, s.deliver)
s.expect(amount, true)
for i, key := range s.delivered {
expKey := crypto.Keccak256([]byte{byte(i)})
if !bytes.Equal(key, expKey) {
t.Fatalf("delivery %v expected to be key %x, got %x", i, expKey, key)
}
}
s.push(amount)
s.expect(amount, false)
for i := amount; i < 2*amount; i++ {
key := s.delivered[i]
expKey := crypto.Keccak256([]byte{byte(i - amount)})
if !bytes.Equal(key, expKey) {
t.Fatalf("delivery %v expected to be key %x, got %x", i, expKey, key)
}
}
s.stop()
s.db.Close()
s = newTestSyncDb(priority, bufferSize, batchSize, s.dbdir, t)
defer s.close()
defer s.stop()
go s.dbRead(false, 0, s.deliver)
s.push(1)
s.expect(1, false)
}

781
swarm/network/syncer.go
View file

@ -1,781 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package network
import (
"encoding/binary"
"encoding/json"
"fmt"
"path/filepath"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/swarm/storage"
)
// syncer parameters (global, not peer specific) default values
const (
requestDbBatchSize = 512 // size of batch before written to request db
keyBufferSize = 1024 // size of buffer for unsynced keys
syncBatchSize = 128 // maximum batchsize for outgoing requests
syncBufferSize = 128 // size of buffer for delivery requests
syncCacheSize = 1024 // cache capacity to store request queue in memory
)
// priorities
const (
Low = iota // 0
Medium // 1
High // 2
priorities // 3 number of priority levels
)
// request types
const (
DeliverReq = iota // 0
PushReq // 1
PropagateReq // 2
HistoryReq // 3
BacklogReq // 4
)
// json serialisable struct to record the syncronisation state between 2 peers
type syncState struct {
*storage.DbSyncState // embeds the following 4 fields:
// Start Key // lower limit of address space
// Stop Key // upper limit of address space
// First uint64 // counter taken from last sync state
// Last uint64 // counter of remote peer dbStore at the time of last connection
SessionAt uint64 // set at the time of connection
LastSeenAt uint64 // set at the time of connection
Latest storage.Key // cursor of dbstore when last (continuously set by syncer)
Synced bool // true iff Sync is done up to the last disconnect
synced chan bool // signal that sync stage finished
}
// wrapper of db-s to provide mockable custom local chunk store access to syncer
type DbAccess struct {
db *storage.DbStore
loc *storage.LocalStore
}
func NewDbAccess(loc *storage.LocalStore) *DbAccess {
return &DbAccess{loc.DbStore.(*storage.DbStore), loc}
}
// to obtain the chunks from key or request db entry only
func (self *DbAccess) get(key storage.Key) (*storage.Chunk, error) {
return self.loc.Get(key)
}
// current storage counter of chunk db
func (self *DbAccess) counter() uint64 {
return self.db.Counter()
}
// implemented by dbStoreSyncIterator
type keyIterator interface {
Next() storage.Key
}
// generator function for iteration by address range and storage counter
func (self *DbAccess) iterator(s *syncState) keyIterator {
it, err := self.db.NewSyncIterator(*(s.DbSyncState))
if err != nil {
return nil
}
return keyIterator(it)
}
func (self syncState) String() string {
if self.Synced {
return fmt.Sprintf(
"session started at: %v, last seen at: %v, latest key: %v",
self.SessionAt, self.LastSeenAt,
self.Latest.Log(),
)
} else {
return fmt.Sprintf(
"address: %v-%v, index: %v-%v, session started at: %v, last seen at: %v, latest key: %v",
self.Start.Log(), self.Stop.Log(),
self.First, self.Last,
self.SessionAt, self.LastSeenAt,
self.Latest.Log(),
)
}
}
// syncer parameters (global, not peer specific)
type SyncParams struct {
RequestDbPath string // path for request db (leveldb)
RequestDbBatchSize uint // nuber of items before batch is saved to requestdb
KeyBufferSize uint // size of key buffer
SyncBatchSize uint // maximum batchsize for outgoing requests
SyncBufferSize uint // size of buffer for
SyncCacheSize uint // cache capacity to store request queue in memory
SyncPriorities []uint // list of priority levels for req types 0-3
SyncModes []bool // list of sync modes for for req types 0-3
}
// constructor with default values
func NewDefaultSyncParams() *SyncParams {
return &SyncParams{
RequestDbBatchSize: requestDbBatchSize,
KeyBufferSize: keyBufferSize,
SyncBufferSize: syncBufferSize,
SyncBatchSize: syncBatchSize,
SyncCacheSize: syncCacheSize,
SyncPriorities: []uint{High, Medium, Medium, Low, Low},
SyncModes: []bool{true, true, true, true, false},
}
}
//this can only finally be set after all config options (file, cmd line, env vars)
//have been evaluated
func (self *SyncParams) Init(path string) {
self.RequestDbPath = filepath.Join(path, "requests")
}
// syncer is the agent that manages content distribution/storage replication/chunk storeRequest forwarding
type syncer struct {
*SyncParams // sync parameters
syncF func() bool // if syncing is needed
key storage.Key // remote peers address key
state *syncState // sync state for our dbStore
syncStates chan *syncState // different stages of sync
deliveryRequest chan bool // one of two triggers needed to send unsyncedKeys
newUnsyncedKeys chan bool // one of two triggers needed to send unsynced keys
quit chan bool // signal to quit loops
// DB related fields
dbAccess *DbAccess // access to dbStore
// native fields
queues [priorities]*syncDb // in-memory cache / queues for sync reqs
keys [priorities]chan interface{} // buffer for unsynced keys
deliveries [priorities]chan *storeRequestMsgData // delivery
// bzz protocol instance outgoing message callbacks (mockable for testing)
unsyncedKeys func([]*syncRequest, *syncState) error // send unsyncedKeysMsg
store func(*storeRequestMsgData) error // send storeRequestMsg
}
// a syncer instance is linked to each peer connection
// constructor is called from protocol after successful handshake
// the returned instance is attached to the peer and can be called
// by the forwarder
func newSyncer(
db *storage.LDBDatabase, remotekey storage.Key,
dbAccess *DbAccess,
unsyncedKeys func([]*syncRequest, *syncState) error,
store func(*storeRequestMsgData) error,
params *SyncParams,
state *syncState,
syncF func() bool,
) (*syncer, error) {
syncBufferSize := params.SyncBufferSize
keyBufferSize := params.KeyBufferSize
dbBatchSize := params.RequestDbBatchSize
self := &syncer{
syncF: syncF,
key: remotekey,
dbAccess: dbAccess,
syncStates: make(chan *syncState, 20),
deliveryRequest: make(chan bool, 1),
newUnsyncedKeys: make(chan bool, 1),
SyncParams: params,
state: state,
quit: make(chan bool),
unsyncedKeys: unsyncedKeys,
store: store,
}
// initialising
for i := 0; i < priorities; i++ {
self.keys[i] = make(chan interface{}, keyBufferSize)
self.deliveries[i] = make(chan *storeRequestMsgData)
// initialise a syncdb instance for each priority queue
self.queues[i] = newSyncDb(db, remotekey, uint(i), syncBufferSize, dbBatchSize, self.deliver(uint(i)))
}
log.Info(fmt.Sprintf("syncer started: %v", state))
// launch chunk delivery service
go self.syncDeliveries()
// launch sync task manager
if self.syncF() {
go self.sync()
}
// process unsynced keys to broadcast
go self.syncUnsyncedKeys()
return self, nil
}
// metadata serialisation
func encodeSync(state *syncState) (*json.RawMessage, error) {
data, err := json.MarshalIndent(state, "", " ")
if err != nil {
return nil, err
}
meta := json.RawMessage(data)
return &meta, nil
}
func decodeSync(meta *json.RawMessage) (*syncState, error) {
if meta == nil {
return nil, fmt.Errorf("unable to deserialise sync state from <nil>")
}
data := []byte(*(meta))
if len(data) == 0 {
return nil, fmt.Errorf("unable to deserialise sync state from <nil>")
}
state := &syncState{DbSyncState: &storage.DbSyncState{}}
err := json.Unmarshal(data, state)
return state, err
}
/*
sync implements the syncing script
* first all items left in the request Db are replayed
* type = StaleSync
* Mode: by default once again via confirmation roundtrip
* Priority: the items are replayed as the proirity specified for StaleSync
* but within the order respects earlier priority level of request
* after all items are consumed for a priority level, the the respective
queue for delivery requests is open (this way new reqs not written to db)
(TODO: this should be checked)
* the sync state provided by the remote peer is used to sync history
* all the backlog from earlier (aborted) syncing is completed starting from latest
* if Last < LastSeenAt then all items in between then process all
backlog from upto last disconnect
* if Last > 0 &&
sync is called from the syncer constructor and is not supposed to be used externally
*/
func (self *syncer) sync() {
state := self.state
// sync finished
defer close(self.syncStates)
// 0. first replay stale requests from request db
if state.SessionAt == 0 {
log.Debug(fmt.Sprintf("syncer[%v]: nothing to sync", self.key.Log()))
return
}
log.Debug(fmt.Sprintf("syncer[%v]: start replaying stale requests from request db", self.key.Log()))
for p := priorities - 1; p >= 0; p-- {
self.queues[p].dbRead(false, 0, self.replay())
}
log.Debug(fmt.Sprintf("syncer[%v]: done replaying stale requests from request db", self.key.Log()))
// unless peer is synced sync unfinished history beginning on
if !state.Synced {
start := state.Start
if !storage.IsZeroKey(state.Latest) {
// 1. there is unfinished earlier sync
state.Start = state.Latest
log.Debug(fmt.Sprintf("syncer[%v]: start syncronising backlog (unfinished sync: %v)", self.key.Log(), state))
// blocks while the entire history upto state is synced
self.syncState(state)
if state.Last < state.SessionAt {
state.First = state.Last + 1
}
}
state.Latest = storage.ZeroKey
state.Start = start
// 2. sync up to last disconnect1
if state.First < state.LastSeenAt {
state.Last = state.LastSeenAt
log.Debug(fmt.Sprintf("syncer[%v]: start syncronising history upto last disconnect at %v: %v", self.key.Log(), state.LastSeenAt, state))
self.syncState(state)
state.First = state.LastSeenAt
}
state.Latest = storage.ZeroKey
} else {
// synchronisation starts at end of last session
state.First = state.LastSeenAt
}
// 3. sync up to current session start
// if there have been new chunks since last session
if state.LastSeenAt < state.SessionAt {
state.Last = state.SessionAt
log.Debug(fmt.Sprintf("syncer[%v]: start syncronising history since last disconnect at %v up until session start at %v: %v", self.key.Log(), state.LastSeenAt, state.SessionAt, state))
// blocks until state syncing is finished
self.syncState(state)
}
log.Info(fmt.Sprintf("syncer[%v]: syncing all history complete", self.key.Log()))
}
// wait till syncronised block uptil state is synced
func (self *syncer) syncState(state *syncState) {
self.syncStates <- state
select {
case <-state.synced:
case <-self.quit:
}
}
// stop quits both request processor and saves the request cache to disk
func (self *syncer) stop() {
close(self.quit)
log.Trace(fmt.Sprintf("syncer[%v]: stop and save sync request db backlog", self.key.Log()))
for _, db := range self.queues {
db.stop()
}
}
// rlp serialisable sync request
type syncRequest struct {
Key storage.Key
Priority uint
}
func (self *syncRequest) String() string {
return fmt.Sprintf("<Key: %v, Priority: %v>", self.Key.Log(), self.Priority)
}
func (self *syncer) newSyncRequest(req interface{}, p int) (*syncRequest, error) {
key, _, _, _, err := parseRequest(req)
// TODO: if req has chunk, it should be put in a cache
// create
if err != nil {
return nil, err
}
return &syncRequest{key, uint(p)}, nil
}
// serves historical items from the DB
// * read is on demand, blocking unless history channel is read
// * accepts sync requests (syncStates) to create new db iterator
// * closes the channel one iteration finishes
func (self *syncer) syncHistory(state *syncState) chan interface{} {
var n uint
history := make(chan interface{})
log.Debug(fmt.Sprintf("syncer[%v]: syncing history between %v - %v for chunk addresses %v - %v", self.key.Log(), state.First, state.Last, state.Start, state.Stop))
it := self.dbAccess.iterator(state)
if it != nil {
go func() {
// signal end of the iteration ended
defer close(history)
IT:
for {
key := it.Next()
if key == nil {
break IT
}
select {
// blocking until history channel is read from
case history <- key:
n++
log.Trace(fmt.Sprintf("syncer[%v]: history: %v (%v keys)", self.key.Log(), key.Log(), n))
state.Latest = key
case <-self.quit:
return
}
}
log.Debug(fmt.Sprintf("syncer[%v]: finished syncing history between %v - %v for chunk addresses %v - %v (at %v) (chunks = %v)", self.key.Log(), state.First, state.Last, state.Start, state.Stop, state.Latest, n))
}()
}
return history
}
// triggers key syncronisation
func (self *syncer) sendUnsyncedKeys() {
select {
case self.deliveryRequest <- true:
default:
}
}
// assembles a new batch of unsynced keys
// * keys are drawn from the key buffers in order of priority queue
// * if the queues of priority for History (HistoryReq) or higher are depleted,
// historical data is used so historical items are lower priority within
// their priority group.
// * Order of historical data is unspecified
func (self *syncer) syncUnsyncedKeys() {
// send out new
var unsynced []*syncRequest
var more, justSynced bool
var keyCount, historyCnt int
var history chan interface{}
priority := High
keys := self.keys[priority]
var newUnsyncedKeys, deliveryRequest chan bool
keyCounts := make([]int, priorities)
histPrior := self.SyncPriorities[HistoryReq]
syncStates := self.syncStates
state := self.state
LOOP:
for {
var req interface{}
// select the highest priority channel to read from
// keys channels are buffered so the highest priority ones
// are checked first - integrity can only be guaranteed if writing
// is locked while selecting
if priority != High || len(keys) == 0 {
// selection is not needed if the High priority queue has items
keys = nil
PRIORITIES:
for priority = High; priority >= 0; priority-- {
// the first priority channel that is non-empty will be assigned to keys
if len(self.keys[priority]) > 0 {
log.Trace(fmt.Sprintf("syncer[%v]: reading request with priority %v", self.key.Log(), priority))
keys = self.keys[priority]
break PRIORITIES
}
log.Trace(fmt.Sprintf("syncer[%v/%v]: queue: [%v, %v, %v]", self.key.Log(), priority, len(self.keys[High]), len(self.keys[Medium]), len(self.keys[Low])))
// if the input queue is empty on this level, resort to history if there is any
if uint(priority) == histPrior && history != nil {
log.Trace(fmt.Sprintf("syncer[%v]: reading history for %v", self.key.Log(), self.key))
keys = history
break PRIORITIES
}
}
}
// if peer ready to receive but nothing to send
if keys == nil && deliveryRequest == nil {
// if no items left and switch to waiting mode
log.Trace(fmt.Sprintf("syncer[%v]: buffers consumed. Waiting", self.key.Log()))
newUnsyncedKeys = self.newUnsyncedKeys
}
// send msg iff
// * peer is ready to receive keys AND (
// * all queues and history are depleted OR
// * batch full OR
// * all history have been consumed, synced)
if deliveryRequest == nil &&
(justSynced ||
len(unsynced) > 0 && keys == nil ||
len(unsynced) == int(self.SyncBatchSize)) {
justSynced = false
// listen to requests
deliveryRequest = self.deliveryRequest
newUnsyncedKeys = nil // not care about data until next req comes in
// set sync to current counter
// (all nonhistorical outgoing traffic sheduled and persisted
state.LastSeenAt = self.dbAccess.counter()
state.Latest = storage.ZeroKey
log.Trace(fmt.Sprintf("syncer[%v]: sending %v", self.key.Log(), unsynced))
// send the unsynced keys
stateCopy := *state
err := self.unsyncedKeys(unsynced, &stateCopy)
if err != nil {
log.Warn(fmt.Sprintf("syncer[%v]: unable to send unsynced keys: %v", self.key.Log(), err))
}
self.state = state
log.Debug(fmt.Sprintf("syncer[%v]: --> %v keys sent: (total: %v (%v), history: %v), sent sync state: %v", self.key.Log(), len(unsynced), keyCounts, keyCount, historyCnt, stateCopy))
unsynced = nil
keys = nil
}
// process item and add it to the batch
select {
case <-self.quit:
break LOOP
case req, more = <-keys:
if keys == history && !more {
log.Trace(fmt.Sprintf("syncer[%v]: syncing history segment complete", self.key.Log()))
// history channel is closed, waiting for new state (called from sync())
syncStates = self.syncStates
state.Synced = true // this signals that the current segment is complete
select {
case state.synced <- false:
case <-self.quit:
break LOOP
}
justSynced = true
history = nil
}
case <-deliveryRequest:
log.Trace(fmt.Sprintf("syncer[%v]: peer ready to receive", self.key.Log()))
// this 1 cap channel can wake up the loop
// signaling that peer is ready to receive unsynced Keys
// the channel is set to nil any further writes will be ignored
deliveryRequest = nil
case <-newUnsyncedKeys:
log.Trace(fmt.Sprintf("syncer[%v]: new unsynced keys available", self.key.Log()))
// this 1 cap channel can wake up the loop
// signals that data is available to send if peer is ready to receive
newUnsyncedKeys = nil
keys = self.keys[High]
case state, more = <-syncStates:
// this resets the state
if !more {
state = self.state
log.Trace(fmt.Sprintf("syncer[%v]: (priority %v) syncing complete upto %v)", self.key.Log(), priority, state))
state.Synced = true
syncStates = nil
} else {
log.Trace(fmt.Sprintf("syncer[%v]: (priority %v) syncing history upto %v priority %v)", self.key.Log(), priority, state, histPrior))
state.Synced = false
history = self.syncHistory(state)
// only one history at a time, only allow another one once the
// history channel is closed
syncStates = nil
}
}
if req == nil {
continue LOOP
}
log.Trace(fmt.Sprintf("syncer[%v]: (priority %v) added to unsynced keys: %v", self.key.Log(), priority, req))
keyCounts[priority]++
keyCount++
if keys == history {
log.Trace(fmt.Sprintf("syncer[%v]: (priority %v) history item %v (synced = %v)", self.key.Log(), priority, req, state.Synced))
historyCnt++
}
if sreq, err := self.newSyncRequest(req, priority); err == nil {
// extract key from req
log.Trace(fmt.Sprintf("syncer[%v]: (priority %v): request %v (synced = %v)", self.key.Log(), priority, req, state.Synced))
unsynced = append(unsynced, sreq)
} else {
log.Warn(fmt.Sprintf("syncer[%v]: (priority %v): error creating request for %v: %v)", self.key.Log(), priority, req, err))
}
}
}
// delivery loop
// takes into account priority, send store Requests with chunk (delivery)
// idle blocking if no new deliveries in any of the queues
func (self *syncer) syncDeliveries() {
var req *storeRequestMsgData
p := High
var deliveries chan *storeRequestMsgData
var msg *storeRequestMsgData
var err error
var c = [priorities]int{}
var n = [priorities]int{}
var total, success uint
for {
deliveries = self.deliveries[p]
select {
case req = <-deliveries:
n[p]++
c[p]++
default:
if p == Low {
// blocking, depletion on all channels, no preference for priority
select {
case req = <-self.deliveries[High]:
n[High]++
case req = <-self.deliveries[Medium]:
n[Medium]++
case req = <-self.deliveries[Low]:
n[Low]++
case <-self.quit:
return
}
p = High
} else {
p--
continue
}
}
total++
msg, err = self.newStoreRequestMsgData(req)
if err != nil {
log.Warn(fmt.Sprintf("syncer[%v]: failed to create store request for %v: %v", self.key.Log(), req, err))
} else {
err = self.store(msg)
if err != nil {
log.Warn(fmt.Sprintf("syncer[%v]: failed to deliver %v: %v", self.key.Log(), req, err))
} else {
success++
log.Trace(fmt.Sprintf("syncer[%v]: %v successfully delivered", self.key.Log(), req))
}
}
if total%self.SyncBatchSize == 0 {
log.Debug(fmt.Sprintf("syncer[%v]: deliver Total: %v, Success: %v, High: %v/%v, Medium: %v/%v, Low %v/%v", self.key.Log(), total, success, c[High], n[High], c[Medium], n[Medium], c[Low], n[Low]))
}
}
}
/*
addRequest handles requests for delivery
it accepts 4 types:
* storeRequestMsgData: coming from netstore propagate response
* chunk: coming from forwarding (questionable: id?)
* key: from incoming syncRequest
* syncDbEntry: key,id encoded in db
If sync mode is on for the type of request, then
it sends the request to the keys queue of the correct priority
channel buffered with capacity (SyncBufferSize)
If sync mode is off then, requests are directly sent to deliveries
*/
func (self *syncer) addRequest(req interface{}, ty int) {
// retrieve priority for request type name int8
priority := self.SyncPriorities[ty]
// sync mode for this type ON
if self.syncF() || ty == DeliverReq {
if self.SyncModes[ty] {
self.addKey(req, priority, self.quit)
} else {
self.addDelivery(req, priority, self.quit)
}
}
}
// addKey queues sync request for sync confirmation with given priority
// ie the key will go out in an unsyncedKeys message
func (self *syncer) addKey(req interface{}, priority uint, quit chan bool) bool {
select {
case self.keys[priority] <- req:
// this wakes up the unsynced keys loop if idle
select {
case self.newUnsyncedKeys <- true:
default:
}
return true
case <-quit:
return false
}
}
// addDelivery queues delivery request for with given priority
// ie the chunk will be delivered ASAP mod priority queueing handled by syncdb
// requests are persisted across sessions for correct sync
func (self *syncer) addDelivery(req interface{}, priority uint, quit chan bool) bool {
select {
case self.queues[priority].buffer <- req:
return true
case <-quit:
return false
}
}
// doDelivery delivers the chunk for the request with given priority
// without queuing
func (self *syncer) doDelivery(req interface{}, priority uint, quit chan bool) bool {
msgdata, err := self.newStoreRequestMsgData(req)
if err != nil {
log.Warn(fmt.Sprintf("unable to deliver request %v: %v", msgdata, err))
return false
}
select {
case self.deliveries[priority] <- msgdata:
return true
case <-quit:
return false
}
}
// returns the delivery function for given priority
// passed on to syncDb
func (self *syncer) deliver(priority uint) func(req interface{}, quit chan bool) bool {
return func(req interface{}, quit chan bool) bool {
return self.doDelivery(req, priority, quit)
}
}
// returns the replay function passed on to syncDb
// depending on sync mode settings for BacklogReq,
// re play of request db backlog sends items via confirmation
// or directly delivers
func (self *syncer) replay() func(req interface{}, quit chan bool) bool {
sync := self.SyncModes[BacklogReq]
priority := self.SyncPriorities[BacklogReq]
// sync mode for this type ON
if sync {
return func(req interface{}, quit chan bool) bool {
return self.addKey(req, priority, quit)
}
} else {
return func(req interface{}, quit chan bool) bool {
return self.doDelivery(req, priority, quit)
}
}
}
// given a request, extends it to a full storeRequestMsgData
// polimorphic: see addRequest for the types accepted
func (self *syncer) newStoreRequestMsgData(req interface{}) (*storeRequestMsgData, error) {
key, id, chunk, sreq, err := parseRequest(req)
if err != nil {
return nil, err
}
if sreq == nil {
if chunk == nil {
var err error
chunk, err = self.dbAccess.get(key)
if err != nil {
return nil, err
}
}
sreq = &storeRequestMsgData{
Id: id,
Key: chunk.Key,
SData: chunk.SData,
}
}
return sreq, nil
}
// parse request types and extracts, key, id, chunk, request if available
// does not do chunk lookup !
func parseRequest(req interface{}) (storage.Key, uint64, *storage.Chunk, *storeRequestMsgData, error) {
var key storage.Key
var entry *syncDbEntry
var chunk *storage.Chunk
var id uint64
var ok bool
var sreq *storeRequestMsgData
var err error
if key, ok = req.(storage.Key); ok {
id = generateId()
} else if entry, ok = req.(*syncDbEntry); ok {
id = binary.BigEndian.Uint64(entry.val[32:])
key = storage.Key(entry.val[:32])
} else if chunk, ok = req.(*storage.Chunk); ok {
key = chunk.Key
id = generateId()
} else if sreq, ok = req.(*storeRequestMsgData); ok {
key = sreq.Key
} else {
err = fmt.Errorf("type not allowed: %v (%T)", req, req)
}
return key, id, chunk, sreq, err
}

290
swarm/services/swap/swap.go
View file

@ -1,290 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package swap
import (
"context"
"crypto/ecdsa"
"fmt"
"math/big"
"os"
"path/filepath"
"sync"
"time"
"github.com/XinFinOrg/XDPoSChain/accounts/abi/bind"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/contracts/chequebook"
"github.com/XinFinOrg/XDPoSChain/contracts/chequebook/contract"
"github.com/XinFinOrg/XDPoSChain/core/types"
"github.com/XinFinOrg/XDPoSChain/crypto"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/swarm/services/swap/swap"
)
// SwAP Swarm Accounting Protocol with
// SWAP^2 Strategies of Withholding Automatic Payments
// SWAP^3 Accreditation: payment via credit SWAP
// using chequebook pkg for delayed payments
// default parameters
var (
autoCashInterval = 300 * time.Second // default interval for autocash
autoCashThreshold = big.NewInt(50000000000000) // threshold that triggers autocash (wei)
autoDepositInterval = 300 * time.Second // default interval for autocash
autoDepositThreshold = big.NewInt(50000000000000) // threshold that triggers autodeposit (wei)
autoDepositBuffer = big.NewInt(100000000000000) // buffer that is surplus for fork protection etc (wei)
buyAt = big.NewInt(20000000000) // maximum chunk price host is willing to pay (wei)
sellAt = big.NewInt(20000000000) // minimum chunk price host requires (wei)
payAt = 100 // threshold that triggers payment {request} (units)
dropAt = 10000 // threshold that triggers disconnect (units)
)
const (
chequebookDeployRetries = 5
chequebookDeployDelay = 1 * time.Second // delay between retries
)
type SwapParams struct {
*swap.Params
*PayProfile
}
type SwapProfile struct {
*swap.Profile
*PayProfile
}
type PayProfile struct {
PublicKey string // check against signature of promise
Contract common.Address // address of chequebook contract
Beneficiary common.Address // recipient address for swarm sales revenue
privateKey *ecdsa.PrivateKey
publicKey *ecdsa.PublicKey
owner common.Address
chbook *chequebook.Chequebook
lock sync.RWMutex
}
//create params with default values
func NewDefaultSwapParams() *SwapParams {
return &SwapParams{
PayProfile: &PayProfile{},
Params: &swap.Params{
Profile: &swap.Profile{
BuyAt: buyAt,
SellAt: sellAt,
PayAt: uint(payAt),
DropAt: uint(dropAt),
},
Strategy: &swap.Strategy{
AutoCashInterval: autoCashInterval,
AutoCashThreshold: autoCashThreshold,
AutoDepositInterval: autoDepositInterval,
AutoDepositThreshold: autoDepositThreshold,
AutoDepositBuffer: autoDepositBuffer,
},
},
}
}
//this can only finally be set after all config options (file, cmd line, env vars)
//have been evaluated
func (self *SwapParams) Init(contract common.Address, prvkey *ecdsa.PrivateKey) {
pubkey := &prvkey.PublicKey
self.PayProfile = &PayProfile{
PublicKey: common.ToHex(crypto.FromECDSAPub(pubkey)),
Contract: contract,
Beneficiary: crypto.PubkeyToAddress(*pubkey),
privateKey: prvkey,
publicKey: pubkey,
owner: crypto.PubkeyToAddress(*pubkey),
}
}
// swap constructor, parameters
// * global chequebook, assume deployed service and
// * the balance is at buffer.
// swap.Add(n) called in netstore
// n > 0 called when sending chunks = receiving retrieve requests
// OR sending cheques.
// n < 0 called when receiving chunks = receiving delivery responses
// OR receiving cheques.
func NewSwap(local *SwapParams, remote *SwapProfile, backend chequebook.Backend, proto swap.Protocol) (self *swap.Swap, err error) {
var (
ctx = context.TODO()
ok bool
in *chequebook.Inbox
out *chequebook.Outbox
)
// check if remote chequebook is valid
// insolvent chequebooks suicide so will signal as invalid
// TODO: monitoring a chequebooks events
ok, err = chequebook.ValidateCode(ctx, backend, remote.Contract)
if !ok {
log.Info(fmt.Sprintf("invalid contract %v for peer %v: %v)", remote.Contract.Hex()[:8], proto, err))
} else {
// remote contract valid, create inbox
in, err = chequebook.NewInbox(local.privateKey, remote.Contract, local.Beneficiary, crypto.ToECDSAPub(common.FromHex(remote.PublicKey)), backend)
if err != nil {
log.Warn(fmt.Sprintf("unable to set up inbox for chequebook contract %v for peer %v: %v)", remote.Contract.Hex()[:8], proto, err))
}
}
// check if local chequebook contract is valid
ok, err = chequebook.ValidateCode(ctx, backend, local.Contract)
if !ok {
log.Warn(fmt.Sprintf("unable to set up outbox for peer %v: chequebook contract (owner: %v): %v)", proto, local.owner.Hex(), err))
} else {
out = chequebook.NewOutbox(local.Chequebook(), remote.Beneficiary)
}
pm := swap.Payment{
In: in,
Out: out,
Buys: out != nil,
Sells: in != nil,
}
self, err = swap.New(local.Params, pm, proto)
if err != nil {
return
}
// remote profile given (first) in handshake
self.SetRemote(remote.Profile)
var buy, sell string
if self.Buys {
buy = "purchase from peer enabled at " + remote.SellAt.String() + " wei/chunk"
} else {
buy = "purchase from peer disabled"
}
if self.Sells {
sell = "selling to peer enabled at " + local.SellAt.String() + " wei/chunk"
} else {
sell = "selling to peer disabled"
}
log.Warn(fmt.Sprintf("SWAP arrangement with <%v>: %v; %v)", proto, buy, sell))
return
}
func (self *SwapParams) Chequebook() *chequebook.Chequebook {
defer self.lock.Unlock()
self.lock.Lock()
return self.chbook
}
func (self *SwapParams) PrivateKey() *ecdsa.PrivateKey {
return self.privateKey
}
// func (self *SwapParams) PublicKey() *ecdsa.PublicKey {
// return self.publicKey
// }
func (self *SwapParams) SetKey(prvkey *ecdsa.PrivateKey) {
self.privateKey = prvkey
self.publicKey = &prvkey.PublicKey
}
// setChequebook(path, backend) wraps the
// chequebook initialiser and sets up autoDeposit to cover spending.
func (self *SwapParams) SetChequebook(ctx context.Context, backend chequebook.Backend, path string) error {
self.lock.Lock()
contract := self.Contract
self.lock.Unlock()
valid, err := chequebook.ValidateCode(ctx, backend, contract)
if err != nil {
return err
} else if valid {
return self.newChequebookFromContract(path, backend)
}
return self.deployChequebook(ctx, backend, path)
}
func (self *SwapParams) deployChequebook(ctx context.Context, backend chequebook.Backend, path string) error {
opts := bind.NewKeyedTransactor(self.privateKey)
opts.Value = self.AutoDepositBuffer
opts.Context = ctx
log.Info(fmt.Sprintf("Deploying new chequebook (owner: %v)", opts.From.Hex()))
contract, err := deployChequebookLoop(opts, backend)
if err != nil {
log.Error(fmt.Sprintf("unable to deploy new chequebook: %v", err))
return err
}
log.Info(fmt.Sprintf("new chequebook deployed at %v (owner: %v)", contract.Hex(), opts.From.Hex()))
// need to save config at this point
self.lock.Lock()
self.Contract = contract
err = self.newChequebookFromContract(path, backend)
self.lock.Unlock()
if err != nil {
log.Warn(fmt.Sprintf("error initialising cheque book (owner: %v): %v", opts.From.Hex(), err))
}
return err
}
// repeatedly tries to deploy a chequebook.
func deployChequebookLoop(opts *bind.TransactOpts, backend chequebook.Backend) (addr common.Address, err error) {
var tx *types.Transaction
for try := 0; try < chequebookDeployRetries; try++ {
if try > 0 {
time.Sleep(chequebookDeployDelay)
}
if _, tx, _, err = contract.DeployChequebook(opts, backend); err != nil {
log.Warn(fmt.Sprintf("can't send chequebook deploy tx (try %d): %v", try, err))
continue
}
if addr, err = bind.WaitDeployed(opts.Context, backend, tx); err != nil {
log.Warn(fmt.Sprintf("chequebook deploy error (try %d): %v", try, err))
continue
}
return addr, nil
}
return addr, err
}
// initialise the chequebook from a persisted json file or create a new one
// caller holds the lock
func (self *SwapParams) newChequebookFromContract(path string, backend chequebook.Backend) error {
hexkey := common.Bytes2Hex(self.Contract.Bytes())
err := os.MkdirAll(filepath.Join(path, "chequebooks"), os.ModePerm)
if err != nil {
return fmt.Errorf("unable to create directory for chequebooks: %v", err)
}
chbookpath := filepath.Join(path, "chequebooks", hexkey+".json")
self.chbook, err = chequebook.LoadChequebook(chbookpath, self.privateKey, backend, true)
if err != nil {
self.chbook, err = chequebook.NewChequebook(chbookpath, self.Contract, self.privateKey, backend)
if err != nil {
log.Warn(fmt.Sprintf("unable to initialise chequebook (owner: %v): %v", self.owner.Hex(), err))
return fmt.Errorf("unable to initialise chequebook (owner: %v): %v", self.owner.Hex(), err)
}
}
self.chbook.AutoDeposit(self.AutoDepositInterval, self.AutoDepositThreshold, self.AutoDepositBuffer)
log.Info(fmt.Sprintf("auto deposit ON for %v -> %v: interval = %v, threshold = %v, buffer = %v)", crypto.PubkeyToAddress(*(self.publicKey)).Hex()[:8], self.Contract.Hex()[:8], self.AutoDepositInterval, self.AutoDepositThreshold, self.AutoDepositBuffer))
return nil
}

252
swarm/services/swap/swap/swap.go
View file

@ -1,252 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package swap
import (
"fmt"
"math/big"
"sync"
"time"
"github.com/XinFinOrg/XDPoSChain/log"
)
// SwAP Swarm Accounting Protocol with
// Swift Automatic Payments
// a peer to peer micropayment system
// public swap profile
// public parameters for SWAP, serializable config struct passed in handshake
type Profile struct {
BuyAt *big.Int // accepted max price for chunk
SellAt *big.Int // offered sale price for chunk
PayAt uint // threshold that triggers payment request
DropAt uint // threshold that triggers disconnect
}
// Strategy encapsulates parameters relating to
// automatic deposit and automatic cashing
type Strategy struct {
AutoCashInterval time.Duration // default interval for autocash
AutoCashThreshold *big.Int // threshold that triggers autocash (wei)
AutoDepositInterval time.Duration // default interval for autocash
AutoDepositThreshold *big.Int // threshold that triggers autodeposit (wei)
AutoDepositBuffer *big.Int // buffer that is surplus for fork protection etc (wei)
}
// Params extends the public profile with private parameters relating to
// automatic deposit and automatic cashing
type Params struct {
*Profile
*Strategy
}
// Promise
// 3rd party Provable Promise of Payment
// issued by outPayment
// serialisable to send with Protocol
type Promise interface{}
// interface for the peer protocol for testing or external alternative payment
type Protocol interface {
Pay(int, Promise) // units, payment proof
Drop()
String() string
}
// interface for the (delayed) ougoing payment system with autodeposit
type OutPayment interface {
Issue(amount *big.Int) (promise Promise, err error)
AutoDeposit(interval time.Duration, threshold, buffer *big.Int)
Stop()
}
// interface for the (delayed) incoming payment system with autocash
type InPayment interface {
Receive(promise Promise) (*big.Int, error)
AutoCash(cashInterval time.Duration, maxUncashed *big.Int)
Stop()
}
// swap is the swarm accounting protocol instance
// * pairwise accounting and payments
type Swap struct {
lock sync.Mutex // mutex for balance access
balance int // units of chunk/retrieval request
local *Params // local peer's swap parameters
remote *Profile // remote peer's swap profile
proto Protocol // peer communication protocol
Payment
}
type Payment struct {
Out OutPayment // outgoing payment handler
In InPayment // incoming payment handler
Buys, Sells bool
}
// swap constructor
func New(local *Params, pm Payment, proto Protocol) (self *Swap, err error) {
self = &Swap{
local: local,
Payment: pm,
proto: proto,
}
self.SetParams(local)
return
}
// entry point for setting remote swap profile (e.g from handshake or other message)
func (self *Swap) SetRemote(remote *Profile) {
defer self.lock.Unlock()
self.lock.Lock()
self.remote = remote
if self.Sells && (remote.BuyAt.Sign() <= 0 || self.local.SellAt.Sign() <= 0 || remote.BuyAt.Cmp(self.local.SellAt) < 0) {
self.Out.Stop()
self.Sells = false
}
if self.Buys && (remote.SellAt.Sign() <= 0 || self.local.BuyAt.Sign() <= 0 || self.local.BuyAt.Cmp(self.remote.SellAt) < 0) {
self.In.Stop()
self.Buys = false
}
log.Debug(fmt.Sprintf("<%v> remote profile set: pay at: %v, drop at: %v, buy at: %v, sell at: %v", self.proto, remote.PayAt, remote.DropAt, remote.BuyAt, remote.SellAt))
}
// to set strategy dynamically
func (self *Swap) SetParams(local *Params) {
defer self.lock.Unlock()
self.lock.Lock()
self.local = local
self.setParams(local)
}
// caller holds the lock
func (self *Swap) setParams(local *Params) {
if self.Sells {
self.In.AutoCash(local.AutoCashInterval, local.AutoCashThreshold)
log.Info(fmt.Sprintf("<%v> set autocash to every %v, max uncashed limit: %v", self.proto, local.AutoCashInterval, local.AutoCashThreshold))
} else {
log.Info(fmt.Sprintf("<%v> autocash off (not selling)", self.proto))
}
if self.Buys {
self.Out.AutoDeposit(local.AutoDepositInterval, local.AutoDepositThreshold, local.AutoDepositBuffer)
log.Info(fmt.Sprintf("<%v> set autodeposit to every %v, pay at: %v, buffer: %v", self.proto, local.AutoDepositInterval, local.AutoDepositThreshold, local.AutoDepositBuffer))
} else {
log.Info(fmt.Sprintf("<%v> autodeposit off (not buying)", self.proto))
}
}
// Add(n)
// n > 0 called when promised/provided n units of service
// n < 0 called when used/requested n units of service
func (self *Swap) Add(n int) error {
defer self.lock.Unlock()
self.lock.Lock()
self.balance += n
if !self.Sells && self.balance > 0 {
log.Trace(fmt.Sprintf("<%v> remote peer cannot have debt (balance: %v)", self.proto, self.balance))
self.proto.Drop()
return fmt.Errorf("[SWAP] <%v> remote peer cannot have debt (balance: %v)", self.proto, self.balance)
}
if !self.Buys && self.balance < 0 {
log.Trace(fmt.Sprintf("<%v> we cannot have debt (balance: %v)", self.proto, self.balance))
return fmt.Errorf("[SWAP] <%v> we cannot have debt (balance: %v)", self.proto, self.balance)
}
if self.balance >= int(self.local.DropAt) {
log.Trace(fmt.Sprintf("<%v> remote peer has too much debt (balance: %v, disconnect threshold: %v)", self.proto, self.balance, self.local.DropAt))
self.proto.Drop()
return fmt.Errorf("[SWAP] <%v> remote peer has too much debt (balance: %v, disconnect threshold: %v)", self.proto, self.balance, self.local.DropAt)
} else if self.balance <= -int(self.remote.PayAt) {
self.send()
}
return nil
}
func (self *Swap) Balance() int {
defer self.lock.Unlock()
self.lock.Lock()
return self.balance
}
// send(units) is called when payment is due
// In case of insolvency no promise is issued and sent, safe against fraud
// No return value: no error = payment is opportunistic = hang in till dropped
func (self *Swap) send() {
if self.local.BuyAt != nil && self.balance < 0 {
amount := big.NewInt(int64(-self.balance))
amount.Mul(amount, self.remote.SellAt)
promise, err := self.Out.Issue(amount)
if err != nil {
log.Warn(fmt.Sprintf("<%v> cannot issue cheque (amount: %v, channel: %v): %v", self.proto, amount, self.Out, err))
} else {
log.Warn(fmt.Sprintf("<%v> cheque issued (amount: %v, channel: %v)", self.proto, amount, self.Out))
self.proto.Pay(-self.balance, promise)
self.balance = 0
}
}
}
// receive(units, promise) is called by the protocol when a payment msg is received
// returns error if promise is invalid.
func (self *Swap) Receive(units int, promise Promise) error {
if units <= 0 {
return fmt.Errorf("invalid units: %v <= 0", units)
}
price := new(big.Int).SetInt64(int64(units))
price.Mul(price, self.local.SellAt)
amount, err := self.In.Receive(promise)
if err != nil {
err = fmt.Errorf("invalid promise: %v", err)
} else if price.Cmp(amount) != 0 {
// verify amount = units * unit sale price
return fmt.Errorf("invalid amount: %v = %v * %v (units sent in msg * agreed sale unit price) != %v (signed in cheque)", price, units, self.local.SellAt, amount)
}
if err != nil {
log.Trace(fmt.Sprintf("<%v> invalid promise (amount: %v, channel: %v): %v", self.proto, amount, self.In, err))
return err
}
// credit remote peer with units
self.Add(-units)
log.Trace(fmt.Sprintf("<%v> received promise (amount: %v, channel: %v): %v", self.proto, amount, self.In, promise))
return nil
}
// stop() causes autocash loop to terminate.
// Called after protocol handle loop terminates.
func (self *Swap) Stop() {
defer self.lock.Unlock()
self.lock.Lock()
if self.Buys {
self.Out.Stop()
}
if self.Sells {
self.In.Stop()
}
}

194
swarm/services/swap/swap/swap_test.go
View file

@ -1,194 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package swap
import (
"math/big"
"testing"
"time"
"github.com/XinFinOrg/XDPoSChain/common"
)
type testInPayment struct {
received []*testPromise
autocashInterval time.Duration
autocashLimit *big.Int
}
type testPromise struct {
amount *big.Int
}
func (self *testInPayment) Receive(promise Promise) (*big.Int, error) {
p := promise.(*testPromise)
self.received = append(self.received, p)
return p.amount, nil
}
func (self *testInPayment) AutoCash(interval time.Duration, limit *big.Int) {
self.autocashInterval = interval
self.autocashLimit = limit
}
func (self *testInPayment) Cash() (string, error) { return "", nil }
func (self *testInPayment) Stop() {}
type testOutPayment struct {
deposits []*big.Int
autodepositInterval time.Duration
autodepositThreshold *big.Int
autodepositBuffer *big.Int
}
func (self *testOutPayment) Issue(amount *big.Int) (promise Promise, err error) {
return &testPromise{amount}, nil
}
func (self *testOutPayment) Deposit(amount *big.Int) (string, error) {
self.deposits = append(self.deposits, amount)
return "", nil
}
func (self *testOutPayment) AutoDeposit(interval time.Duration, threshold, buffer *big.Int) {
self.autodepositInterval = interval
self.autodepositThreshold = threshold
self.autodepositBuffer = buffer
}
func (self *testOutPayment) Stop() {}
type testProtocol struct {
drop bool
amounts []int
promises []*testPromise
}
func (self *testProtocol) Drop() {
self.drop = true
}
func (self *testProtocol) String() string {
return ""
}
func (self *testProtocol) Pay(amount int, promise Promise) {
p := promise.(*testPromise)
self.promises = append(self.promises, p)
self.amounts = append(self.amounts, amount)
}
func TestSwap(t *testing.T) {
strategy := &Strategy{
AutoCashInterval: 1 * time.Second,
AutoCashThreshold: big.NewInt(20),
AutoDepositInterval: 1 * time.Second,
AutoDepositThreshold: big.NewInt(20),
AutoDepositBuffer: big.NewInt(40),
}
local := &Params{
Profile: &Profile{
PayAt: 5,
DropAt: 10,
BuyAt: common.Big3,
SellAt: common.Big2,
},
Strategy: strategy,
}
in := &testInPayment{}
out := &testOutPayment{}
proto := &testProtocol{}
swap, _ := New(local, Payment{In: in, Out: out, Buys: true, Sells: true}, proto)
if in.autocashInterval != strategy.AutoCashInterval {
t.Fatalf("autocash interval not properly set, expect %v, got %v", strategy.AutoCashInterval, in.autocashInterval)
}
if out.autodepositInterval != strategy.AutoDepositInterval {
t.Fatalf("autodeposit interval not properly set, expect %v, got %v", strategy.AutoDepositInterval, out.autodepositInterval)
}
remote := &Profile{
PayAt: 3,
DropAt: 10,
BuyAt: common.Big2,
SellAt: common.Big3,
}
swap.SetRemote(remote)
swap.Add(9)
if proto.drop {
t.Fatalf("not expected peer to be dropped")
}
swap.Add(1)
if !proto.drop {
t.Fatalf("expected peer to be dropped")
}
if !proto.drop {
t.Fatalf("expected peer to be dropped")
}
proto.drop = false
swap.Receive(10, &testPromise{big.NewInt(20)})
if swap.balance != 0 {
t.Fatalf("expected zero balance, got %v", swap.balance)
}
if len(proto.amounts) != 0 {
t.Fatalf("expected zero balance, got %v", swap.balance)
}
swap.Add(-2)
if len(proto.amounts) > 0 {
t.Fatalf("expected no payments yet, got %v", proto.amounts)
}
swap.Add(-1)
if len(proto.amounts) != 1 {
t.Fatalf("expected one payment, got %v", len(proto.amounts))
}
if proto.amounts[0] != 3 {
t.Fatalf("expected payment for %v units, got %v", proto.amounts[0], 3)
}
exp := new(big.Int).Mul(big.NewInt(int64(proto.amounts[0])), remote.SellAt)
if proto.promises[0].amount.Cmp(exp) != 0 {
t.Fatalf("expected payment amount %v, got %v", exp, proto.promises[0].amount)
}
swap.SetParams(&Params{
Profile: &Profile{
PayAt: 5,
DropAt: 10,
BuyAt: common.Big3,
SellAt: common.Big2,
},
Strategy: &Strategy{
AutoCashInterval: 2 * time.Second,
AutoCashThreshold: big.NewInt(40),
AutoDepositInterval: 2 * time.Second,
AutoDepositThreshold: big.NewInt(40),
AutoDepositBuffer: big.NewInt(60),
},
})
}

533
swarm/storage/chunker.go
View file

@ -1,533 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package storage
import (
"encoding/binary"
"errors"
"fmt"
"io"
"sync"
"time"
"github.com/XinFinOrg/XDPoSChain/metrics"
)
/*
The distributed storage implemented in this package requires fix sized chunks of content.
Chunker is the interface to a component that is responsible for disassembling and assembling larger data.
TreeChunker implements a Chunker based on a tree structure defined as follows:
1 each node in the tree including the root and other branching nodes are stored as a chunk.
2 branching nodes encode data contents that includes the size of the dataslice covered by its entire subtree under the node as well as the hash keys of all its children :
data_{i} := size(subtree_{i}) || key_{j} || key_{j+1} .... || key_{j+n-1}
3 Leaf nodes encode an actual subslice of the input data.
4 if data size is not more than maximum chunksize, the data is stored in a single chunk
key = hash(int64(size) + data)
5 if data size is more than chunksize*branches^l, but no more than chunksize*
branches^(l+1), the data vector is split into slices of chunksize*
branches^l length (except the last one).
key = hash(int64(size) + key(slice0) + key(slice1) + ...)
The underlying hash function is configurable
*/
/*
Tree chunker is a concrete implementation of data chunking.
This chunker works in a simple way, it builds a tree out of the document so that each node either represents a chunk of real data or a chunk of data representing an branching non-leaf node of the tree. In particular each such non-leaf chunk will represent is a concatenation of the hash of its respective children. This scheme simultaneously guarantees data integrity as well as self addressing. Abstract nodes are transparent since their represented size component is strictly greater than their maximum data size, since they encode a subtree.
If all is well it is possible to implement this by simply composing readers so that no extra allocation or buffering is necessary for the data splitting and joining. This means that in principle there can be direct IO between : memory, file system, network socket (bzz peers storage request is read from the socket). In practice there may be need for several stages of internal buffering.
The hashing itself does use extra copies and allocation though, since it does need it.
*/
var (
errAppendOppNotSuported = errors.New("Append operation not supported")
errOperationTimedOut = errors.New("operation timed out")
)
//metrics variables
var (
newChunkCounter = metrics.NewRegisteredCounter("storage.chunks.new", nil)
)
type TreeChunker struct {
branches int64
hashFunc SwarmHasher
// calculated
hashSize int64 // self.hashFunc.New().Size()
chunkSize int64 // hashSize* branches
workerCount int64 // the number of worker routines used
workerLock sync.RWMutex // lock for the worker count
}
func NewTreeChunker(params *ChunkerParams) (self *TreeChunker) {
self = &TreeChunker{}
self.hashFunc = MakeHashFunc(params.Hash)
self.branches = params.Branches
self.hashSize = int64(self.hashFunc().Size())
self.chunkSize = self.hashSize * self.branches
self.workerCount = 0
return
}
// func (self *TreeChunker) KeySize() int64 {
// return self.hashSize
// }
// String() for pretty printing
func (self *Chunk) String() string {
return fmt.Sprintf("Key: %v TreeSize: %v Chunksize: %v", self.Key.Log(), self.Size, len(self.SData))
}
type hashJob struct {
key Key
chunk []byte
size int64
parentWg *sync.WaitGroup
}
func (self *TreeChunker) incrementWorkerCount() {
self.workerLock.Lock()
defer self.workerLock.Unlock()
self.workerCount += 1
}
func (self *TreeChunker) getWorkerCount() int64 {
self.workerLock.RLock()
defer self.workerLock.RUnlock()
return self.workerCount
}
func (self *TreeChunker) decrementWorkerCount() {
self.workerLock.Lock()
defer self.workerLock.Unlock()
self.workerCount -= 1
}
func (self *TreeChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, swg, wwg *sync.WaitGroup) (Key, error) {
if self.chunkSize <= 0 {
panic("chunker must be initialised")
}
jobC := make(chan *hashJob, 2*ChunkProcessors)
wg := &sync.WaitGroup{}
errC := make(chan error)
quitC := make(chan bool)
// wwg = workers waitgroup keeps track of hashworkers spawned by this split call
if wwg != nil {
wwg.Add(1)
}
self.incrementWorkerCount()
go self.hashWorker(jobC, chunkC, errC, quitC, swg, wwg)
depth := 0
treeSize := self.chunkSize
// takes lowest depth such that chunksize*HashCount^(depth+1) > size
// power series, will find the order of magnitude of the data size in base hashCount or numbers of levels of branching in the resulting tree.
for ; treeSize < size; treeSize *= self.branches {
depth++
}
key := make([]byte, self.hashFunc().Size())
// this waitgroup member is released after the root hash is calculated
wg.Add(1)
//launch actual recursive function passing the waitgroups
go self.split(depth, treeSize/self.branches, key, data, size, jobC, chunkC, errC, quitC, wg, swg, wwg)
// closes internal error channel if all subprocesses in the workgroup finished
go func() {
// waiting for all threads to finish
wg.Wait()
// if storage waitgroup is non-nil, we wait for storage to finish too
if swg != nil {
swg.Wait()
}
close(errC)
}()
defer close(quitC)
select {
case err := <-errC:
if err != nil {
return nil, err
}
case <-time.NewTimer(splitTimeout).C:
return nil, errOperationTimedOut
}
return key, nil
}
func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reader, size int64, jobC chan *hashJob, chunkC chan *Chunk, errC chan error, quitC chan bool, parentWg, swg, wwg *sync.WaitGroup) {
//
for depth > 0 && size < treeSize {
treeSize /= self.branches
depth--
}
if depth == 0 {
// leaf nodes -> content chunks
chunkData := make([]byte, size+8)
binary.LittleEndian.PutUint64(chunkData[0:8], uint64(size))
var readBytes int64
for readBytes < size {
n, err := data.Read(chunkData[8+readBytes:])
readBytes += int64(n)
if err != nil && !(err == io.EOF && readBytes == size) {
errC <- err
return
}
}
select {
case jobC <- &hashJob{key, chunkData, size, parentWg}:
case <-quitC:
}
return
}
// dept > 0
// intermediate chunk containing child nodes hashes
branchCnt := (size + treeSize - 1) / treeSize
var chunk = make([]byte, branchCnt*self.hashSize+8)
var pos, i int64
binary.LittleEndian.PutUint64(chunk[0:8], uint64(size))
childrenWg := &sync.WaitGroup{}
var secSize int64
for i < branchCnt {
// the last item can have shorter data
if size-pos < treeSize {
secSize = size - pos
} else {
secSize = treeSize
}
// the hash of that data
subTreeKey := chunk[8+i*self.hashSize : 8+(i+1)*self.hashSize]
childrenWg.Add(1)
self.split(depth-1, treeSize/self.branches, subTreeKey, data, secSize, jobC, chunkC, errC, quitC, childrenWg, swg, wwg)
i++
pos += treeSize
}
// wait for all the children to complete calculating their hashes and copying them onto sections of the chunk
// parentWg.Add(1)
// go func() {
childrenWg.Wait()
worker := self.getWorkerCount()
if int64(len(jobC)) > worker && worker < ChunkProcessors {
if wwg != nil {
wwg.Add(1)
}
self.incrementWorkerCount()
go self.hashWorker(jobC, chunkC, errC, quitC, swg, wwg)
}
select {
case jobC <- &hashJob{key, chunk, size, parentWg}:
case <-quitC:
}
}
func (self *TreeChunker) hashWorker(jobC chan *hashJob, chunkC chan *Chunk, errC chan error, quitC chan bool, swg, wwg *sync.WaitGroup) {
defer self.decrementWorkerCount()
hasher := self.hashFunc()
if wwg != nil {
defer wwg.Done()
}
for {
select {
case job, ok := <-jobC:
if !ok {
return
}
// now we got the hashes in the chunk, then hash the chunks
self.hashChunk(hasher, job, chunkC, swg)
case <-quitC:
return
}
}
}
// The treeChunkers own Hash hashes together
// - the size (of the subtree encoded in the Chunk)
// - the Chunk, ie. the contents read from the input reader
func (self *TreeChunker) hashChunk(hasher SwarmHash, job *hashJob, chunkC chan *Chunk, swg *sync.WaitGroup) {
hasher.ResetWithLength(job.chunk[:8]) // 8 bytes of length
hasher.Write(job.chunk[8:]) // minus 8 []byte length
h := hasher.Sum(nil)
newChunk := &Chunk{
Key: h,
SData: job.chunk,
Size: job.size,
wg: swg,
}
// report hash of this chunk one level up (keys corresponds to the proper subslice of the parent chunk)
copy(job.key, h)
// send off new chunk to storage
if chunkC != nil {
if swg != nil {
swg.Add(1)
}
}
job.parentWg.Done()
if chunkC != nil {
//NOTE: this increases the chunk count even if the local node already has this chunk;
//on file upload the node will increase this counter even if the same file has already been uploaded
//So it should be evaluated whether it is worth keeping this counter
//and/or actually better track when the chunk is Put to the local database
//(which may question the need for disambiguation when a completely new chunk has been created
//and/or a chunk is being put to the local DB; for chunk tracking it may be worth distinguishing
newChunkCounter.Inc(1)
chunkC <- newChunk
}
}
func (self *TreeChunker) Append(key Key, data io.Reader, chunkC chan *Chunk, swg, wwg *sync.WaitGroup) (Key, error) {
return nil, errAppendOppNotSuported
}
// LazyChunkReader implements LazySectionReader
type LazyChunkReader struct {
key Key // root key
chunkC chan *Chunk // chunk channel to send retrieve requests on
chunk *Chunk // size of the entire subtree
off int64 // offset
chunkSize int64 // inherit from chunker
branches int64 // inherit from chunker
hashSize int64 // inherit from chunker
}
// implements the Joiner interface
func (self *TreeChunker) Join(key Key, chunkC chan *Chunk) LazySectionReader {
return &LazyChunkReader{
key: key,
chunkC: chunkC,
chunkSize: self.chunkSize,
branches: self.branches,
hashSize: self.hashSize,
}
}
// Size is meant to be called on the LazySectionReader
func (self *LazyChunkReader) Size(quitC chan bool) (n int64, err error) {
if self.chunk != nil {
return self.chunk.Size, nil
}
chunk := retrieve(self.key, self.chunkC, quitC)
if chunk == nil {
select {
case <-quitC:
return 0, errors.New("aborted")
default:
return 0, fmt.Errorf("root chunk not found for %v", self.key.Hex())
}
}
self.chunk = chunk
return chunk.Size, nil
}
// read at can be called numerous times
// concurrent reads are allowed
// Size() needs to be called synchronously on the LazyChunkReader first
func (self *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) {
// this is correct, a swarm doc cannot be zero length, so no EOF is expected
if len(b) == 0 {
return 0, nil
}
quitC := make(chan bool)
size, err := self.Size(quitC)
if err != nil {
return 0, err
}
errC := make(chan error)
// }
var treeSize int64
var depth int
// calculate depth and max treeSize
treeSize = self.chunkSize
for ; treeSize < size; treeSize *= self.branches {
depth++
}
wg := sync.WaitGroup{}
wg.Add(1)
go self.join(b, off, off+int64(len(b)), depth, treeSize/self.branches, self.chunk, &wg, errC, quitC)
go func() {
wg.Wait()
close(errC)
}()
err = <-errC
if err != nil {
close(quitC)
return 0, err
}
if off+int64(len(b)) >= size {
return len(b), io.EOF
}
return len(b), nil
}
func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeSize int64, chunk *Chunk, parentWg *sync.WaitGroup, errC chan error, quitC chan bool) {
defer parentWg.Done()
// return NewDPA(&LocalStore{})
// chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8]))
// find appropriate block level
for chunk.Size < treeSize && depth > 0 {
treeSize /= self.branches
depth--
}
// leaf chunk found
if depth == 0 {
extra := 8 + eoff - int64(len(chunk.SData))
if extra > 0 {
eoff -= extra
}
copy(b, chunk.SData[8+off:8+eoff])
return // simply give back the chunks reader for content chunks
}
// subtree
start := off / treeSize
end := (eoff + treeSize - 1) / treeSize
wg := &sync.WaitGroup{}
defer wg.Wait()
for i := start; i < end; i++ {
soff := i * treeSize
roff := soff
seoff := soff + treeSize
if soff < off {
soff = off
}
if seoff > eoff {
seoff = eoff
}
if depth > 1 {
wg.Wait()
}
wg.Add(1)
go func(j int64) {
childKey := chunk.SData[8+j*self.hashSize : 8+(j+1)*self.hashSize]
chunk := retrieve(childKey, self.chunkC, quitC)
if chunk == nil {
select {
case errC <- fmt.Errorf("chunk %v-%v not found", off, off+treeSize):
case <-quitC:
}
return
}
if soff < off {
soff = off
}
self.join(b[soff-off:seoff-off], soff-roff, seoff-roff, depth-1, treeSize/self.branches, chunk, wg, errC, quitC)
}(i)
} //for
}
// the helper method submits chunks for a key to a oueue (DPA) and
// block until they time out or arrive
// abort if quitC is readable
func retrieve(key Key, chunkC chan *Chunk, quitC chan bool) *Chunk {
chunk := &Chunk{
Key: key,
C: make(chan bool), // close channel to signal data delivery
}
// submit chunk for retrieval
select {
case chunkC <- chunk: // submit retrieval request, someone should be listening on the other side (or we will time out globally)
case <-quitC:
return nil
}
// waiting for the chunk retrieval
select { // chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8]))
case <-quitC:
// this is how we control process leakage (quitC is closed once join is finished (after timeout))
return nil
case <-chunk.C: // bells are ringing, data have been delivered
}
if len(chunk.SData) == 0 {
return nil // chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8]))
}
return chunk
}
// Read keeps a cursor so cannot be called simulateously, see ReadAt
func (self *LazyChunkReader) Read(b []byte) (read int, err error) {
read, err = self.ReadAt(b, self.off)
self.off += int64(read)
return
}
// completely analogous to standard SectionReader implementation
var errWhence = errors.New("Seek: invalid whence")
var errOffset = errors.New("Seek: invalid offset")
func (s *LazyChunkReader) Seek(offset int64, whence int) (int64, error) {
switch whence {
default:
return 0, errWhence
case 0:
offset += 0
case 1:
offset += s.off
case 2:
if s.chunk == nil { //seek from the end requires rootchunk for size. call Size first
_, err := s.Size(nil)
if err != nil {
return 0, fmt.Errorf("can't get size: %v", err)
}
}
offset += s.chunk.Size
}
if offset < 0 {
return 0, errOffset
}
s.off = offset
return offset, nil
}

552
swarm/storage/chunker_test.go
View file

@ -1,552 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package storage
import (
"bytes"
"crypto/rand"
"encoding/binary"
"errors"
"fmt"
"io"
"sync"
"testing"
"time"
"github.com/XinFinOrg/XDPoSChain/crypto/sha3"
)
/*
Tests TreeChunker by splitting and joining a random byte slice
*/
type test interface {
Fatalf(string, ...interface{})
Logf(string, ...interface{})
}
type chunkerTester struct {
inputs map[uint64][]byte
chunks map[string]*Chunk
t test
}
func (self *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, chunkC chan *Chunk, swg *sync.WaitGroup, expectedError error) (key Key, err error) {
// reset
self.chunks = make(map[string]*Chunk)
if self.inputs == nil {
self.inputs = make(map[uint64][]byte)
}
quitC := make(chan bool)
timeout := time.After(600 * time.Second)
if chunkC != nil {
go func() error {
for {
select {
case <-timeout:
return errors.New("Split timeout error")
case <-quitC:
return nil
case chunk := <-chunkC:
// self.chunks = append(self.chunks, chunk)
self.chunks[chunk.Key.String()] = chunk
if chunk.wg != nil {
chunk.wg.Done()
}
}
}
}()
}
key, err = chunker.Split(data, size, chunkC, swg, nil)
if err != nil && expectedError == nil {
err = fmt.Errorf("Split error: %v", err)
}
if chunkC != nil {
if swg != nil {
swg.Wait()
}
close(quitC)
}
return key, err
}
func (self *chunkerTester) Append(chunker Splitter, rootKey Key, data io.Reader, chunkC chan *Chunk, swg *sync.WaitGroup, expectedError error) (key Key, err error) {
quitC := make(chan bool)
timeout := time.After(60 * time.Second)
if chunkC != nil {
go func() error {
for {
select {
case <-timeout:
return errors.New("Append timeout error")
case <-quitC:
return nil
case chunk := <-chunkC:
if chunk != nil {
stored, success := self.chunks[chunk.Key.String()]
if !success {
// Requesting data
self.chunks[chunk.Key.String()] = chunk
if chunk.wg != nil {
chunk.wg.Done()
}
} else {
// getting data
chunk.SData = stored.SData
chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8]))
close(chunk.C)
}
}
}
}
}()
}
key, err = chunker.Append(rootKey, data, chunkC, swg, nil)
if err != nil && expectedError == nil {
err = fmt.Errorf("Append error: %v", err)
}
if chunkC != nil {
if swg != nil {
swg.Wait()
}
close(quitC)
}
return key, err
}
func (self *chunkerTester) Join(chunker Chunker, key Key, c int, chunkC chan *Chunk, quitC chan bool) LazySectionReader {
// reset but not the chunks
reader := chunker.Join(key, chunkC)
timeout := time.After(600 * time.Second)
i := 0
go func() error {
for {
select {
case <-timeout:
return errors.New("Join timeout error")
case chunk, ok := <-chunkC:
if !ok {
close(quitC)
return nil
}
// this just mocks the behaviour of a chunk store retrieval
stored, success := self.chunks[chunk.Key.String()]
if !success {
return errors.New("Not found")
}
chunk.SData = stored.SData
chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8]))
close(chunk.C)
i++
}
}
}()
return reader
}
func testRandomBrokenData(splitter Splitter, n int, tester *chunkerTester) {
data := io.LimitReader(rand.Reader, int64(n))
brokendata := brokenLimitReader(data, n, n/2)
buf := make([]byte, n)
_, err := brokendata.Read(buf)
if err == nil || err.Error() != "Broken reader" {
tester.t.Fatalf("Broken reader is not broken, hence broken. Returns: %v", err)
}
data = io.LimitReader(rand.Reader, int64(n))
brokendata = brokenLimitReader(data, n, n/2)
chunkC := make(chan *Chunk, 1000)
swg := &sync.WaitGroup{}
expectedError := fmt.Errorf("Broken reader")
key, err := tester.Split(splitter, brokendata, int64(n), chunkC, swg, expectedError)
if err == nil || err.Error() != expectedError.Error() {
tester.t.Fatalf("Not receiving the correct error! Expected %v, received %v", expectedError, err)
}
tester.t.Logf(" Key = %v\n", key)
}
func testRandomData(splitter Splitter, n int, tester *chunkerTester) Key {
if tester.inputs == nil {
tester.inputs = make(map[uint64][]byte)
}
input, found := tester.inputs[uint64(n)]
var data io.Reader
if !found {
data, input = testDataReaderAndSlice(n)
tester.inputs[uint64(n)] = input
} else {
data = io.LimitReader(bytes.NewReader(input), int64(n))
}
chunkC := make(chan *Chunk, 1000)
swg := &sync.WaitGroup{}
key, err := tester.Split(splitter, data, int64(n), chunkC, swg, nil)
if err != nil {
tester.t.Fatalf(err.Error())
}
tester.t.Logf(" Key = %v\n", key)
chunkC = make(chan *Chunk, 1000)
quitC := make(chan bool)
chunker := NewTreeChunker(NewChunkerParams())
reader := tester.Join(chunker, key, 0, chunkC, quitC)
output := make([]byte, n)
r, err := reader.Read(output)
if r != n || err != io.EOF {
tester.t.Fatalf("read error read: %v n = %v err = %v\n", r, n, err)
}
if input != nil {
if !bytes.Equal(output, input) {
tester.t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", input, output)
}
}
close(chunkC)
<-quitC
return key
}
func testRandomDataAppend(splitter Splitter, n, m int, tester *chunkerTester) {
if tester.inputs == nil {
tester.inputs = make(map[uint64][]byte)
}
input, found := tester.inputs[uint64(n)]
var data io.Reader
if !found {
data, input = testDataReaderAndSlice(n)
tester.inputs[uint64(n)] = input
} else {
data = io.LimitReader(bytes.NewReader(input), int64(n))
}
chunkC := make(chan *Chunk, 1000)
swg := &sync.WaitGroup{}
key, err := tester.Split(splitter, data, int64(n), chunkC, swg, nil)
if err != nil {
tester.t.Fatalf(err.Error())
}
tester.t.Logf(" Key = %v\n", key)
//create a append data stream
appendInput, found := tester.inputs[uint64(m)]
var appendData io.Reader
if !found {
appendData, appendInput = testDataReaderAndSlice(m)
tester.inputs[uint64(m)] = appendInput
} else {
appendData = io.LimitReader(bytes.NewReader(appendInput), int64(m))
}
chunkC = make(chan *Chunk, 1000)
swg = &sync.WaitGroup{}
newKey, err := tester.Append(splitter, key, appendData, chunkC, swg, nil)
if err != nil {
tester.t.Fatalf(err.Error())
}
tester.t.Logf(" NewKey = %v\n", newKey)
chunkC = make(chan *Chunk, 1000)
quitC := make(chan bool)
chunker := NewTreeChunker(NewChunkerParams())
reader := tester.Join(chunker, newKey, 0, chunkC, quitC)
newOutput := make([]byte, n+m)
r, err := reader.Read(newOutput)
if r != (n + m) {
tester.t.Fatalf("read error read: %v n = %v err = %v\n", r, n, err)
}
newInput := append(input, appendInput...)
if !bytes.Equal(newOutput, newInput) {
tester.t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", newInput, newOutput)
}
close(chunkC)
}
func TestSha3ForCorrectness(t *testing.T) {
tester := &chunkerTester{t: t}
size := 4096
input := make([]byte, size+8)
binary.LittleEndian.PutUint64(input[:8], uint64(size))
io.LimitReader(bytes.NewReader(input[8:]), int64(size))
rawSha3 := sha3.NewKeccak256()
rawSha3.Reset()
rawSha3.Write(input)
rawSha3Output := rawSha3.Sum(nil)
sha3FromMakeFunc := MakeHashFunc(SHA3Hash)()
sha3FromMakeFunc.ResetWithLength(input[:8])
sha3FromMakeFunc.Write(input[8:])
sha3FromMakeFuncOutput := sha3FromMakeFunc.Sum(nil)
if len(rawSha3Output) != len(sha3FromMakeFuncOutput) {
tester.t.Fatalf("Original SHA3 and abstracted Sha3 has different length %v:%v\n", len(rawSha3Output), len(sha3FromMakeFuncOutput))
}
if !bytes.Equal(rawSha3Output, sha3FromMakeFuncOutput) {
tester.t.Fatalf("Original SHA3 and abstracted Sha3 mismatch %v:%v\n", rawSha3Output, sha3FromMakeFuncOutput)
}
}
func TestDataAppend(t *testing.T) {
sizes := []int{1, 1, 1, 4095, 4096, 4097, 1, 1, 1, 123456, 2345678, 2345678}
appendSizes := []int{4095, 4096, 4097, 1, 1, 1, 8191, 8192, 8193, 9000, 3000, 5000}
tester := &chunkerTester{t: t}
chunker := NewPyramidChunker(NewChunkerParams())
for i, s := range sizes {
testRandomDataAppend(chunker, s, appendSizes[i], tester)
}
}
func TestRandomData(t *testing.T) {
sizes := []int{1, 60, 83, 179, 253, 1024, 4095, 4096, 4097, 8191, 8192, 8193, 12287, 12288, 12289, 123456, 2345678}
tester := &chunkerTester{t: t}
chunker := NewTreeChunker(NewChunkerParams())
pyramid := NewPyramidChunker(NewChunkerParams())
for _, s := range sizes {
treeChunkerKey := testRandomData(chunker, s, tester)
pyramidChunkerKey := testRandomData(pyramid, s, tester)
if treeChunkerKey.String() != pyramidChunkerKey.String() {
tester.t.Fatalf("tree chunker and pyramid chunker key mismatch for size %v\n TC: %v\n PC: %v\n", s, treeChunkerKey.String(), pyramidChunkerKey.String())
}
}
cp := NewChunkerParams()
cp.Hash = BMTHash
chunker = NewTreeChunker(cp)
pyramid = NewPyramidChunker(cp)
for _, s := range sizes {
treeChunkerKey := testRandomData(chunker, s, tester)
pyramidChunkerKey := testRandomData(pyramid, s, tester)
if treeChunkerKey.String() != pyramidChunkerKey.String() {
tester.t.Fatalf("tree chunker BMT and pyramid chunker BMT key mismatch for size %v \n TC: %v\n PC: %v\n", s, treeChunkerKey.String(), pyramidChunkerKey.String())
}
}
}
func XTestRandomBrokenData(t *testing.T) {
sizes := []int{1, 60, 83, 179, 253, 1024, 4095, 4096, 4097, 8191, 8192, 8193, 12287, 12288, 12289, 123456, 2345678}
tester := &chunkerTester{t: t}
chunker := NewTreeChunker(NewChunkerParams())
for _, s := range sizes {
testRandomBrokenData(chunker, s, tester)
}
}
func benchReadAll(reader LazySectionReader) {
size, _ := reader.Size(nil)
output := make([]byte, 1000)
for pos := int64(0); pos < size; pos += 1000 {
reader.ReadAt(output, pos)
}
}
func benchmarkJoin(n int, t *testing.B) {
t.ReportAllocs()
for i := 0; i < t.N; i++ {
chunker := NewTreeChunker(NewChunkerParams())
tester := &chunkerTester{t: t}
data := testDataReader(n)
chunkC := make(chan *Chunk, 1000)
swg := &sync.WaitGroup{}
key, err := tester.Split(chunker, data, int64(n), chunkC, swg, nil)
if err != nil {
tester.t.Fatalf(err.Error())
}
chunkC = make(chan *Chunk, 1000)
quitC := make(chan bool)
reader := tester.Join(chunker, key, i, chunkC, quitC)
benchReadAll(reader)
close(chunkC)
<-quitC
}
}
func benchmarkSplitTreeSHA3(n int, t *testing.B) {
t.ReportAllocs()
for i := 0; i < t.N; i++ {
chunker := NewTreeChunker(NewChunkerParams())
tester := &chunkerTester{t: t}
data := testDataReader(n)
_, err := tester.Split(chunker, data, int64(n), nil, nil, nil)
if err != nil {
tester.t.Fatalf(err.Error())
}
}
}
func benchmarkSplitTreeBMT(n int, t *testing.B) {
t.ReportAllocs()
for i := 0; i < t.N; i++ {
cp := NewChunkerParams()
cp.Hash = BMTHash
chunker := NewTreeChunker(cp)
tester := &chunkerTester{t: t}
data := testDataReader(n)
_, err := tester.Split(chunker, data, int64(n), nil, nil, nil)
if err != nil {
tester.t.Fatalf(err.Error())
}
}
}
func benchmarkSplitPyramidSHA3(n int, t *testing.B) {
t.ReportAllocs()
for i := 0; i < t.N; i++ {
splitter := NewPyramidChunker(NewChunkerParams())
tester := &chunkerTester{t: t}
data := testDataReader(n)
_, err := tester.Split(splitter, data, int64(n), nil, nil, nil)
if err != nil {
tester.t.Fatalf(err.Error())
}
}
}
func benchmarkSplitPyramidBMT(n int, t *testing.B) {
t.ReportAllocs()
for i := 0; i < t.N; i++ {
cp := NewChunkerParams()
cp.Hash = BMTHash
splitter := NewPyramidChunker(cp)
tester := &chunkerTester{t: t}
data := testDataReader(n)
_, err := tester.Split(splitter, data, int64(n), nil, nil, nil)
if err != nil {
tester.t.Fatalf(err.Error())
}
}
}
func benchmarkAppendPyramid(n, m int, t *testing.B) {
t.ReportAllocs()
for i := 0; i < t.N; i++ {
chunker := NewPyramidChunker(NewChunkerParams())
tester := &chunkerTester{t: t}
data := testDataReader(n)
data1 := testDataReader(m)
chunkC := make(chan *Chunk, 1000)
swg := &sync.WaitGroup{}
key, err := tester.Split(chunker, data, int64(n), chunkC, swg, nil)
if err != nil {
tester.t.Fatalf(err.Error())
}
chunkC = make(chan *Chunk, 1000)
swg = &sync.WaitGroup{}
_, err = tester.Append(chunker, key, data1, chunkC, swg, nil)
if err != nil {
tester.t.Fatalf(err.Error())
}
close(chunkC)
}
}
func BenchmarkJoin_2(t *testing.B) { benchmarkJoin(100, t) }
func BenchmarkJoin_3(t *testing.B) { benchmarkJoin(1000, t) }
func BenchmarkJoin_4(t *testing.B) { benchmarkJoin(10000, t) }
func BenchmarkJoin_5(t *testing.B) { benchmarkJoin(100000, t) }
func BenchmarkJoin_6(t *testing.B) { benchmarkJoin(1000000, t) }
func BenchmarkJoin_7(t *testing.B) { benchmarkJoin(10000000, t) }
func BenchmarkJoin_8(t *testing.B) { benchmarkJoin(100000000, t) }
func BenchmarkSplitTreeSHA3_2(t *testing.B) { benchmarkSplitTreeSHA3(100, t) }
func BenchmarkSplitTreeSHA3_2h(t *testing.B) { benchmarkSplitTreeSHA3(500, t) }
func BenchmarkSplitTreeSHA3_3(t *testing.B) { benchmarkSplitTreeSHA3(1000, t) }
func BenchmarkSplitTreeSHA3_3h(t *testing.B) { benchmarkSplitTreeSHA3(5000, t) }
func BenchmarkSplitTreeSHA3_4(t *testing.B) { benchmarkSplitTreeSHA3(10000, t) }
func BenchmarkSplitTreeSHA3_4h(t *testing.B) { benchmarkSplitTreeSHA3(50000, t) }
func BenchmarkSplitTreeSHA3_5(t *testing.B) { benchmarkSplitTreeSHA3(100000, t) }
func BenchmarkSplitTreeSHA3_6(t *testing.B) { benchmarkSplitTreeSHA3(1000000, t) }
func BenchmarkSplitTreeSHA3_7(t *testing.B) { benchmarkSplitTreeSHA3(10000000, t) }
func BenchmarkSplitTreeSHA3_8(t *testing.B) { benchmarkSplitTreeSHA3(100000000, t) }
func BenchmarkSplitTreeBMT_2(t *testing.B) { benchmarkSplitTreeBMT(100, t) }
func BenchmarkSplitTreeBMT_2h(t *testing.B) { benchmarkSplitTreeBMT(500, t) }
func BenchmarkSplitTreeBMT_3(t *testing.B) { benchmarkSplitTreeBMT(1000, t) }
func BenchmarkSplitTreeBMT_3h(t *testing.B) { benchmarkSplitTreeBMT(5000, t) }
func BenchmarkSplitTreeBMT_4(t *testing.B) { benchmarkSplitTreeBMT(10000, t) }
func BenchmarkSplitTreeBMT_4h(t *testing.B) { benchmarkSplitTreeBMT(50000, t) }
func BenchmarkSplitTreeBMT_5(t *testing.B) { benchmarkSplitTreeBMT(100000, t) }
func BenchmarkSplitTreeBMT_6(t *testing.B) { benchmarkSplitTreeBMT(1000000, t) }
func BenchmarkSplitTreeBMT_7(t *testing.B) { benchmarkSplitTreeBMT(10000000, t) }
func BenchmarkSplitTreeBMT_8(t *testing.B) { benchmarkSplitTreeBMT(100000000, t) }
func BenchmarkSplitPyramidSHA3_2(t *testing.B) { benchmarkSplitPyramidSHA3(100, t) }
func BenchmarkSplitPyramidSHA3_2h(t *testing.B) { benchmarkSplitPyramidSHA3(500, t) }
func BenchmarkSplitPyramidSHA3_3(t *testing.B) { benchmarkSplitPyramidSHA3(1000, t) }
func BenchmarkSplitPyramidSHA3_3h(t *testing.B) { benchmarkSplitPyramidSHA3(5000, t) }
func BenchmarkSplitPyramidSHA3_4(t *testing.B) { benchmarkSplitPyramidSHA3(10000, t) }
func BenchmarkSplitPyramidSHA3_4h(t *testing.B) { benchmarkSplitPyramidSHA3(50000, t) }
func BenchmarkSplitPyramidSHA3_5(t *testing.B) { benchmarkSplitPyramidSHA3(100000, t) }
func BenchmarkSplitPyramidSHA3_6(t *testing.B) { benchmarkSplitPyramidSHA3(1000000, t) }
func BenchmarkSplitPyramidSHA3_7(t *testing.B) { benchmarkSplitPyramidSHA3(10000000, t) }
func BenchmarkSplitPyramidSHA3_8(t *testing.B) { benchmarkSplitPyramidSHA3(100000000, t) }
func BenchmarkSplitPyramidBMT_2(t *testing.B) { benchmarkSplitPyramidBMT(100, t) }
func BenchmarkSplitPyramidBMT_2h(t *testing.B) { benchmarkSplitPyramidBMT(500, t) }
func BenchmarkSplitPyramidBMT_3(t *testing.B) { benchmarkSplitPyramidBMT(1000, t) }
func BenchmarkSplitPyramidBMT_3h(t *testing.B) { benchmarkSplitPyramidBMT(5000, t) }
func BenchmarkSplitPyramidBMT_4(t *testing.B) { benchmarkSplitPyramidBMT(10000, t) }
func BenchmarkSplitPyramidBMT_4h(t *testing.B) { benchmarkSplitPyramidBMT(50000, t) }
func BenchmarkSplitPyramidBMT_5(t *testing.B) { benchmarkSplitPyramidBMT(100000, t) }
func BenchmarkSplitPyramidBMT_6(t *testing.B) { benchmarkSplitPyramidBMT(1000000, t) }
func BenchmarkSplitPyramidBMT_7(t *testing.B) { benchmarkSplitPyramidBMT(10000000, t) }
func BenchmarkSplitPyramidBMT_8(t *testing.B) { benchmarkSplitPyramidBMT(100000000, t) }
func BenchmarkAppendPyramid_2(t *testing.B) { benchmarkAppendPyramid(100, 1000, t) }
func BenchmarkAppendPyramid_2h(t *testing.B) { benchmarkAppendPyramid(500, 1000, t) }
func BenchmarkAppendPyramid_3(t *testing.B) { benchmarkAppendPyramid(1000, 1000, t) }
func BenchmarkAppendPyramid_4(t *testing.B) { benchmarkAppendPyramid(10000, 1000, t) }
func BenchmarkAppendPyramid_4h(t *testing.B) { benchmarkAppendPyramid(50000, 1000, t) }
func BenchmarkAppendPyramid_5(t *testing.B) { benchmarkAppendPyramid(1000000, 1000, t) }
func BenchmarkAppendPyramid_6(t *testing.B) { benchmarkAppendPyramid(1000000, 1000, t) }
func BenchmarkAppendPyramid_7(t *testing.B) { benchmarkAppendPyramid(10000000, 1000, t) }
func BenchmarkAppendPyramid_8(t *testing.B) { benchmarkAppendPyramid(100000000, 1000, t) }
// go test -timeout 20m -cpu 4 -bench=./swarm/storage -run no
// If you dont add the timeout argument above .. the benchmark will timeout and dump

116
swarm/storage/common_test.go
View file

@ -1,116 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package storage
import (
"bytes"
"crypto/rand"
"fmt"
"io"
"sync"
"testing"
"github.com/XinFinOrg/XDPoSChain/log"
)
type brokenLimitedReader struct {
lr io.Reader
errAt int
off int
size int
}
func brokenLimitReader(data io.Reader, size int, errAt int) *brokenLimitedReader {
return &brokenLimitedReader{
lr: data,
errAt: errAt,
size: size,
}
}
func testDataReader(l int) (r io.Reader) {
return io.LimitReader(rand.Reader, int64(l))
}
func (self *brokenLimitedReader) Read(buf []byte) (int, error) {
if self.off+len(buf) > self.errAt {
return 0, fmt.Errorf("Broken reader")
}
self.off += len(buf)
return self.lr.Read(buf)
}
func testDataReaderAndSlice(l int) (r io.Reader, slice []byte) {
slice = make([]byte, l)
if _, err := rand.Read(slice); err != nil {
panic("rand error")
}
r = io.LimitReader(bytes.NewReader(slice), int64(l))
return
}
func testStore(m ChunkStore, l int64, branches int64, t *testing.T) {
chunkC := make(chan *Chunk)
go func() {
for chunk := range chunkC {
m.Put(chunk)
if chunk.wg != nil {
chunk.wg.Done()
}
}
}()
chunker := NewTreeChunker(&ChunkerParams{
Branches: branches,
Hash: SHA3Hash,
})
swg := &sync.WaitGroup{}
key, _ := chunker.Split(rand.Reader, l, chunkC, swg, nil)
swg.Wait()
close(chunkC)
chunkC = make(chan *Chunk)
quit := make(chan bool)
go func() {
for ch := range chunkC {
go func(chunk *Chunk) {
storedChunk, err := m.Get(chunk.Key)
if err == notFound {
log.Trace(fmt.Sprintf("chunk '%v' not found", chunk.Key.Log()))
} else if err != nil {
log.Trace(fmt.Sprintf("error retrieving chunk %v: %v", chunk.Key.Log(), err))
} else {
chunk.SData = storedChunk.SData
chunk.Size = storedChunk.Size
}
log.Trace(fmt.Sprintf("chunk '%v' not found", chunk.Key.Log()))
close(chunk.C)
}(ch)
}
close(quit)
}()
r := chunker.Join(key, chunkC)
b := make([]byte, l)
n, err := r.ReadAt(b, 0)
if err != io.EOF {
t.Fatalf("read error (%v/%v) %v", n, l, err)
}
close(chunkC)
<-quit
}

99
swarm/storage/database.go
View file

@ -1,99 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package storage
// this is a clone of an earlier state of the ethereum ethdb/database
// no need for queueing/caching
import (
"fmt"
"github.com/XinFinOrg/XDPoSChain/compression/rle"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/iterator"
"github.com/syndtr/goleveldb/leveldb/opt"
)
const openFileLimit = 128
type LDBDatabase struct {
db *leveldb.DB
comp bool
}
func NewLDBDatabase(file string) (*LDBDatabase, error) {
// Open the db
db, err := leveldb.OpenFile(file, &opt.Options{OpenFilesCacheCapacity: openFileLimit})
if err != nil {
return nil, err
}
database := &LDBDatabase{db: db, comp: false}
return database, nil
}
func (self *LDBDatabase) Put(key []byte, value []byte) {
if self.comp {
value = rle.Compress(value)
}
err := self.db.Put(key, value, nil)
if err != nil {
fmt.Println("Error put", err)
}
}
func (self *LDBDatabase) Get(key []byte) ([]byte, error) {
dat, err := self.db.Get(key, nil)
if err != nil {
return nil, err
}
if self.comp {
return rle.Decompress(dat)
}
return dat, nil
}
func (self *LDBDatabase) Delete(key []byte) error {
return self.db.Delete(key, nil)
}
func (self *LDBDatabase) LastKnownTD() []byte {
data, _ := self.Get([]byte("LTD"))
if len(data) == 0 {
data = []byte{0x0}
}
return data
}
func (self *LDBDatabase) NewIterator() iterator.Iterator {
return self.db.NewIterator(nil, nil)
}
func (self *LDBDatabase) Write(batch *leveldb.Batch) error {
return self.db.Write(batch, nil)
}
func (self *LDBDatabase) Close() {
// Close the leveldb database
self.db.Close()
}

601
swarm/storage/dbstore.go
View file

@ -1,601 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// disk storage layer for the package bzz
// DbStore implements the ChunkStore interface and is used by the DPA as
// persistent storage of chunks
// it implements purging based on access count allowing for external control of
// max capacity
package storage
import (
"archive/tar"
"bytes"
"encoding/binary"
"encoding/hex"
"fmt"
"io"
"sync"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/metrics"
"github.com/XinFinOrg/XDPoSChain/rlp"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/iterator"
)
// metrics variables
var (
gcCounter = metrics.NewRegisteredCounter("storage.db.dbstore.gc.count", nil)
dbStoreDeleteCounter = metrics.NewRegisteredCounter("storage.db.dbstore.rm.count", nil)
)
const (
defaultDbCapacity = 5000000
defaultRadius = 0 // not yet used
gcArraySize = 10000
gcArrayFreeRatio = 0.1
// key prefixes for leveldb storage
kpIndex = 0
kpData = 1
)
var (
keyAccessCnt = []byte{2}
keyEntryCnt = []byte{3}
keyDataIdx = []byte{4}
keyGCPos = []byte{5}
)
type gcItem struct {
idx uint64
value uint64
idxKey []byte
}
type DbStore struct {
db *LDBDatabase
// this should be stored in db, accessed transactionally
entryCnt, accessCnt, dataIdx, capacity uint64
gcPos, gcStartPos []byte
gcArray []*gcItem
hashfunc SwarmHasher
lock sync.Mutex
}
func NewDbStore(path string, hash SwarmHasher, capacity uint64, radius int) (s *DbStore, err error) {
s = new(DbStore)
s.hashfunc = hash
s.db, err = NewLDBDatabase(path)
if err != nil {
return
}
s.setCapacity(capacity)
s.gcStartPos = make([]byte, 1)
s.gcStartPos[0] = kpIndex
s.gcArray = make([]*gcItem, gcArraySize)
data, _ := s.db.Get(keyEntryCnt)
s.entryCnt = BytesToU64(data)
data, _ = s.db.Get(keyAccessCnt)
s.accessCnt = BytesToU64(data)
data, _ = s.db.Get(keyDataIdx)
s.dataIdx = BytesToU64(data)
s.gcPos, _ = s.db.Get(keyGCPos)
if s.gcPos == nil {
s.gcPos = s.gcStartPos
}
return
}
type dpaDBIndex struct {
Idx uint64
Access uint64
}
func BytesToU64(data []byte) uint64 {
if len(data) < 8 {
return 0
}
return binary.LittleEndian.Uint64(data)
}
func U64ToBytes(val uint64) []byte {
data := make([]byte, 8)
binary.LittleEndian.PutUint64(data, val)
return data
}
func getIndexGCValue(index *dpaDBIndex) uint64 {
return index.Access
}
func (s *DbStore) updateIndexAccess(index *dpaDBIndex) {
index.Access = s.accessCnt
}
func getIndexKey(hash Key) []byte {
HashSize := len(hash)
key := make([]byte, HashSize+1)
key[0] = 0
copy(key[1:], hash[:])
return key
}
func getDataKey(idx uint64) []byte {
key := make([]byte, 9)
key[0] = 1
binary.BigEndian.PutUint64(key[1:9], idx)
return key
}
func encodeIndex(index *dpaDBIndex) []byte {
data, _ := rlp.EncodeToBytes(index)
return data
}
func encodeData(chunk *Chunk) []byte {
return chunk.SData
}
func decodeIndex(data []byte, index *dpaDBIndex) {
dec := rlp.NewStream(bytes.NewReader(data), 0)
dec.Decode(index)
}
func decodeData(data []byte, chunk *Chunk) {
chunk.SData = data
chunk.Size = int64(binary.LittleEndian.Uint64(data[0:8]))
}
func gcListPartition(list []*gcItem, left int, right int, pivotIndex int) int {
pivotValue := list[pivotIndex].value
dd := list[pivotIndex]
list[pivotIndex] = list[right]
list[right] = dd
storeIndex := left
for i := left; i < right; i++ {
if list[i].value < pivotValue {
dd = list[storeIndex]
list[storeIndex] = list[i]
list[i] = dd
storeIndex++
}
}
dd = list[storeIndex]
list[storeIndex] = list[right]
list[right] = dd
return storeIndex
}
func gcListSelect(list []*gcItem, left int, right int, n int) int {
if left == right {
return left
}
pivotIndex := (left + right) / 2
pivotIndex = gcListPartition(list, left, right, pivotIndex)
if n == pivotIndex {
return n
} else {
if n < pivotIndex {
return gcListSelect(list, left, pivotIndex-1, n)
} else {
return gcListSelect(list, pivotIndex+1, right, n)
}
}
}
func (s *DbStore) collectGarbage(ratio float32) {
it := s.db.NewIterator()
it.Seek(s.gcPos)
if it.Valid() {
s.gcPos = it.Key()
} else {
s.gcPos = nil
}
gcnt := 0
for (gcnt < gcArraySize) && (uint64(gcnt) < s.entryCnt) {
if (s.gcPos == nil) || (s.gcPos[0] != kpIndex) {
it.Seek(s.gcStartPos)
if it.Valid() {
s.gcPos = it.Key()
} else {
s.gcPos = nil
}
}
if (s.gcPos == nil) || (s.gcPos[0] != kpIndex) {
break
}
gci := new(gcItem)
gci.idxKey = s.gcPos
var index dpaDBIndex
decodeIndex(it.Value(), &index)
gci.idx = index.Idx
// the smaller, the more likely to be gc'd
gci.value = getIndexGCValue(&index)
s.gcArray[gcnt] = gci
gcnt++
it.Next()
if it.Valid() {
s.gcPos = it.Key()
} else {
s.gcPos = nil
}
}
it.Release()
cutidx := gcListSelect(s.gcArray, 0, gcnt-1, int(float32(gcnt)*ratio))
cutval := s.gcArray[cutidx].value
// fmt.Print(gcnt, " ", s.entryCnt, " ")
// actual gc
for i := 0; i < gcnt; i++ {
if s.gcArray[i].value <= cutval {
gcCounter.Inc(1)
s.delete(s.gcArray[i].idx, s.gcArray[i].idxKey)
}
}
// fmt.Println(s.entryCnt)
s.db.Put(keyGCPos, s.gcPos)
}
// Export writes all chunks from the store to a tar archive, returning the
// number of chunks written.
func (s *DbStore) Export(out io.Writer) (int64, error) {
tw := tar.NewWriter(out)
defer tw.Close()
it := s.db.NewIterator()
defer it.Release()
var count int64
for ok := it.Seek([]byte{kpIndex}); ok; ok = it.Next() {
key := it.Key()
if (key == nil) || (key[0] != kpIndex) {
break
}
var index dpaDBIndex
decodeIndex(it.Value(), &index)
data, err := s.db.Get(getDataKey(index.Idx))
if err != nil {
log.Warn(fmt.Sprintf("Chunk %x found but could not be accessed: %v", key[:], err))
continue
}
hdr := &tar.Header{
Name: hex.EncodeToString(key[1:]),
Mode: 0644,
Size: int64(len(data)),
}
if err := tw.WriteHeader(hdr); err != nil {
return count, err
}
if _, err := tw.Write(data); err != nil {
return count, err
}
count++
}
return count, nil
}
// Import reads chunks into the store from a tar archive, returning the number
// of chunks read.
func (s *DbStore) Import(in io.Reader) (int64, error) {
tr := tar.NewReader(in)
var count int64
for {
hdr, err := tr.Next()
if err == io.EOF {
break
} else if err != nil {
return count, err
}
if len(hdr.Name) != 64 {
log.Warn("ignoring non-chunk file", "name", hdr.Name)
continue
}
key, err := hex.DecodeString(hdr.Name)
if err != nil {
log.Warn("ignoring invalid chunk file", "name", hdr.Name, "err", err)
continue
}
data, err := io.ReadAll(tr)
if err != nil {
return count, err
}
s.Put(&Chunk{Key: key, SData: data})
count++
}
return count, nil
}
func (s *DbStore) Cleanup() {
//Iterates over the database and checks that there are no faulty chunks
it := s.db.NewIterator()
startPosition := []byte{kpIndex}
it.Seek(startPosition)
var key []byte
var errorsFound, total int
for it.Valid() {
key = it.Key()
if (key == nil) || (key[0] != kpIndex) {
break
}
total++
var index dpaDBIndex
decodeIndex(it.Value(), &index)
data, err := s.db.Get(getDataKey(index.Idx))
if err != nil {
log.Warn(fmt.Sprintf("Chunk %x found but could not be accessed: %v", key[:], err))
s.delete(index.Idx, getIndexKey(key[1:]))
errorsFound++
} else {
hasher := s.hashfunc()
hasher.Write(data)
hash := hasher.Sum(nil)
if !bytes.Equal(hash, key[1:]) {
log.Warn(fmt.Sprintf("Found invalid chunk. Hash mismatch. hash=%x, key=%x", hash, key[:]))
s.delete(index.Idx, getIndexKey(key[1:]))
errorsFound++
}
}
it.Next()
}
it.Release()
log.Warn(fmt.Sprintf("Found %v errors out of %v entries", errorsFound, total))
}
func (s *DbStore) delete(idx uint64, idxKey []byte) {
batch := new(leveldb.Batch)
batch.Delete(idxKey)
batch.Delete(getDataKey(idx))
dbStoreDeleteCounter.Inc(1)
s.entryCnt--
batch.Put(keyEntryCnt, U64ToBytes(s.entryCnt))
s.db.Write(batch)
}
func (s *DbStore) Counter() uint64 {
s.lock.Lock()
defer s.lock.Unlock()
return s.dataIdx
}
func (s *DbStore) Put(chunk *Chunk) {
s.lock.Lock()
defer s.lock.Unlock()
ikey := getIndexKey(chunk.Key)
var index dpaDBIndex
if s.tryAccessIdx(ikey, &index) {
if chunk.dbStored != nil {
close(chunk.dbStored)
}
log.Trace(fmt.Sprintf("Storing to DB: chunk already exists, only update access"))
return // already exists, only update access
}
data := encodeData(chunk)
//data := ethutil.Encode([]interface{}{entry})
if s.entryCnt >= s.capacity {
s.collectGarbage(gcArrayFreeRatio)
}
batch := new(leveldb.Batch)
batch.Put(getDataKey(s.dataIdx), data)
index.Idx = s.dataIdx
s.updateIndexAccess(&index)
idata := encodeIndex(&index)
batch.Put(ikey, idata)
batch.Put(keyEntryCnt, U64ToBytes(s.entryCnt))
s.entryCnt++
batch.Put(keyDataIdx, U64ToBytes(s.dataIdx))
s.dataIdx++
batch.Put(keyAccessCnt, U64ToBytes(s.accessCnt))
s.accessCnt++
s.db.Write(batch)
if chunk.dbStored != nil {
close(chunk.dbStored)
}
log.Trace(fmt.Sprintf("DbStore.Put: %v. db storage counter: %v ", chunk.Key.Log(), s.dataIdx))
}
// try to find index; if found, update access cnt and return true
func (s *DbStore) tryAccessIdx(ikey []byte, index *dpaDBIndex) bool {
idata, err := s.db.Get(ikey)
if err != nil {
return false
}
decodeIndex(idata, index)
batch := new(leveldb.Batch)
batch.Put(keyAccessCnt, U64ToBytes(s.accessCnt))
s.accessCnt++
s.updateIndexAccess(index)
idata = encodeIndex(index)
batch.Put(ikey, idata)
s.db.Write(batch)
return true
}
func (s *DbStore) Get(key Key) (chunk *Chunk, err error) {
s.lock.Lock()
defer s.lock.Unlock()
var index dpaDBIndex
if s.tryAccessIdx(getIndexKey(key), &index) {
var data []byte
data, err = s.db.Get(getDataKey(index.Idx))
if err != nil {
log.Trace(fmt.Sprintf("DBStore: Chunk %v found but could not be accessed: %v", key.Log(), err))
s.delete(index.Idx, getIndexKey(key))
return
}
hasher := s.hashfunc()
hasher.Write(data)
hash := hasher.Sum(nil)
if !bytes.Equal(hash, key) {
s.delete(index.Idx, getIndexKey(key))
log.Warn("Invalid Chunk in Database. Please repair with command: 'swarm cleandb'")
}
chunk = &Chunk{
Key: key,
}
decodeData(data, chunk)
} else {
err = notFound
}
return
}
func (s *DbStore) updateAccessCnt(key Key) {
s.lock.Lock()
defer s.lock.Unlock()
var index dpaDBIndex
s.tryAccessIdx(getIndexKey(key), &index) // result_chn == nil, only update access cnt
}
func (s *DbStore) setCapacity(c uint64) {
s.lock.Lock()
defer s.lock.Unlock()
s.capacity = c
if s.entryCnt > c {
ratio := float32(1.01) - float32(c)/float32(s.entryCnt)
if ratio < gcArrayFreeRatio {
ratio = gcArrayFreeRatio
}
if ratio > 1 {
ratio = 1
}
for s.entryCnt > c {
s.collectGarbage(ratio)
}
}
}
func (s *DbStore) Close() {
s.db.Close()
}
// describes a section of the DbStore representing the unsynced
//
// domain relevant to a peer
// Start - Stop designate a continuous area Keys in an address space
// typically the addresses closer to us than to the peer but not closer
// another closer peer in between
// From - To designates a time interval typically from the last disconnect
// till the latest connection (real time traffic is relayed)
type DbSyncState struct {
Start, Stop Key
First, Last uint64
}
// implements the syncer iterator interface
// iterates by storage index (~ time of storage = first entry to db)
type dbSyncIterator struct {
it iterator.Iterator
DbSyncState
}
// initialises a sync iterator from a syncToken (passed in with the handshake)
func (self *DbStore) NewSyncIterator(state DbSyncState) (si *dbSyncIterator, err error) {
if state.First > state.Last {
return nil, fmt.Errorf("no entries found")
}
si = &dbSyncIterator{
it: self.db.NewIterator(),
DbSyncState: state,
}
si.it.Seek(getIndexKey(state.Start))
return si, nil
}
// walk the area from Start to Stop and returns items within time interval
// First to Last
func (self *dbSyncIterator) Next() (key Key) {
for self.it.Valid() {
dbkey := self.it.Key()
if dbkey[0] != 0 {
break
}
key = Key(make([]byte, len(dbkey)-1))
copy(key[:], dbkey[1:])
if bytes.Compare(key[:], self.Start) <= 0 {
self.it.Next()
continue
}
if bytes.Compare(key[:], self.Stop) > 0 {
break
}
var index dpaDBIndex
decodeIndex(self.it.Value(), &index)
self.it.Next()
if (index.Idx >= self.First) && (index.Idx < self.Last) {
return
}
}
self.it.Release()
return nil
}

191
swarm/storage/dbstore_test.go
View file

@ -1,191 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package storage
import (
"bytes"
"os"
"testing"
"github.com/XinFinOrg/XDPoSChain/common"
)
func initDbStore(t *testing.T) *DbStore {
dir, err := os.MkdirTemp("", "bzz-storage-test")
if err != nil {
t.Fatal(err)
}
m, err := NewDbStore(dir, MakeHashFunc(SHA3Hash), defaultDbCapacity, defaultRadius)
if err != nil {
t.Fatal("can't create store:", err)
}
return m
}
func testDbStore(l int64, branches int64, t *testing.T) {
m := initDbStore(t)
defer m.Close()
testStore(m, l, branches, t)
}
func TestDbStore128_0x1000000(t *testing.T) {
testDbStore(0x1000000, 128, t)
}
func TestDbStore128_10000_(t *testing.T) {
testDbStore(10000, 128, t)
}
func TestDbStore128_1000_(t *testing.T) {
testDbStore(1000, 128, t)
}
func TestDbStore128_100_(t *testing.T) {
testDbStore(100, 128, t)
}
func TestDbStore2_100_(t *testing.T) {
testDbStore(100, 2, t)
}
func TestDbStoreNotFound(t *testing.T) {
m := initDbStore(t)
defer m.Close()
_, err := m.Get(ZeroKey)
if err != notFound {
t.Errorf("Expected notFound, got %v", err)
}
}
func TestDbStoreSyncIterator(t *testing.T) {
m := initDbStore(t)
defer m.Close()
keys := []Key{
Key(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000")),
Key(common.Hex2Bytes("4000000000000000000000000000000000000000000000000000000000000000")),
Key(common.Hex2Bytes("5000000000000000000000000000000000000000000000000000000000000000")),
Key(common.Hex2Bytes("3000000000000000000000000000000000000000000000000000000000000000")),
Key(common.Hex2Bytes("2000000000000000000000000000000000000000000000000000000000000000")),
Key(common.Hex2Bytes("1000000000000000000000000000000000000000000000000000000000000000")),
}
for _, key := range keys {
m.Put(NewChunk(key, nil))
}
it, err := m.NewSyncIterator(DbSyncState{
Start: Key(common.Hex2Bytes("1000000000000000000000000000000000000000000000000000000000000000")),
Stop: Key(common.Hex2Bytes("4000000000000000000000000000000000000000000000000000000000000000")),
First: 2,
Last: 4,
})
if err != nil {
t.Fatalf("unexpected error creating NewSyncIterator")
}
var chunk Key
var res []Key
for {
chunk = it.Next()
if chunk == nil {
break
}
res = append(res, chunk)
}
if len(res) != 1 {
t.Fatalf("Expected 1 chunk, got %v: %v", len(res), res)
}
if !bytes.Equal(res[0][:], keys[3]) {
t.Fatalf("Expected %v chunk, got %v", keys[3], res[0])
}
if err != nil {
t.Fatalf("unexpected error creating NewSyncIterator")
}
it, err = m.NewSyncIterator(DbSyncState{
Start: Key(common.Hex2Bytes("1000000000000000000000000000000000000000000000000000000000000000")),
Stop: Key(common.Hex2Bytes("5000000000000000000000000000000000000000000000000000000000000000")),
First: 2,
Last: 4,
})
res = nil
for {
chunk = it.Next()
if chunk == nil {
break
}
res = append(res, chunk)
}
if len(res) != 2 {
t.Fatalf("Expected 2 chunk, got %v: %v", len(res), res)
}
if !bytes.Equal(res[0][:], keys[3]) {
t.Fatalf("Expected %v chunk, got %v", keys[3], res[0])
}
if !bytes.Equal(res[1][:], keys[2]) {
t.Fatalf("Expected %v chunk, got %v", keys[2], res[1])
}
if err != nil {
t.Fatalf("unexpected error creating NewSyncIterator")
}
it, _ = m.NewSyncIterator(DbSyncState{
Start: Key(common.Hex2Bytes("1000000000000000000000000000000000000000000000000000000000000000")),
Stop: Key(common.Hex2Bytes("4000000000000000000000000000000000000000000000000000000000000000")),
First: 2,
Last: 5,
})
res = nil
for {
chunk = it.Next()
if chunk == nil {
break
}
res = append(res, chunk)
}
if len(res) != 2 {
t.Fatalf("Expected 2 chunk, got %v", len(res))
}
if !bytes.Equal(res[0][:], keys[4]) {
t.Fatalf("Expected %v chunk, got %v", keys[4], res[0])
}
if !bytes.Equal(res[1][:], keys[3]) {
t.Fatalf("Expected %v chunk, got %v", keys[3], res[1])
}
it, _ = m.NewSyncIterator(DbSyncState{
Start: Key(common.Hex2Bytes("2000000000000000000000000000000000000000000000000000000000000000")),
Stop: Key(common.Hex2Bytes("4000000000000000000000000000000000000000000000000000000000000000")),
First: 2,
Last: 5,
})
res = nil
for {
chunk = it.Next()
if chunk == nil {
break
}
res = append(res, chunk)
}
if len(res) != 1 {
t.Fatalf("Expected 1 chunk, got %v", len(res))
}
if !bytes.Equal(res[0][:], keys[3]) {
t.Fatalf("Expected %v chunk, got %v", keys[3], res[0])
}
}

241
swarm/storage/dpa.go
View file

@ -1,241 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package storage
import (
"errors"
"fmt"
"io"
"sync"
"time"
"github.com/XinFinOrg/XDPoSChain/log"
)
/*
DPA provides the client API entrypoints Store and Retrieve to store and retrieve
It can store anything that has a byte slice representation, so files or serialised objects etc.
Storage: DPA calls the Chunker to segment the input datastream of any size to a merkle hashed tree of chunks. The key of the root block is returned to the client.
Retrieval: given the key of the root block, the DPA retrieves the block chunks and reconstructs the original data and passes it back as a lazy reader. A lazy reader is a reader with on-demand delayed processing, i.e. the chunks needed to reconstruct a large file are only fetched and processed if that particular part of the document is actually read.
As the chunker produces chunks, DPA dispatches them to its own chunk store
implementation for storage or retrieval.
*/
const (
storeChanCapacity = 100
retrieveChanCapacity = 100
singletonSwarmDbCapacity = 50000
singletonSwarmCacheCapacity = 500
maxStoreProcesses = 8
maxRetrieveProcesses = 8
)
var (
notFound = errors.New("not found")
)
type DPA struct {
ChunkStore
storeC chan *Chunk
retrieveC chan *Chunk
Chunker Chunker
lock sync.Mutex
running bool
quitC chan bool
}
// for testing locally
func NewLocalDPA(datadir string) (*DPA, error) {
hash := MakeHashFunc("SHA256")
dbStore, err := NewDbStore(datadir, hash, singletonSwarmDbCapacity, 0)
if err != nil {
return nil, err
}
return NewDPA(&LocalStore{
NewMemStore(dbStore, singletonSwarmCacheCapacity),
dbStore,
}, NewChunkerParams()), nil
}
func NewDPA(store ChunkStore, params *ChunkerParams) *DPA {
chunker := NewTreeChunker(params)
return &DPA{
Chunker: chunker,
ChunkStore: store,
}
}
// Public API. Main entry point for document retrieval directly. Used by the
// FS-aware API and httpaccess
// Chunk retrieval blocks on netStore requests with a timeout so reader will
// report error if retrieval of chunks within requested range time out.
func (self *DPA) Retrieve(key Key) LazySectionReader {
return self.Chunker.Join(key, self.retrieveC)
}
// Public API. Main entry point for document storage directly. Used by the
// FS-aware API and httpaccess
func (self *DPA) Store(data io.Reader, size int64, swg *sync.WaitGroup, wwg *sync.WaitGroup) (key Key, err error) {
return self.Chunker.Split(data, size, self.storeC, swg, wwg)
}
func (self *DPA) Start() {
self.lock.Lock()
defer self.lock.Unlock()
if self.running {
return
}
self.running = true
self.retrieveC = make(chan *Chunk, retrieveChanCapacity)
self.storeC = make(chan *Chunk, storeChanCapacity)
self.quitC = make(chan bool)
self.storeLoop()
self.retrieveLoop()
}
func (self *DPA) Stop() {
self.lock.Lock()
defer self.lock.Unlock()
if !self.running {
return
}
self.running = false
close(self.quitC)
}
// retrieveLoop dispatches the parallel chunk retrieval requests received on the
// retrieve channel to its ChunkStore (NetStore or LocalStore)
func (self *DPA) retrieveLoop() {
for i := 0; i < maxRetrieveProcesses; i++ {
go self.retrieveWorker()
}
log.Trace(fmt.Sprintf("dpa: retrieve loop spawning %v workers", maxRetrieveProcesses))
}
func (self *DPA) retrieveWorker() {
for chunk := range self.retrieveC {
log.Trace(fmt.Sprintf("dpa: retrieve loop : chunk %v", chunk.Key.Log()))
storedChunk, err := self.Get(chunk.Key)
if err == notFound {
log.Trace(fmt.Sprintf("chunk %v not found", chunk.Key.Log()))
} else if err != nil {
log.Trace(fmt.Sprintf("error retrieving chunk %v: %v", chunk.Key.Log(), err))
} else {
chunk.SData = storedChunk.SData
chunk.Size = storedChunk.Size
}
close(chunk.C)
select {
case <-self.quitC:
return
default:
}
}
}
// storeLoop dispatches the parallel chunk store request processors
// received on the store channel to its ChunkStore (NetStore or LocalStore)
func (self *DPA) storeLoop() {
for i := 0; i < maxStoreProcesses; i++ {
go self.storeWorker()
}
log.Trace(fmt.Sprintf("dpa: store spawning %v workers", maxStoreProcesses))
}
func (self *DPA) storeWorker() {
for chunk := range self.storeC {
self.Put(chunk)
if chunk.wg != nil {
log.Trace(fmt.Sprintf("dpa: store processor %v", chunk.Key.Log()))
chunk.wg.Done()
}
select {
case <-self.quitC:
return
default:
}
}
}
// DpaChunkStore implements the ChunkStore interface,
// this chunk access layer assumed 2 chunk stores
// local storage eg. LocalStore and network storage eg., NetStore
// access by calling network is blocking with a timeout
type dpaChunkStore struct {
n int
localStore ChunkStore
netStore ChunkStore
}
func NewDpaChunkStore(localStore, netStore ChunkStore) *dpaChunkStore {
return &dpaChunkStore{0, localStore, netStore}
}
// Get is the entrypoint for local retrieve requests
// waits for response or times out
func (self *dpaChunkStore) Get(key Key) (chunk *Chunk, err error) {
chunk, err = self.netStore.Get(key)
// timeout := time.Now().Add(searchTimeout)
if chunk.SData != nil {
log.Trace(fmt.Sprintf("DPA.Get: %v found locally, %d bytes", key.Log(), len(chunk.SData)))
return
}
// TODO: use self.timer time.Timer and reset with defer disableTimer
timer := time.After(searchTimeout)
select {
case <-timer:
log.Trace(fmt.Sprintf("DPA.Get: %v request time out ", key.Log()))
err = notFound
case <-chunk.Req.C:
log.Trace(fmt.Sprintf("DPA.Get: %v retrieved, %d bytes (%p)", key.Log(), len(chunk.SData), chunk))
}
return
}
// Put is the entrypoint for local store requests coming from storeLoop
func (self *dpaChunkStore) Put(entry *Chunk) {
chunk, err := self.localStore.Get(entry.Key)
if err != nil {
log.Trace(fmt.Sprintf("DPA.Put: %v new chunk. call netStore.Put", entry.Key.Log()))
chunk = entry
} else if chunk.SData == nil {
log.Trace(fmt.Sprintf("DPA.Put: %v request entry found", entry.Key.Log()))
chunk.SData = entry.SData
chunk.Size = entry.Size
} else {
log.Trace(fmt.Sprintf("DPA.Put: %v chunk already known", entry.Key.Log()))
return
}
// from this point on the storage logic is the same with network storage requests
log.Trace(fmt.Sprintf("DPA.Put %v: %v", self.n, chunk.Key.Log()))
self.n++
self.netStore.Put(chunk)
}
// Close chunk store
func (self *dpaChunkStore) Close() {}

142
swarm/storage/dpa_test.go
View file

@ -1,142 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package storage
import (
"bytes"
"io"
"os"
"sync"
"testing"
)
const testDataSize = 0x1000000
func TestDPArandom(t *testing.T) {
dbStore := initDbStore(t)
dbStore.setCapacity(50000)
memStore := NewMemStore(dbStore, defaultCacheCapacity)
localStore := &LocalStore{
memStore,
dbStore,
}
chunker := NewTreeChunker(NewChunkerParams())
dpa := &DPA{
Chunker: chunker,
ChunkStore: localStore,
}
dpa.Start()
defer dpa.Stop()
defer os.RemoveAll("/tmp/bzz")
reader, slice := testDataReaderAndSlice(testDataSize)
wg := &sync.WaitGroup{}
key, err := dpa.Store(reader, testDataSize, wg, nil)
if err != nil {
t.Errorf("Store error: %v", err)
}
wg.Wait()
resultReader := dpa.Retrieve(key)
resultSlice := make([]byte, len(slice))
n, err := resultReader.ReadAt(resultSlice, 0)
if err != io.EOF {
t.Errorf("Retrieve error: %v", err)
}
if n != len(slice) {
t.Errorf("Slice size error got %d, expected %d.", n, len(slice))
}
if !bytes.Equal(slice, resultSlice) {
t.Errorf("Comparison error.")
}
os.WriteFile("/tmp/slice.bzz.16M", slice, 0666)
os.WriteFile("/tmp/result.bzz.16M", resultSlice, 0666)
localStore.memStore = NewMemStore(dbStore, defaultCacheCapacity)
resultReader = dpa.Retrieve(key)
for i := range resultSlice {
resultSlice[i] = 0
}
n, err = resultReader.ReadAt(resultSlice, 0)
if err != io.EOF {
t.Errorf("Retrieve error after removing memStore: %v", err)
}
if n != len(slice) {
t.Errorf("Slice size error after removing memStore got %d, expected %d.", n, len(slice))
}
if !bytes.Equal(slice, resultSlice) {
t.Errorf("Comparison error after removing memStore.")
}
}
func TestDPA_capacity(t *testing.T) {
dbStore := initDbStore(t)
memStore := NewMemStore(dbStore, defaultCacheCapacity)
localStore := &LocalStore{
memStore,
dbStore,
}
memStore.setCapacity(0)
chunker := NewTreeChunker(NewChunkerParams())
dpa := &DPA{
Chunker: chunker,
ChunkStore: localStore,
}
dpa.Start()
reader, slice := testDataReaderAndSlice(testDataSize)
wg := &sync.WaitGroup{}
key, err := dpa.Store(reader, testDataSize, wg, nil)
if err != nil {
t.Errorf("Store error: %v", err)
}
wg.Wait()
resultReader := dpa.Retrieve(key)
resultSlice := make([]byte, len(slice))
n, err := resultReader.ReadAt(resultSlice, 0)
if err != io.EOF {
t.Errorf("Retrieve error: %v", err)
}
if n != len(slice) {
t.Errorf("Slice size error got %d, expected %d.", n, len(slice))
}
if !bytes.Equal(slice, resultSlice) {
t.Errorf("Comparison error.")
}
// Clear memStore
memStore.setCapacity(0)
// check whether it is, indeed, empty
dpa.ChunkStore = memStore
resultReader = dpa.Retrieve(key)
if _, err = resultReader.ReadAt(resultSlice, 0); err == nil {
t.Errorf("Was able to read %d bytes from an empty memStore.", len(slice))
}
// check how it works with localStore
dpa.ChunkStore = localStore
// localStore.dbStore.setCapacity(0)
resultReader = dpa.Retrieve(key)
for i := range resultSlice {
resultSlice[i] = 0
}
n, err = resultReader.ReadAt(resultSlice, 0)
if err != io.EOF {
t.Errorf("Retrieve error after clearing memStore: %v", err)
}
if n != len(slice) {
t.Errorf("Slice size error after clearing memStore got %d, expected %d.", n, len(slice))
}
if !bytes.Equal(slice, resultSlice) {
t.Errorf("Comparison error after clearing memStore.")
}
}

93
swarm/storage/localstore.go
View file

@ -1,93 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package storage
import (
"encoding/binary"
"github.com/XinFinOrg/XDPoSChain/metrics"
)
//metrics variables
var (
dbStorePutCounter = metrics.NewRegisteredCounter("storage.db.dbstore.put.count", nil)
)
// LocalStore is a combination of inmemory db over a disk persisted db
// implements a Get/Put with fallback (caching) logic using any 2 ChunkStores
type LocalStore struct {
memStore ChunkStore
DbStore ChunkStore
}
// This constructor uses MemStore and DbStore as components
func NewLocalStore(hash SwarmHasher, params *StoreParams) (*LocalStore, error) {
dbStore, err := NewDbStore(params.ChunkDbPath, hash, params.DbCapacity, params.Radius)
if err != nil {
return nil, err
}
return &LocalStore{
memStore: NewMemStore(dbStore, params.CacheCapacity),
DbStore: dbStore,
}, nil
}
func (self *LocalStore) CacheCounter() uint64 {
return uint64(self.memStore.(*MemStore).Counter())
}
func (self *LocalStore) DbCounter() uint64 {
return self.DbStore.(*DbStore).Counter()
}
// LocalStore is itself a chunk store
// unsafe, in that the data is not integrity checked
func (self *LocalStore) Put(chunk *Chunk) {
chunk.dbStored = make(chan bool)
self.memStore.Put(chunk)
if chunk.wg != nil {
chunk.wg.Add(1)
}
go func() {
dbStorePutCounter.Inc(1)
self.DbStore.Put(chunk)
if chunk.wg != nil {
chunk.wg.Done()
}
}()
}
// Get(chunk *Chunk) looks up a chunk in the local stores
// This method is blocking until the chunk is retrieved
// so additional timeout may be needed to wrap this call if
// ChunkStores are remote and can have long latency
func (self *LocalStore) Get(key Key) (chunk *Chunk, err error) {
chunk, err = self.memStore.Get(key)
if err == nil {
return
}
chunk, err = self.DbStore.Get(key)
if err != nil {
return
}
chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8]))
self.memStore.Put(chunk)
return
}
// Close local store
func (self *LocalStore) Close() {}

334
swarm/storage/memstore.go
View file

@ -1,334 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// memory storage layer for the package blockhash
package storage
import (
"fmt"
"sync"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/metrics"
)
//metrics variables
var (
memstorePutCounter = metrics.NewRegisteredCounter("storage.db.memstore.put.count", nil)
memstoreRemoveCounter = metrics.NewRegisteredCounter("storage.db.memstore.rm.count", nil)
)
const (
memTreeLW = 2 // log2(subtree count) of the subtrees
memTreeFLW = 14 // log2(subtree count) of the root layer
dbForceUpdateAccessCnt = 1000
defaultCacheCapacity = 5000
)
type MemStore struct {
memtree *memTree
entryCnt, capacity uint // stored entries
accessCnt uint64 // access counter; oldest is thrown away when full
dbAccessCnt uint64
dbStore *DbStore
lock sync.Mutex
}
/*
a hash prefix subtree containing subtrees or one storage entry (but never both)
- access[0] stores the smallest (oldest) access count value in this subtree
- if it contains more subtrees and its subtree count is at least 4, access[1:2]
stores the smallest access count in the first and second halves of subtrees
(so that access[0] = min(access[1], access[2])
- likewise, if subtree count is at least 8,
access[1] = min(access[3], access[4])
access[2] = min(access[5], access[6])
(access[] is a binary tree inside the multi-bit leveled hash tree)
*/
func NewMemStore(d *DbStore, capacity uint) (m *MemStore) {
m = &MemStore{}
m.memtree = newMemTree(memTreeFLW, nil, 0)
m.dbStore = d
m.setCapacity(capacity)
return
}
type memTree struct {
subtree []*memTree
parent *memTree
parentIdx uint
bits uint // log2(subtree count)
width uint // subtree count
entry *Chunk // if subtrees are present, entry should be nil
lastDBaccess uint64
access []uint64
}
func newMemTree(b uint, parent *memTree, pidx uint) (node *memTree) {
node = new(memTree)
node.bits = b
node.width = 1 << b
node.subtree = make([]*memTree, node.width)
node.access = make([]uint64, node.width-1)
node.parent = parent
node.parentIdx = pidx
if parent != nil {
parent.subtree[pidx] = node
}
return node
}
func (node *memTree) updateAccess(a uint64) {
aidx := uint(0)
var aa uint64
oa := node.access[0]
for node.access[aidx] == oa {
node.access[aidx] = a
if aidx > 0 {
aa = node.access[((aidx-1)^1)+1]
aidx = (aidx - 1) >> 1
} else {
pidx := node.parentIdx
node = node.parent
if node == nil {
return
}
nn := node.subtree[pidx^1]
if nn != nil {
aa = nn.access[0]
} else {
aa = 0
}
aidx = (node.width + pidx - 2) >> 1
}
if (aa != 0) && (aa < a) {
a = aa
}
}
}
func (s *MemStore) setCapacity(c uint) {
s.lock.Lock()
defer s.lock.Unlock()
for c < s.entryCnt {
s.removeOldest()
}
s.capacity = c
}
func (s *MemStore) Counter() uint {
return s.entryCnt
}
// entry (not its copy) is going to be in MemStore
func (s *MemStore) Put(entry *Chunk) {
if s.capacity == 0 {
return
}
s.lock.Lock()
defer s.lock.Unlock()
if s.entryCnt >= s.capacity {
s.removeOldest()
}
s.accessCnt++
memstorePutCounter.Inc(1)
node := s.memtree
bitpos := uint(0)
for node.entry == nil {
l := entry.Key.bits(bitpos, node.bits)
st := node.subtree[l]
if st == nil {
st = newMemTree(memTreeLW, node, l)
bitpos += node.bits
node = st
break
}
bitpos += node.bits
node = st
}
if node.entry != nil {
if node.entry.Key.isEqual(entry.Key) {
node.updateAccess(s.accessCnt)
if entry.SData == nil {
entry.Size = node.entry.Size
entry.SData = node.entry.SData
}
if entry.Req == nil {
entry.Req = node.entry.Req
}
entry.C = node.entry.C
node.entry = entry
return
}
for node.entry != nil {
l := node.entry.Key.bits(bitpos, node.bits)
st := node.subtree[l]
if st == nil {
st = newMemTree(memTreeLW, node, l)
}
st.entry = node.entry
node.entry = nil
st.updateAccess(node.access[0])
l = entry.Key.bits(bitpos, node.bits)
st = node.subtree[l]
if st == nil {
st = newMemTree(memTreeLW, node, l)
}
bitpos += node.bits
node = st
}
}
node.entry = entry
node.lastDBaccess = s.dbAccessCnt
node.updateAccess(s.accessCnt)
s.entryCnt++
}
func (s *MemStore) Get(hash Key) (chunk *Chunk, err error) {
s.lock.Lock()
defer s.lock.Unlock()
node := s.memtree
bitpos := uint(0)
for node.entry == nil {
l := hash.bits(bitpos, node.bits)
st := node.subtree[l]
if st == nil {
return nil, notFound
}
bitpos += node.bits
node = st
}
if node.entry.Key.isEqual(hash) {
s.accessCnt++
node.updateAccess(s.accessCnt)
chunk = node.entry
if s.dbAccessCnt-node.lastDBaccess > dbForceUpdateAccessCnt {
s.dbAccessCnt++
node.lastDBaccess = s.dbAccessCnt
if s.dbStore != nil {
s.dbStore.updateAccessCnt(hash)
}
}
} else {
err = notFound
}
return
}
func (s *MemStore) removeOldest() {
node := s.memtree
for node.entry == nil {
aidx := uint(0)
av := node.access[aidx]
for aidx < node.width/2-1 {
if av == node.access[aidx*2+1] {
node.access[aidx] = node.access[aidx*2+2]
aidx = aidx*2 + 1
} else if av == node.access[aidx*2+2] {
node.access[aidx] = node.access[aidx*2+1]
aidx = aidx*2 + 2
} else {
panic(nil)
}
}
pidx := aidx*2 + 2 - node.width
if (node.subtree[pidx] != nil) && (av == node.subtree[pidx].access[0]) {
if node.subtree[pidx+1] != nil {
node.access[aidx] = node.subtree[pidx+1].access[0]
} else {
node.access[aidx] = 0
}
} else if (node.subtree[pidx+1] != nil) && (av == node.subtree[pidx+1].access[0]) {
if node.subtree[pidx] != nil {
node.access[aidx] = node.subtree[pidx].access[0]
} else {
node.access[aidx] = 0
}
pidx++
} else {
panic(nil)
}
//fmt.Println(pidx)
node = node.subtree[pidx]
}
if node.entry.dbStored != nil {
log.Trace(fmt.Sprintf("Memstore Clean: Waiting for chunk %v to be saved", node.entry.Key.Log()))
<-node.entry.dbStored
log.Trace(fmt.Sprintf("Memstore Clean: Chunk %v saved to DBStore. Ready to clear from mem.", node.entry.Key.Log()))
} else {
log.Trace(fmt.Sprintf("Memstore Clean: Chunk %v already in DB. Ready to delete.", node.entry.Key.Log()))
}
if node.entry.SData != nil {
memstoreRemoveCounter.Inc(1)
node.entry = nil
s.entryCnt--
}
node.access[0] = 0
//---
aidx := uint(0)
for {
aa := node.access[aidx]
if aidx > 0 {
aidx = (aidx - 1) >> 1
} else {
pidx := node.parentIdx
node = node.parent
if node == nil {
return
}
aidx = (node.width + pidx - 2) >> 1
}
if (aa != 0) && ((aa < node.access[aidx]) || (node.access[aidx] == 0)) {
node.access[aidx] = aa
}
}
}
// Close memstore
func (s *MemStore) Close() {}

50
swarm/storage/memstore_test.go
View file

@ -1,50 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package storage
import (
"testing"
)
func testMemStore(l int64, branches int64, t *testing.T) {
m := NewMemStore(nil, defaultCacheCapacity)
testStore(m, l, branches, t)
}
func TestMemStore128_10000(t *testing.T) {
testMemStore(10000, 128, t)
}
func TestMemStore128_1000(t *testing.T) {
testMemStore(1000, 128, t)
}
func TestMemStore128_100(t *testing.T) {
testMemStore(100, 128, t)
}
func TestMemStore2_100(t *testing.T) {
testMemStore(100, 2, t)
}
func TestMemStoreNotFound(t *testing.T) {
m := NewMemStore(nil, defaultCacheCapacity)
_, err := m.Get(ZeroKey)
if err != notFound {
t.Errorf("Expected notFound, got %v", err)
}
}

141
swarm/storage/netstore.go
View file

@ -1,141 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package storage
import (
"fmt"
"path/filepath"
"time"
"github.com/XinFinOrg/XDPoSChain/log"
)
/*
NetStore is a cloud storage access abstaction layer for swarm
it contains the shared logic of network served chunk store/retrieval requests
both local (coming from DPA api) and remote (coming from peers via bzz protocol)
it implements the ChunkStore interface and embeds LocalStore
It is called by the bzz protocol instances via Depo (the store/retrieve request handler)
a protocol instance is running on each peer, so this is heavily parallelised.
NetStore falls back to a backend (CloudStorage interface)
implemented by bzz/network/forwarder. forwarder or IPFS or IPΞS
*/
type NetStore struct {
hashfunc SwarmHasher
localStore *LocalStore
cloud CloudStore
}
// backend engine for cloud store
// It can be aggregate dispatching to several parallel implementations:
// bzz/network/forwarder. forwarder or IPFS or IPΞS
type CloudStore interface {
Store(*Chunk)
Deliver(*Chunk)
Retrieve(*Chunk)
}
type StoreParams struct {
ChunkDbPath string
DbCapacity uint64
CacheCapacity uint
Radius int
}
//create params with default values
func NewDefaultStoreParams() (self *StoreParams) {
return &StoreParams{
DbCapacity: defaultDbCapacity,
CacheCapacity: defaultCacheCapacity,
Radius: defaultRadius,
}
}
//this can only finally be set after all config options (file, cmd line, env vars)
//have been evaluated
func (self *StoreParams) Init(path string) {
self.ChunkDbPath = filepath.Join(path, "chunks")
}
// netstore contructor, takes path argument that is used to initialise dbStore,
// the persistent (disk) storage component of LocalStore
// the second argument is the hive, the connection/logistics manager for the node
func NewNetStore(hash SwarmHasher, lstore *LocalStore, cloud CloudStore, params *StoreParams) *NetStore {
return &NetStore{
hashfunc: hash,
localStore: lstore,
cloud: cloud,
}
}
const (
// maximum number of peers that a retrieved message is delivered to
requesterCount = 3
)
var (
// timeout interval before retrieval is timed out
searchTimeout = 3 * time.Second
)
// store logic common to local and network chunk store requests
// ~ unsafe put in localdb no check if exists no extra copy no hash validation
// the chunk is forced to propagate (Cloud.Store) even if locally found!
// caller needs to make sure if that is wanted
func (self *NetStore) Put(entry *Chunk) {
self.localStore.Put(entry)
// handle deliveries
if entry.Req != nil {
log.Trace(fmt.Sprintf("NetStore.Put: localStore.Put %v hit existing request...delivering", entry.Key.Log()))
// closing C signals to other routines (local requests)
// that the chunk is has been retrieved
close(entry.Req.C)
// deliver the chunk to requesters upstream
go self.cloud.Deliver(entry)
} else {
log.Trace(fmt.Sprintf("NetStore.Put: localStore.Put %v stored locally", entry.Key.Log()))
// handle propagating store requests
// go self.cloud.Store(entry)
go self.cloud.Store(entry)
}
}
// retrieve logic common for local and network chunk retrieval requests
func (self *NetStore) Get(key Key) (*Chunk, error) {
var err error
chunk, err := self.localStore.Get(key)
if err == nil {
if chunk.Req == nil {
log.Trace(fmt.Sprintf("NetStore.Get: %v found locally", key))
} else {
log.Trace(fmt.Sprintf("NetStore.Get: %v hit on an existing request", key))
// no need to launch again
}
return chunk, err
}
// no data and no request status
log.Trace(fmt.Sprintf("NetStore.Get: %v not found locally. open new request", key))
chunk = NewChunk(key, newRequestStatus(key))
self.localStore.memStore.Put(chunk)
go self.cloud.Retrieve(chunk)
return chunk, nil
}
// Close netstore
func (self *NetStore) Close() {}

637
swarm/storage/pyramid.go
View file

@ -1,637 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package storage
import (
"encoding/binary"
"errors"
"io"
"sync"
"time"
)
/*
The main idea of a pyramid chunker is to process the input data without knowing the entire size apriori.
For this to be achieved, the chunker tree is built from the ground up until the data is exhausted.
This opens up new aveneus such as easy append and other sort of modifications to the tree thereby avoiding
duplication of data chunks.
Below is an example of a two level chunks tree. The leaf chunks are called data chunks and all the above
chunks are called tree chunks. The tree chunk above data chunks is level 0 and so on until it reaches
the root tree chunk.
T10 <- Tree chunk lvl1
|
__________________________|_____________________________
/ | | \
/ | \ \
__T00__ ___T01__ ___T02__ ___T03__ <- Tree chunks lvl 0
/ / \ / / \ / / \ / / \
/ / \ / / \ / / \ / / \
D1 D2 ... D128 D1 D2 ... D128 D1 D2 ... D128 D1 D2 ... D128 <- Data Chunks
The split function continuously read the data and creates data chunks and send them to storage.
When certain no of data chunks are created (defaultBranches), a signal is sent to create a tree
entry. When the level 0 tree entries reaches certain threshold (defaultBranches), another signal
is sent to a tree entry one level up.. and so on... until only the data is exhausted AND only one
tree entry is present in certain level. The key of tree entry is given out as the rootKey of the file.
*/
var (
errLoadingTreeRootChunk = errors.New("LoadTree Error: Could not load root chunk")
errLoadingTreeChunk = errors.New("LoadTree Error: Could not load chunk")
)
const (
ChunkProcessors = 8
DefaultBranches int64 = 128
splitTimeout = time.Minute * 5
)
const (
DataChunk = 0
TreeChunk = 1
)
type ChunkerParams struct {
Branches int64
Hash string
}
func NewChunkerParams() *ChunkerParams {
return &ChunkerParams{
Branches: DefaultBranches,
Hash: SHA3Hash,
}
}
// Entry to create a tree node
type TreeEntry struct {
level int
branchCount int64
subtreeSize uint64
chunk []byte
key []byte
index int // used in append to indicate the index of existing tree entry
updatePending bool // indicates if the entry is loaded from existing tree
}
func NewTreeEntry(pyramid *PyramidChunker) *TreeEntry {
return &TreeEntry{
level: 0,
branchCount: 0,
subtreeSize: 0,
chunk: make([]byte, pyramid.chunkSize+8),
key: make([]byte, pyramid.hashSize),
index: 0,
updatePending: false,
}
}
// Used by the hash processor to create a data/tree chunk and send to storage
type chunkJob struct {
key Key
chunk []byte
size int64
parentWg *sync.WaitGroup
chunkType int // used to identify the tree related chunks for debugging
chunkLvl int // leaf-1 is level 0 and goes upwards until it reaches root
}
type PyramidChunker struct {
hashFunc SwarmHasher
chunkSize int64
hashSize int64
branches int64
workerCount int64
workerLock sync.RWMutex
}
func NewPyramidChunker(params *ChunkerParams) (self *PyramidChunker) {
self = &PyramidChunker{}
self.hashFunc = MakeHashFunc(params.Hash)
self.branches = params.Branches
self.hashSize = int64(self.hashFunc().Size())
self.chunkSize = self.hashSize * self.branches
self.workerCount = 0
return
}
func (self *PyramidChunker) Join(key Key, chunkC chan *Chunk) LazySectionReader {
return &LazyChunkReader{
key: key,
chunkC: chunkC,
chunkSize: self.chunkSize,
branches: self.branches,
hashSize: self.hashSize,
}
}
func (self *PyramidChunker) incrementWorkerCount() {
self.workerLock.Lock()
defer self.workerLock.Unlock()
self.workerCount += 1
}
func (self *PyramidChunker) getWorkerCount() int64 {
self.workerLock.Lock()
defer self.workerLock.Unlock()
return self.workerCount
}
func (self *PyramidChunker) decrementWorkerCount() {
self.workerLock.Lock()
defer self.workerLock.Unlock()
self.workerCount -= 1
}
func (self *PyramidChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, storageWG, processorWG *sync.WaitGroup) (Key, error) {
jobC := make(chan *chunkJob, 2*ChunkProcessors)
wg := &sync.WaitGroup{}
errC := make(chan error)
quitC := make(chan bool)
rootKey := make([]byte, self.hashSize)
chunkLevel := make([][]*TreeEntry, self.branches)
wg.Add(1)
go self.prepareChunks(false, chunkLevel, data, rootKey, quitC, wg, jobC, processorWG, chunkC, errC, storageWG)
// closes internal error channel if all subprocesses in the workgroup finished
go func() {
// waiting for all chunks to finish
wg.Wait()
// if storage waitgroup is non-nil, we wait for storage to finish too
if storageWG != nil {
storageWG.Wait()
}
//We close errC here because this is passed down to 8 parallel routines underneath.
// if a error happens in one of them.. that particular routine raises error...
// once they all complete successfully, the control comes back and we can safely close this here.
close(errC)
}()
defer close(quitC)
select {
case err := <-errC:
if err != nil {
return nil, err
}
case <-time.NewTimer(splitTimeout).C:
}
return rootKey, nil
}
func (self *PyramidChunker) Append(key Key, data io.Reader, chunkC chan *Chunk, storageWG, processorWG *sync.WaitGroup) (Key, error) {
quitC := make(chan bool)
rootKey := make([]byte, self.hashSize)
chunkLevel := make([][]*TreeEntry, self.branches)
// Load the right most unfinished tree chunks in every level
self.loadTree(chunkLevel, key, chunkC, quitC)
jobC := make(chan *chunkJob, 2*ChunkProcessors)
wg := &sync.WaitGroup{}
errC := make(chan error)
wg.Add(1)
go self.prepareChunks(true, chunkLevel, data, rootKey, quitC, wg, jobC, processorWG, chunkC, errC, storageWG)
// closes internal error channel if all subprocesses in the workgroup finished
go func() {
// waiting for all chunks to finish
wg.Wait()
// if storage waitgroup is non-nil, we wait for storage to finish too
if storageWG != nil {
storageWG.Wait()
}
close(errC)
}()
defer close(quitC)
select {
case err := <-errC:
if err != nil {
return nil, err
}
case <-time.NewTimer(splitTimeout).C:
}
return rootKey, nil
}
func (self *PyramidChunker) processor(id int64, jobC chan *chunkJob, chunkC chan *Chunk, errC chan error, quitC chan bool, swg, wwg *sync.WaitGroup) {
defer self.decrementWorkerCount()
hasher := self.hashFunc()
if wwg != nil {
defer wwg.Done()
}
for {
select {
case job, ok := <-jobC:
if !ok {
return
}
self.processChunk(id, hasher, job, chunkC, swg)
case <-quitC:
return
}
}
}
func (self *PyramidChunker) processChunk(id int64, hasher SwarmHash, job *chunkJob, chunkC chan *Chunk, swg *sync.WaitGroup) {
hasher.ResetWithLength(job.chunk[:8]) // 8 bytes of length
hasher.Write(job.chunk[8:]) // minus 8 []byte length
h := hasher.Sum(nil)
newChunk := &Chunk{
Key: h,
SData: job.chunk,
Size: job.size,
wg: swg,
}
// report hash of this chunk one level up (keys corresponds to the proper subslice of the parent chunk)
copy(job.key, h)
// send off new chunk to storage
if chunkC != nil {
if swg != nil {
swg.Add(1)
}
}
job.parentWg.Done()
if chunkC != nil {
chunkC <- newChunk
}
}
func (self *PyramidChunker) loadTree(chunkLevel [][]*TreeEntry, key Key, chunkC chan *Chunk, quitC chan bool) error {
// Get the root chunk to get the total size
chunk := retrieve(key, chunkC, quitC)
if chunk == nil {
return errLoadingTreeRootChunk
}
//if data size is less than a chunk... add a parent with update as pending
if chunk.Size <= self.chunkSize {
newEntry := &TreeEntry{
level: 0,
branchCount: 1,
subtreeSize: uint64(chunk.Size),
chunk: make([]byte, self.chunkSize+8),
key: make([]byte, self.hashSize),
index: 0,
updatePending: true,
}
copy(newEntry.chunk[8:], chunk.Key)
chunkLevel[0] = append(chunkLevel[0], newEntry)
return nil
}
var treeSize int64
var depth int
treeSize = self.chunkSize
for ; treeSize < chunk.Size; treeSize *= self.branches {
depth++
}
// Add the root chunk entry
branchCount := int64(len(chunk.SData)-8) / self.hashSize
newEntry := &TreeEntry{
level: depth - 1,
branchCount: branchCount,
subtreeSize: uint64(chunk.Size),
chunk: chunk.SData,
key: key,
index: 0,
updatePending: true,
}
chunkLevel[depth-1] = append(chunkLevel[depth-1], newEntry)
// Add the rest of the tree
for lvl := depth - 1; lvl >= 1; lvl-- {
//TODO(jmozah): instead of loading finished branches and then trim in the end,
//avoid loading them in the first place
for _, ent := range chunkLevel[lvl] {
branchCount = int64(len(ent.chunk)-8) / self.hashSize
for i := int64(0); i < branchCount; i++ {
key := ent.chunk[8+(i*self.hashSize) : 8+((i+1)*self.hashSize)]
newChunk := retrieve(key, chunkC, quitC)
if newChunk == nil {
return errLoadingTreeChunk
}
bewBranchCount := int64(len(newChunk.SData)-8) / self.hashSize
newEntry := &TreeEntry{
level: lvl - 1,
branchCount: bewBranchCount,
subtreeSize: uint64(newChunk.Size),
chunk: newChunk.SData,
key: key,
index: 0,
updatePending: true,
}
chunkLevel[lvl-1] = append(chunkLevel[lvl-1], newEntry)
}
// We need to get only the right most unfinished branch.. so trim all finished branches
if int64(len(chunkLevel[lvl-1])) >= self.branches {
chunkLevel[lvl-1] = nil
}
}
}
return nil
}
func (self *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEntry, data io.Reader, rootKey []byte, quitC chan bool, wg *sync.WaitGroup, jobC chan *chunkJob, processorWG *sync.WaitGroup, chunkC chan *Chunk, errC chan error, storageWG *sync.WaitGroup) {
defer wg.Done()
chunkWG := &sync.WaitGroup{}
totalDataSize := 0
// processorWG keeps track of workers spawned for hashing chunks
if processorWG != nil {
processorWG.Add(1)
}
self.incrementWorkerCount()
go self.processor(self.workerCount, jobC, chunkC, errC, quitC, storageWG, processorWG)
parent := NewTreeEntry(self)
var unFinishedChunk *Chunk
if isAppend && len(chunkLevel[0]) != 0 {
lastIndex := len(chunkLevel[0]) - 1
ent := chunkLevel[0][lastIndex]
if ent.branchCount < self.branches {
parent = &TreeEntry{
level: 0,
branchCount: ent.branchCount,
subtreeSize: ent.subtreeSize,
chunk: ent.chunk,
key: ent.key,
index: lastIndex,
updatePending: true,
}
lastBranch := parent.branchCount - 1
lastKey := parent.chunk[8+lastBranch*self.hashSize : 8+(lastBranch+1)*self.hashSize]
unFinishedChunk = retrieve(lastKey, chunkC, quitC)
if unFinishedChunk.Size < self.chunkSize {
parent.subtreeSize = parent.subtreeSize - uint64(unFinishedChunk.Size)
parent.branchCount = parent.branchCount - 1
} else {
unFinishedChunk = nil
}
}
}
for index := 0; ; index++ {
var n int
var err error
chunkData := make([]byte, self.chunkSize+8)
if unFinishedChunk != nil {
copy(chunkData, unFinishedChunk.SData)
n, err = data.Read(chunkData[8+unFinishedChunk.Size:])
n += int(unFinishedChunk.Size)
unFinishedChunk = nil
} else {
n, err = data.Read(chunkData[8:])
}
totalDataSize += n
if err != nil {
if err == io.EOF || err == io.ErrUnexpectedEOF {
if parent.branchCount == 1 {
// Data is exactly one chunk.. pick the last chunk key as root
chunkWG.Wait()
lastChunksKey := parent.chunk[8 : 8+self.hashSize]
copy(rootKey, lastChunksKey)
break
}
} else {
close(quitC)
break
}
}
// Data ended in chunk boundary.. just signal to start bulding tree
if n == 0 {
self.buildTree(isAppend, chunkLevel, parent, chunkWG, jobC, quitC, true, rootKey)
break
} else {
pkey := self.enqueueDataChunk(chunkData, uint64(n), parent, chunkWG, jobC, quitC)
// update tree related parent data structures
parent.subtreeSize += uint64(n)
parent.branchCount++
// Data got exhausted... signal to send any parent tree related chunks
if int64(n) < self.chunkSize {
// only one data chunk .. so dont add any parent chunk
if parent.branchCount <= 1 {
chunkWG.Wait()
copy(rootKey, pkey)
break
}
self.buildTree(isAppend, chunkLevel, parent, chunkWG, jobC, quitC, true, rootKey)
break
}
if parent.branchCount == self.branches {
self.buildTree(isAppend, chunkLevel, parent, chunkWG, jobC, quitC, false, rootKey)
parent = NewTreeEntry(self)
}
}
workers := self.getWorkerCount()
if int64(len(jobC)) > workers && workers < ChunkProcessors {
if processorWG != nil {
processorWG.Add(1)
}
self.incrementWorkerCount()
go self.processor(self.workerCount, jobC, chunkC, errC, quitC, storageWG, processorWG)
}
}
}
func (self *PyramidChunker) buildTree(isAppend bool, chunkLevel [][]*TreeEntry, ent *TreeEntry, chunkWG *sync.WaitGroup, jobC chan *chunkJob, quitC chan bool, last bool, rootKey []byte) {
chunkWG.Wait()
self.enqueueTreeChunk(chunkLevel, ent, chunkWG, jobC, quitC, last)
compress := false
endLvl := self.branches
for lvl := int64(0); lvl < self.branches; lvl++ {
lvlCount := int64(len(chunkLevel[lvl]))
if lvlCount >= self.branches {
endLvl = lvl + 1
compress = true
break
}
}
if !compress && !last {
return
}
// Wait for all the keys to be processed before compressing the tree
chunkWG.Wait()
for lvl := int64(ent.level); lvl < endLvl; lvl++ {
lvlCount := int64(len(chunkLevel[lvl]))
if lvlCount == 1 && last {
copy(rootKey, chunkLevel[lvl][0].key)
return
}
for startCount := int64(0); startCount < lvlCount; startCount += self.branches {
endCount := startCount + self.branches
if endCount > lvlCount {
endCount = lvlCount
}
var nextLvlCount int64
var tempEntry *TreeEntry
if len(chunkLevel[lvl+1]) > 0 {
nextLvlCount = int64(len(chunkLevel[lvl+1]) - 1)
tempEntry = chunkLevel[lvl+1][nextLvlCount]
}
if isAppend && tempEntry != nil && tempEntry.updatePending {
updateEntry := &TreeEntry{
level: int(lvl + 1),
branchCount: 0,
subtreeSize: 0,
chunk: make([]byte, self.chunkSize+8),
key: make([]byte, self.hashSize),
index: int(nextLvlCount),
updatePending: true,
}
for index := int64(0); index < lvlCount; index++ {
updateEntry.branchCount++
updateEntry.subtreeSize += chunkLevel[lvl][index].subtreeSize
copy(updateEntry.chunk[8+(index*self.hashSize):8+((index+1)*self.hashSize)], chunkLevel[lvl][index].key[:self.hashSize])
}
self.enqueueTreeChunk(chunkLevel, updateEntry, chunkWG, jobC, quitC, last)
} else {
noOfBranches := endCount - startCount
newEntry := &TreeEntry{
level: int(lvl + 1),
branchCount: noOfBranches,
subtreeSize: 0,
chunk: make([]byte, (noOfBranches*self.hashSize)+8),
key: make([]byte, self.hashSize),
index: int(nextLvlCount),
updatePending: false,
}
index := int64(0)
for i := startCount; i < endCount; i++ {
entry := chunkLevel[lvl][i]
newEntry.subtreeSize += entry.subtreeSize
copy(newEntry.chunk[8+(index*self.hashSize):8+((index+1)*self.hashSize)], entry.key[:self.hashSize])
index++
}
self.enqueueTreeChunk(chunkLevel, newEntry, chunkWG, jobC, quitC, last)
}
}
if !isAppend {
chunkWG.Wait()
if compress {
chunkLevel[lvl] = nil
}
}
}
}
func (self *PyramidChunker) enqueueTreeChunk(chunkLevel [][]*TreeEntry, ent *TreeEntry, chunkWG *sync.WaitGroup, jobC chan *chunkJob, quitC chan bool, last bool) {
if ent != nil {
// wait for data chunks to get over before processing the tree chunk
if last {
chunkWG.Wait()
}
binary.LittleEndian.PutUint64(ent.chunk[:8], ent.subtreeSize)
ent.key = make([]byte, self.hashSize)
chunkWG.Add(1)
select {
case jobC <- &chunkJob{ent.key, ent.chunk[:ent.branchCount*self.hashSize+8], int64(ent.subtreeSize), chunkWG, TreeChunk, 0}:
case <-quitC:
}
// Update or append based on weather it is a new entry or being reused
if ent.updatePending {
chunkWG.Wait()
chunkLevel[ent.level][ent.index] = ent
} else {
chunkLevel[ent.level] = append(chunkLevel[ent.level], ent)
}
}
}
func (self *PyramidChunker) enqueueDataChunk(chunkData []byte, size uint64, parent *TreeEntry, chunkWG *sync.WaitGroup, jobC chan *chunkJob, quitC chan bool) Key {
binary.LittleEndian.PutUint64(chunkData[:8], size)
pkey := parent.chunk[8+parent.branchCount*self.hashSize : 8+(parent.branchCount+1)*self.hashSize]
chunkWG.Add(1)
select {
case jobC <- &chunkJob{pkey, chunkData[:size+8], int64(size), chunkWG, DataChunk, -1}:
case <-quitC:
}
return pkey
}

40
swarm/storage/swarmhasher.go
View file

@ -1,40 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package storage
import (
"hash"
)
const (
BMTHash = "BMT"
SHA3Hash = "SHA3" // http://golang.org/pkg/hash/#Hash
)
type SwarmHash interface {
hash.Hash
ResetWithLength([]byte)
}
type HashWithLength struct {
hash.Hash
}
func (self *HashWithLength) ResetWithLength(length []byte) {
self.Reset()
self.Write(length)
}

248
swarm/storage/types.go
View file

@ -1,248 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package storage
import (
"bytes"
"crypto"
"fmt"
"hash"
"io"
"sync"
"github.com/XinFinOrg/XDPoSChain/bmt"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/crypto/sha3"
)
type Hasher func() hash.Hash
type SwarmHasher func() SwarmHash
// Peer is the recorded as Source on the chunk
// should probably not be here? but network should wrap chunk object
type Peer interface{}
type Key []byte
func (x Key) Size() uint {
return uint(len(x))
}
func (x Key) isEqual(y Key) bool {
return bytes.Equal(x, y)
}
func (h Key) bits(i, j uint) uint {
ii := i >> 3
jj := i & 7
if ii >= h.Size() {
return 0
}
if jj+j <= 8 {
return uint((h[ii] >> jj) & ((1 << j) - 1))
}
res := uint(h[ii] >> jj)
jj = 8 - jj
j -= jj
for j != 0 {
ii++
if j < 8 {
res += uint(h[ii]&((1<<j)-1)) << jj
return res
}
res += uint(h[ii]) << jj
jj += 8
j -= 8
}
return res
}
func IsZeroKey(key Key) bool {
return len(key) == 0 || bytes.Equal(key, ZeroKey)
}
var ZeroKey = Key(common.Hash{}.Bytes())
func MakeHashFunc(hash string) SwarmHasher {
switch hash {
case "SHA256":
return func() SwarmHash { return &HashWithLength{crypto.SHA256.New()} }
case "SHA3":
return func() SwarmHash { return &HashWithLength{sha3.NewKeccak256()} }
case "BMT":
return func() SwarmHash {
hasher := sha3.NewKeccak256
pool := bmt.NewTreePool(hasher, bmt.DefaultSegmentCount, bmt.DefaultPoolSize)
return bmt.New(pool)
}
}
return nil
}
func (key Key) Hex() string {
return fmt.Sprintf("%064x", []byte(key[:]))
}
func (key Key) Log() string {
if len(key[:]) < 4 {
return fmt.Sprintf("%x", []byte(key[:]))
}
return fmt.Sprintf("%08x", []byte(key[:4]))
}
func (key Key) String() string {
return fmt.Sprintf("%064x", []byte(key)[:])
}
func (key Key) MarshalJSON() (out []byte, err error) {
return []byte(`"` + key.String() + `"`), nil
}
func (key *Key) UnmarshalJSON(value []byte) error {
s := string(value)
*key = make([]byte, 32)
h := common.Hex2Bytes(s[1 : len(s)-1])
copy(*key, h)
return nil
}
// each chunk when first requested opens a record associated with the request
// next time a request for the same chunk arrives, this record is updated
// this request status keeps track of the request ID-s as well as the requesting
// peers and has a channel that is closed when the chunk is retrieved. Multiple
// local callers can wait on this channel (or combined with a timeout, block with a
// select).
type RequestStatus struct {
Key Key
Source Peer
C chan bool
Requesters map[uint64][]interface{}
}
func newRequestStatus(key Key) *RequestStatus {
return &RequestStatus{
Key: key,
Requesters: make(map[uint64][]interface{}),
C: make(chan bool),
}
}
// Chunk also serves as a request object passed to ChunkStores
// in case it is a retrieval request, Data is nil and Size is 0
// Note that Size is not the size of the data chunk, which is Data.Size()
// but the size of the subtree encoded in the chunk
// 0 if request, to be supplied by the dpa
type Chunk struct {
Key Key // always
SData []byte // nil if request, to be supplied by dpa
Size int64 // size of the data covered by the subtree encoded in this chunk
Source Peer // peer
C chan bool // to signal data delivery by the dpa
Req *RequestStatus // request Status needed by netStore
wg *sync.WaitGroup // wg to synchronize
dbStored chan bool // never remove a chunk from memStore before it is written to dbStore
}
func NewChunk(key Key, rs *RequestStatus) *Chunk {
return &Chunk{Key: key, Req: rs}
}
/*
The ChunkStore interface is implemented by :
- MemStore: a memory cache
- DbStore: local disk/db store
- LocalStore: a combination (sequence of) memStore and dbStore
- NetStore: cloud storage abstraction layer
- DPA: local requests for swarm storage and retrieval
*/
type ChunkStore interface {
Put(*Chunk) // effectively there is no error even if there is an error
Get(Key) (*Chunk, error)
Close()
}
/*
Chunker is the interface to a component that is responsible for disassembling and assembling larger data and indended to be the dependency of a DPA storage system with fixed maximum chunksize.
It relies on the underlying chunking model.
When calling Split, the caller provides a channel (chan *Chunk) on which it receives chunks to store. The DPA delegates to storage layers (implementing ChunkStore interface).
Split returns an error channel, which the caller can monitor.
After getting notified that all the data has been split (the error channel is closed), the caller can safely read or save the root key. Optionally it times out if not all chunks get stored or not the entire stream of data has been processed. By inspecting the errc channel the caller can check if any explicit errors (typically IO read/write failures) occurred during splitting.
When calling Join with a root key, the caller gets returned a seekable lazy reader. The caller again provides a channel on which the caller receives placeholder chunks with missing data. The DPA is supposed to forward this to the chunk stores and notify the chunker if the data has been delivered (i.e. retrieved from memory cache, disk-persisted db or cloud based swarm delivery). As the seekable reader is used, the chunker then puts these together the relevant parts on demand.
*/
type Splitter interface {
/*
When splitting, data is given as a SectionReader, and the key is a hashSize long byte slice (Key), the root hash of the entire content will fill this once processing finishes.
New chunks to store are coming to caller via the chunk storage channel, which the caller provides.
wg is a Waitgroup (can be nil) that can be used to block until the local storage finishes
The caller gets returned an error channel, if an error is encountered during splitting, it is fed to errC error channel.
A closed error signals process completion at which point the key can be considered final if there were no errors.
*/
Split(io.Reader, int64, chan *Chunk, *sync.WaitGroup, *sync.WaitGroup) (Key, error)
/* This is the first step in making files mutable (not chunks)..
Append allows adding more data chunks to the end of the already existsing file.
The key for the root chunk is supplied to load the respective tree.
Rest of the parameters behave like Split.
*/
Append(Key, io.Reader, chan *Chunk, *sync.WaitGroup, *sync.WaitGroup) (Key, error)
}
type Joiner interface {
/*
Join reconstructs original content based on a root key.
When joining, the caller gets returned a Lazy SectionReader, which is
seekable and implements on-demand fetching of chunks as and where it is read.
New chunks to retrieve are coming to caller via the Chunk channel, which the caller provides.
If an error is encountered during joining, it appears as a reader error.
The SectionReader.
As a result, partial reads from a document are possible even if other parts
are corrupt or lost.
The chunks are not meant to be validated by the chunker when joining. This
is because it is left to the DPA to decide which sources are trusted.
*/
Join(key Key, chunkC chan *Chunk) LazySectionReader
}
type Chunker interface {
Joiner
Splitter
// returns the key length
// KeySize() int64
}
// Size, Seek, Read, ReadAt
type LazySectionReader interface {
Size(chan bool) (int64, error)
io.Seeker
io.Reader
io.ReaderAt
}
type LazyTestSectionReader struct {
*io.SectionReader
}
func (self *LazyTestSectionReader) Size(chan bool) (int64, error) {
return self.SectionReader.Size(), nil
}

471
swarm/swarm.go
View file

@ -1,471 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package swarm
import (
"bytes"
"context"
"crypto/ecdsa"
"fmt"
"math/big"
"net"
"strings"
"time"
"unicode"
"github.com/XinFinOrg/XDPoSChain/accounts/abi/bind"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/contracts/chequebook"
"github.com/XinFinOrg/XDPoSChain/contracts/ens"
"github.com/XinFinOrg/XDPoSChain/crypto"
"github.com/XinFinOrg/XDPoSChain/ethclient"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/metrics"
"github.com/XinFinOrg/XDPoSChain/node"
"github.com/XinFinOrg/XDPoSChain/p2p"
"github.com/XinFinOrg/XDPoSChain/p2p/discover"
"github.com/XinFinOrg/XDPoSChain/params"
"github.com/XinFinOrg/XDPoSChain/rpc"
"github.com/XinFinOrg/XDPoSChain/swarm/api"
httpapi "github.com/XinFinOrg/XDPoSChain/swarm/api/http"
"github.com/XinFinOrg/XDPoSChain/swarm/fuse"
"github.com/XinFinOrg/XDPoSChain/swarm/network"
"github.com/XinFinOrg/XDPoSChain/swarm/storage"
)
var (
startTime time.Time
updateGaugesPeriod = 5 * time.Second
startCounter = metrics.NewRegisteredCounter("stack,start", nil)
stopCounter = metrics.NewRegisteredCounter("stack,stop", nil)
uptimeGauge = metrics.NewRegisteredGauge("stack.uptime", nil)
dbSizeGauge = metrics.NewRegisteredGauge("storage.db.chunks.size", nil)
cacheSizeGauge = metrics.NewRegisteredGauge("storage.db.cache.size", nil)
)
// the swarm stack
type Swarm struct {
config *api.Config // swarm configuration
api *api.Api // high level api layer (fs/manifest)
dns api.Resolver // DNS registrar
dbAccess *network.DbAccess // access to local chunk db iterator and storage counter
storage storage.ChunkStore // internal access to storage, common interface to cloud storage backends
dpa *storage.DPA // distributed preimage archive, the local API to the storage with document level storage/retrieval support
depo network.StorageHandler // remote request handler, interface between bzz protocol and the storage
cloud storage.CloudStore // procurement, cloud storage backend (can multi-cloud)
hive *network.Hive // the logistic manager
backend chequebook.Backend // simple blockchain Backend
privateKey *ecdsa.PrivateKey
corsString string
swapEnabled bool
lstore *storage.LocalStore // local store, needs to store for releasing resources after node stopped
sfs *fuse.SwarmFS // need this to cleanup all the active mounts on node exit
}
type SwarmAPI struct {
Api *api.Api
Backend chequebook.Backend
PrvKey *ecdsa.PrivateKey
}
func (self *Swarm) API() *SwarmAPI {
return &SwarmAPI{
Api: self.api,
Backend: self.backend,
PrvKey: self.privateKey,
}
}
// creates a new swarm service instance
// implements node.Service
func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api.Config) (self *Swarm, err error) {
if bytes.Equal(common.FromHex(config.PublicKey), storage.ZeroKey) {
return nil, fmt.Errorf("empty public key")
}
if bytes.Equal(common.FromHex(config.BzzKey), storage.ZeroKey) {
return nil, fmt.Errorf("empty bzz key")
}
self = &Swarm{
config: config,
swapEnabled: config.SwapEnabled,
backend: backend,
privateKey: config.Swap.PrivateKey(),
corsString: config.Cors,
}
log.Debug(fmt.Sprintf("Setting up Swarm service components"))
hash := storage.MakeHashFunc(config.ChunkerParams.Hash)
self.lstore, err = storage.NewLocalStore(hash, config.StoreParams)
if err != nil {
return
}
// setup local store
log.Debug(fmt.Sprintf("Set up local storage"))
self.dbAccess = network.NewDbAccess(self.lstore)
log.Debug(fmt.Sprintf("Set up local db access (iterator/counter)"))
// set up the kademlia hive
self.hive = network.NewHive(
common.HexToHash(self.config.BzzKey), // key to hive (kademlia base address)
config.HiveParams, // configuration parameters
config.SwapEnabled, // SWAP enabled
config.SyncEnabled, // syncronisation enabled
)
log.Debug(fmt.Sprintf("Set up swarm network with Kademlia hive"))
// setup cloud storage backend
self.cloud = network.NewForwarder(self.hive)
log.Debug(fmt.Sprintf("-> set swarm forwarder as cloud storage backend"))
// setup cloud storage internal access layer
self.storage = storage.NewNetStore(hash, self.lstore, self.cloud, config.StoreParams)
log.Debug(fmt.Sprintf("-> swarm net store shared access layer to Swarm Chunk Store"))
// set up Depo (storage handler = cloud storage access layer for incoming remote requests)
self.depo = network.NewDepo(hash, self.lstore, self.storage)
log.Debug(fmt.Sprintf("-> REmote Access to CHunks"))
// set up DPA, the cloud storage local access layer
dpaChunkStore := storage.NewDpaChunkStore(self.lstore, self.storage)
log.Debug(fmt.Sprintf("-> Local Access to Swarm"))
// Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage
self.dpa = storage.NewDPA(dpaChunkStore, self.config.ChunkerParams)
log.Debug(fmt.Sprintf("-> Content Store API"))
if len(config.EnsAPIs) > 0 {
opts := []api.MultiResolverOption{}
for _, c := range config.EnsAPIs {
tld, endpoint, addr := parseEnsAPIAddress(c)
r, err := newEnsClient(endpoint, addr, config)
if err != nil {
return nil, err
}
opts = append(opts, api.MultiResolverOptionWithResolver(r, tld))
}
self.dns = api.NewMultiResolver(opts...)
}
self.api = api.NewApi(self.dpa, self.dns)
// Manifests for Smart Hosting
log.Debug(fmt.Sprintf("-> Web3 virtual server API"))
self.sfs = fuse.NewSwarmFS(self.api)
log.Debug("-> Initializing Fuse file system")
return self, nil
}
// parseEnsAPIAddress parses string according to format
// [tld:][contract-addr@]url and returns ENSClientConfig structure
// with endpoint, contract address and TLD.
func parseEnsAPIAddress(s string) (tld, endpoint string, addr common.Address) {
isAllLetterString := func(s string) bool {
for _, r := range s {
if !unicode.IsLetter(r) {
return false
}
}
return true
}
endpoint = s
if i := strings.Index(endpoint, ":"); i > 0 {
if isAllLetterString(endpoint[:i]) && len(endpoint) > i+2 && endpoint[i+1:i+3] != "//" {
tld = endpoint[:i]
endpoint = endpoint[i+1:]
}
}
if i := strings.Index(endpoint, "@"); i > 0 {
addr = common.HexToAddress(endpoint[:i])
endpoint = endpoint[i+1:]
}
return
}
// newEnsClient creates a new ENS client for that is a consumer of
// a ENS API on a specific endpoint. It is used as a helper function
// for creating multiple resolvers in NewSwarm function.
func newEnsClient(endpoint string, addr common.Address, config *api.Config) (*ens.ENS, error) {
log.Info("connecting to ENS API", "url", endpoint)
client, err := rpc.Dial(endpoint)
if err != nil {
return nil, fmt.Errorf("error connecting to ENS API %s: %s", endpoint, err)
}
ensClient := ethclient.NewClient(client)
ensRoot := config.EnsRoot
if addr != (common.Address{}) {
ensRoot = addr
} else {
a, err := detectEnsAddr(client)
if err == nil {
ensRoot = a
} else {
log.Warn(fmt.Sprintf("could not determine ENS contract address, using default %s", ensRoot), "err", err)
}
}
transactOpts := bind.NewKeyedTransactor(config.Swap.PrivateKey())
dns, err := ens.NewENS(transactOpts, ensRoot, ensClient)
if err != nil {
return nil, err
}
log.Debug(fmt.Sprintf("-> Swarm Domain Name Registrar %v @ address %v", endpoint, ensRoot.Hex()))
return dns, err
}
// detectEnsAddr determines the ENS contract address by getting both the
// version and genesis hash using the client and matching them to either
// mainnet or testnet addresses
func detectEnsAddr(client *rpc.Client) (common.Address, error) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
var version string
if err := client.CallContext(ctx, &version, "net_version"); err != nil {
return common.Address{}, err
}
block, err := ethclient.NewClient(client).BlockByNumber(ctx, big.NewInt(0))
if err != nil {
return common.Address{}, err
}
switch {
case version == "1" && block.Hash() == params.MainnetGenesisHash:
log.Info("using Mainnet ENS contract address", "addr", ens.MainNetAddress)
return ens.MainNetAddress, nil
case version == "3" && block.Hash() == params.TestnetGenesisHash:
log.Info("using Testnet ENS contract address", "addr", ens.TestNetAddress)
return ens.TestNetAddress, nil
default:
return common.Address{}, fmt.Errorf("unknown version and genesis hash: %s %s", version, block.Hash())
}
}
/*
Start is called when the stack is started
* starts the network kademlia hive peer management
* (starts netStore level 0 api)
* starts DPA level 1 api (chunking -> store/retrieve requests)
* (starts level 2 api)
* starts http proxy server
* registers url scheme handlers for bzz, etc
* TODO: start subservices like sword, swear, swarmdns
*/
// implements the node.Service interface
func (self *Swarm) Start(srv *p2p.Server) error {
startTime = time.Now()
connectPeer := func(url string) error {
node, err := discover.ParseNode(url)
if err != nil {
return fmt.Errorf("invalid node URL: %v", err)
}
srv.AddPeer(node)
return nil
}
// set chequebook
if self.swapEnabled {
ctx := context.Background() // The initial setup has no deadline.
err := self.SetChequebook(ctx)
if err != nil {
return fmt.Errorf("Unable to set chequebook for SWAP: %v", err)
}
log.Debug(fmt.Sprintf("-> cheque book for SWAP: %v", self.config.Swap.Chequebook()))
} else {
log.Debug(fmt.Sprintf("SWAP disabled: no cheque book set"))
}
log.Warn(fmt.Sprintf("Starting Swarm service"))
self.hive.Start(
discover.PubkeyID(&srv.PrivateKey.PublicKey),
func() string { return srv.ListenAddr },
connectPeer,
)
log.Info(fmt.Sprintf("Swarm network started on bzz address: %v", self.hive.Addr()))
self.dpa.Start()
log.Debug(fmt.Sprintf("Swarm DPA started"))
// start swarm http proxy server
if self.config.Port != "" {
addr := net.JoinHostPort(self.config.ListenAddr, self.config.Port)
go httpapi.StartHttpServer(self.api, &httpapi.ServerConfig{
Addr: addr,
CorsString: self.corsString,
})
log.Info(fmt.Sprintf("Swarm http proxy started on %v", addr))
if self.corsString != "" {
log.Debug(fmt.Sprintf("Swarm http proxy started with corsdomain: %v", self.corsString))
}
}
self.periodicallyUpdateGauges()
startCounter.Inc(1)
return nil
}
func (self *Swarm) periodicallyUpdateGauges() {
ticker := time.NewTicker(updateGaugesPeriod)
go func() {
for range ticker.C {
self.updateGauges()
}
}()
}
func (self *Swarm) updateGauges() {
dbSizeGauge.Update(int64(self.lstore.DbCounter()))
cacheSizeGauge.Update(int64(self.lstore.CacheCounter()))
uptimeGauge.Update(time.Since(startTime).Nanoseconds())
}
func (self *Swarm) SaveData() {
}
// implements the node.Service interface
// stops all component services.
func (self *Swarm) Stop() error {
self.dpa.Stop()
err := self.hive.Stop()
if ch := self.config.Swap.Chequebook(); ch != nil {
ch.Stop()
ch.Save()
}
if self.lstore != nil {
self.lstore.DbStore.Close()
}
self.sfs.Stop()
stopCounter.Inc(1)
return err
}
// implements the node.Service interface
func (self *Swarm) Protocols() []p2p.Protocol {
proto, err := network.Bzz(self.depo, self.backend, self.hive, self.dbAccess, self.config.Swap, self.config.SyncParams, self.config.NetworkId)
if err != nil {
return nil
}
return []p2p.Protocol{proto}
}
// implements node.Service
// Apis returns the RPC Api descriptors the Swarm implementation offers
func (self *Swarm) APIs() []rpc.API {
return []rpc.API{
// public APIs
{
Namespace: "bzz",
Version: "0.1",
Service: &Info{self.config, chequebook.ContractParams},
Public: true,
},
// admin APIs
{
Namespace: "bzz",
Version: "0.1",
Service: api.NewControl(self.api, self.hive),
Public: false,
},
{
Namespace: "chequebook",
Version: chequebook.Version,
Service: chequebook.NewApi(self.config.Swap.Chequebook),
Public: false,
},
{
Namespace: "swarmfs",
Version: fuse.Swarmfs_Version,
Service: self.sfs,
Public: false,
},
// storage APIs
// DEPRECATED: Use the HTTP API instead
{
Namespace: "bzz",
Version: "0.1",
Service: api.NewStorage(self.api),
Public: true,
},
{
Namespace: "bzz",
Version: "0.1",
Service: api.NewFileSystem(self.api),
Public: false,
},
// {Namespace, Version, api.NewAdmin(self), false},
}
}
func (self *Swarm) Api() *api.Api {
return self.api
}
// SetChequebook ensures that the local checquebook is set up on chain.
func (self *Swarm) SetChequebook(ctx context.Context) error {
err := self.config.Swap.SetChequebook(ctx, self.backend, self.config.Path)
if err != nil {
return err
}
log.Info(fmt.Sprintf("new chequebook set (%v): saving config file, resetting all connections in the hive", self.config.Swap.Contract.Hex()))
self.hive.DropAll()
return nil
}
// Local swarm without netStore
func NewLocalSwarm(datadir, port string) (self *Swarm, err error) {
prvKey, err := crypto.GenerateKey()
if err != nil {
return
}
config := api.NewDefaultConfig()
config.Path = datadir
config.Init(prvKey)
config.Port = port
dpa, err := storage.NewLocalDPA(datadir)
if err != nil {
return
}
self = &Swarm{
api: api.NewApi(dpa, nil),
config: config,
}
return
}
// serialisable info about swarm
type Info struct {
*api.Config
*chequebook.Params
}
func (self *Info) Info() *Info {
return self
}

119
swarm/swarm_test.go
View file

@ -1,119 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package swarm
import (
"testing"
"github.com/XinFinOrg/XDPoSChain/common"
)
func TestParseEnsAPIAddress(t *testing.T) {
for _, x := range []struct {
description string
value string
tld string
endpoint string
addr common.Address
}{
{
description: "IPC endpoint",
value: "/data/testnet/geth.ipc",
endpoint: "/data/testnet/geth.ipc",
},
{
description: "HTTP endpoint",
value: "http://127.0.0.1:1234",
endpoint: "http://127.0.0.1:1234",
},
{
description: "WS endpoint",
value: "ws://127.0.0.1:1234",
endpoint: "ws://127.0.0.1:1234",
},
{
description: "IPC Endpoint and TLD",
value: "test:/data/testnet/geth.ipc",
endpoint: "/data/testnet/geth.ipc",
tld: "test",
},
{
description: "HTTP endpoint and TLD",
value: "test:http://127.0.0.1:1234",
endpoint: "http://127.0.0.1:1234",
tld: "test",
},
{
description: "WS endpoint and TLD",
value: "test:ws://127.0.0.1:1234",
endpoint: "ws://127.0.0.1:1234",
tld: "test",
},
{
description: "IPC Endpoint and contract address",
value: "314159265dD8dbb310642f98f50C066173C1259b@/data/testnet/geth.ipc",
endpoint: "/data/testnet/geth.ipc",
addr: common.HexToAddress("314159265dD8dbb310642f98f50C066173C1259b"),
},
{
description: "HTTP endpoint and contract address",
value: "314159265dD8dbb310642f98f50C066173C1259b@http://127.0.0.1:1234",
endpoint: "http://127.0.0.1:1234",
addr: common.HexToAddress("314159265dD8dbb310642f98f50C066173C1259b"),
},
{
description: "WS endpoint and contract address",
value: "314159265dD8dbb310642f98f50C066173C1259b@ws://127.0.0.1:1234",
endpoint: "ws://127.0.0.1:1234",
addr: common.HexToAddress("314159265dD8dbb310642f98f50C066173C1259b"),
},
{
description: "IPC Endpoint, TLD and contract address",
value: "test:314159265dD8dbb310642f98f50C066173C1259b@/data/testnet/geth.ipc",
endpoint: "/data/testnet/geth.ipc",
addr: common.HexToAddress("314159265dD8dbb310642f98f50C066173C1259b"),
tld: "test",
},
{
description: "HTTP endpoint, TLD and contract address",
value: "eth:314159265dD8dbb310642f98f50C066173C1259b@http://127.0.0.1:1234",
endpoint: "http://127.0.0.1:1234",
addr: common.HexToAddress("314159265dD8dbb310642f98f50C066173C1259b"),
tld: "eth",
},
{
description: "WS endpoint, TLD and contract address",
value: "eth:314159265dD8dbb310642f98f50C066173C1259b@ws://127.0.0.1:1234",
endpoint: "ws://127.0.0.1:1234",
addr: common.HexToAddress("314159265dD8dbb310642f98f50C066173C1259b"),
tld: "eth",
},
} {
t.Run(x.description, func(t *testing.T) {
tld, endpoint, addr := parseEnsAPIAddress(x.value)
if endpoint != x.endpoint {
t.Errorf("expected Endpoint %q, got %q", x.endpoint, endpoint)
}
if addr != x.addr {
t.Errorf("expected ContractAddress %q, got %q", x.addr.String(), addr.String())
}
if tld != x.tld {
t.Errorf("expected TLD %q, got %q", x.tld, tld)
}
})
}
}

71
swarm/testutil/http.go
View file

@ -1,71 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package testutil
import (
"net/http/httptest"
"os"
"testing"
"github.com/XinFinOrg/XDPoSChain/swarm/api"
httpapi "github.com/XinFinOrg/XDPoSChain/swarm/api/http"
"github.com/XinFinOrg/XDPoSChain/swarm/storage"
)
func NewTestSwarmServer(t *testing.T) *TestSwarmServer {
dir, err := os.MkdirTemp("", "swarm-storage-test")
if err != nil {
t.Fatal(err)
}
storeparams := &storage.StoreParams{
ChunkDbPath: dir,
DbCapacity: 5000000,
CacheCapacity: 5000,
Radius: 0,
}
localStore, err := storage.NewLocalStore(storage.MakeHashFunc("SHA3"), storeparams)
if err != nil {
os.RemoveAll(dir)
t.Fatal(err)
}
chunker := storage.NewTreeChunker(storage.NewChunkerParams())
dpa := &storage.DPA{
Chunker: chunker,
ChunkStore: localStore,
}
dpa.Start()
a := api.NewApi(dpa, nil)
srv := httptest.NewServer(httpapi.NewServer(a))
return &TestSwarmServer{
Server: srv,
Dpa: dpa,
dir: dir,
}
}
type TestSwarmServer struct {
*httptest.Server
Dpa *storage.DPA
dir string
}
func (t *TestSwarmServer) Close() {
t.Server.Close()
t.Dpa.Stop()
os.RemoveAll(t.dir)
}