forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-18 01:43:47 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge pull request #189 from ethersphere/swarm-network-rewrite-syncer

Browse source 
Swarm network rewrite syncer
This commit is contained in:
Balint Gabor 2018-02-19 17:03:43 +01:00 committed by GitHub
commit b5bd009ddc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
81 changed files with 6151 additions and 1346 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
.travis.yml
View file

@ -3,17 +3,6 @@ go_import_path: github.com/ethereum/go-ethereum
sudo: false
matrix:
include:
- os: linux
dist: trusty
sudo: required
go: 1.7.x
script:
- sudo modprobe fuse
- sudo chmod 666 /dev/fuse
- sudo chown root:$USER /etc/fuse.conf
- go run build/ci.go install
- go run build/ci.go test -coverage
- os: linux
dist: trusty
sudo: required
@ -25,7 +14,6 @@ matrix:
- go run build/ci.go install
- go run build/ci.go test -coverage
# These are the latest Go versions.
- os: linux
dist: trusty
sudo: required
@ -47,6 +35,28 @@ matrix:
- go run build/ci.go install
- go run build/ci.go test -coverage
# These are the latest Go versions.
- os: linux
dist: trusty
sudo: required
go: "1.10"
script:
- sudo modprobe fuse
- sudo chmod 666 /dev/fuse
- sudo chown root:$USER /etc/fuse.conf
- go run build/ci.go install
- go run build/ci.go test -coverage
- os: osx
go: "1.10"
script:
- unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703
- brew update
- brew install caskroom/cask/brew-cask
- brew cask install osxfuse
- go run build/ci.go install
- go run build/ci.go test -coverage
# This builder only tests code linters on latest version of Go
- os: linux
dist: trusty
@ -185,6 +195,8 @@ matrix:
- xctool -version
- xcrun simctl list
# Workaround for https://github.com/golang/go/issues/23749
- export CGO_CFLAGS_ALLOW='-fmodules|-fblocks|-fobjc-arc'
- go run build/ci.go xcode -signer IOS_SIGNING_KEY -deploy trunk -upload gethstore/builds
# This builder does the Azure archive purges to avoid accumulating junk

23
cmd/swarm/db.go
View file

@ -23,6 +23,7 @@ import (
"path/filepath"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/storage"
"gopkg.in/urfave/cli.v1"
@ -30,11 +31,11 @@ import (
func dbExport(ctx *cli.Context) {
args := ctx.Args()
if len(args) != 2 {
utils.Fatalf("invalid arguments, please specify both <chunkdb> (path to a local chunk database) and <file> (path to write the tar archive to, - for stdout)")
if len(args) != 3 {
utils.Fatalf("invalid arguments, please specify both <chunkdb> (path to a local chunk database), <file> (path to write the tar archive to, - for stdout) and the base key")
}
store, err := openDbStore(args[0])
store, err := openDbStore(args[0], common.Hex2Bytes(args[2]))
if err != nil {
utils.Fatalf("error opening local chunk database: %s", err)
}
@ -62,11 +63,11 @@ func dbExport(ctx *cli.Context) {
func dbImport(ctx *cli.Context) {
args := ctx.Args()
if len(args) != 2 {
utils.Fatalf("invalid arguments, please specify both <chunkdb> (path to a local chunk database) and <file> (path to read the tar archive from, - for stdin)")
if len(args) != 3 {
utils.Fatalf("invalid arguments, please specify both <chunkdb> (path to a local chunk database), <file> (path to read the tar archive from, - for stdin) and the base key")
}
store, err := openDbStore(args[0])
store, err := openDbStore(args[0], common.Hex2Bytes(args[2]))
if err != nil {
utils.Fatalf("error opening local chunk database: %s", err)
}
@ -94,11 +95,11 @@ func dbImport(ctx *cli.Context) {
func dbClean(ctx *cli.Context) {
args := ctx.Args()
if len(args) != 1 {
utils.Fatalf("invalid arguments, please specify <chunkdb> (path to a local chunk database)")
if len(args) != 2 {
utils.Fatalf("invalid arguments, please specify <chunkdb> (path to a local chunk database) and the base key")
}
store, err := openDbStore(args[0])
store, err := openDbStore(args[0], common.Hex2Bytes(args[1]))
if err != nil {
utils.Fatalf("error opening local chunk database: %s", err)
}
@ -107,10 +108,10 @@ func dbClean(ctx *cli.Context) {
store.Cleanup()
}
func openDbStore(path string) (*storage.DbStore, error) {
func openDbStore(path string, basekey []byte) (*storage.DbStore, error) {
if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil {
return nil, fmt.Errorf("invalid chunkdb path: %s", err)
}
hash := storage.MakeHashFunc("SHA3")
return storage.NewDbStore(path, hash, 10000000, 0)
return storage.NewDbStore(path, hash, 10000000, func(k storage.Key) (ret uint8) { return uint8(storage.Proximity(basekey[:], k[:])) })
}

2
cmd/swarm/hash.go
View file

@ -39,7 +39,7 @@ func hash(ctx *cli.Context) {
stat, _ := f.Stat()
chunker := storage.NewTreeChunker(storage.NewChunkerParams())
key, err := chunker.Split(f, stat.Size(), nil, nil, nil)
key, _, err := chunker.Split(f, stat.Size(), nil)
if err != nil {
utils.Fatalf("%v\n", err)
} else {

4
contracts/chequebook/cheque_test.go
View file

@ -281,8 +281,8 @@ func TestDeposit(t *testing.T) {
t.Fatalf("expected balance %v, got %v", exp, chbook.Balance())
}
// autodeposit every 30ms if new cheque issued
interval := 30 * time.Millisecond
// autodeposit every 200ms if new cheque issued
interval := 200 * time.Millisecond
chbook.AutoDeposit(interval, common.Big1, balance)
_, err = chbook.Issue(addr1, amount)
if err != nil {

2
internal/jsre/deps/web3.js
View file

@ -2307,7 +2307,7 @@ var toChecksumAddress = function (address) {
};
/**
* Transforms given string to valid 20 bytes-length addres with 0x prefix
* Transforms given string to valid 20 bytes-length address with 0x prefix
*
* @method toAddress
* @param {String} address

16
log/logger.go
View file

@ -126,13 +126,13 @@ type logger struct {
h *swapHandler
}
func (l *logger) write(msg string, lvl Lvl, ctx []interface{}) {
func (l *logger) write(msg string, lvl Lvl, ctx []interface{}, skip int) {
l.h.Log(&Record{
Time: time.Now(),
Lvl: lvl,
Msg: msg,
Ctx: newContext(l.ctx, ctx),
Call: stack.Caller(2),
Call: stack.Caller(skip),
KeyNames: RecordKeyNames{
Time: timeKey,
Msg: msgKey,
@ -156,27 +156,27 @@ func newContext(prefix []interface{}, suffix []interface{}) []interface{} {
}
func (l *logger) Trace(msg string, ctx ...interface{}) {
l.write(msg, LvlTrace, ctx)
l.write(msg, LvlTrace, ctx, 2)
}
func (l *logger) Debug(msg string, ctx ...interface{}) {
l.write(msg, LvlDebug, ctx)
l.write(msg, LvlDebug, ctx, 2)
}
func (l *logger) Info(msg string, ctx ...interface{}) {
l.write(msg, LvlInfo, ctx)
l.write(msg, LvlInfo, ctx, 2)
}
func (l *logger) Warn(msg string, ctx ...interface{}) {
l.write(msg, LvlWarn, ctx)
l.write(msg, LvlWarn, ctx, 2)
}
func (l *logger) Error(msg string, ctx ...interface{}) {
l.write(msg, LvlError, ctx)
l.write(msg, LvlError, ctx, 2)
}
func (l *logger) Crit(msg string, ctx ...interface{}) {
l.write(msg, LvlCrit, ctx)
l.write(msg, LvlCrit, ctx, 2)
os.Exit(1)
}

17
log/root.go
View file

@ -31,31 +31,36 @@ func Root() Logger {
// Trace is a convenient alias for Root().Trace
func Trace(msg string, ctx ...interface{}) {
root.write(msg, LvlTrace, ctx)
root.write(msg, LvlTrace, ctx, 2)
}
// Debug is a convenient alias for Root().Debug
func Debug(msg string, ctx ...interface{}) {
root.write(msg, LvlDebug, ctx)
root.write(msg, LvlDebug, ctx, 2)
}
// Info is a convenient alias for Root().Info
func Info(msg string, ctx ...interface{}) {
root.write(msg, LvlInfo, ctx)
root.write(msg, LvlInfo, ctx, 2)
}
// Warn is a convenient alias for Root().Warn
func Warn(msg string, ctx ...interface{}) {
root.write(msg, LvlWarn, ctx)
root.write(msg, LvlWarn, ctx, 2)
}
// Error is a convenient alias for Root().Error
func Error(msg string, ctx ...interface{}) {
root.write(msg, LvlError, ctx)
root.write(msg, LvlError, ctx, 2)
}
// Crit is a convenient alias for Root().Crit
func Crit(msg string, ctx ...interface{}) {
root.write(msg, LvlCrit, ctx)
root.write(msg, LvlCrit, ctx, 2)
os.Exit(1)
}
// Output is a convenient alias for write
func Output(msg string, lvl Lvl, skip int, ctx ...interface{}) {
root.write(msg, lvl, ctx, skip)
}

4
p2p/protocols/protocol_test.go
View file

@ -365,14 +365,14 @@ func runMultiplePeers(t *testing.T, peer int, errs ...error) {
}
func TestMultiplePeersDropSelf(t *testing.T) {
func XTestMultiplePeersDropSelf(t *testing.T) {
runMultiplePeers(t, 0,
fmt.Errorf("subprotocol error"),
fmt.Errorf("Message handler error: (msg code 3): dropped"),
)
}
func TestMultiplePeersDropOther(t *testing.T) {
func XTestMultiplePeersDropOther(t *testing.T) {
runMultiplePeers(t, 1,
fmt.Errorf("Message handler error: (msg code 3): dropped"),
fmt.Errorf("subprotocol error"),

6
p2p/simulations/adapters/docker.go
View file

@ -33,6 +33,10 @@ import (
"github.com/ethereum/go-ethereum/p2p/discover"
)
var (
ErrLinuxOnly = errors.New("DockerAdapter can only be used on Linux as it uses the current binary (which must be a Linux binary)")
)
// DockerAdapter is a NodeAdapter which runs simulation nodes inside Docker
// containers.
//
@ -52,7 +56,7 @@ func NewDockerAdapter() (*DockerAdapter, error) {
// It is reasonable to require this because the caller can just
// compile the current binary in a Docker container.
if runtime.GOOS != "linux" {
return nil, errors.New("DockerAdapter can only be used on Linux as it uses the current binary (which must be a Linux binary)")
return nil, ErrLinuxOnly
}
if err := buildDockerImage(); err != nil {

6
p2p/simulations/adapters/exec.go
View file

@ -105,9 +105,9 @@ func (e *ExecAdapter) NewNode(config *NodeConfig) (Node, error) {
conf.Stack.P2P.NAT = nil
conf.Stack.NoUSB = true
// listen on a random localhost port (we'll get the actual port after
// starting the node through the RPC admin.nodeInfo method)
conf.Stack.P2P.ListenAddr = "127.0.0.1:0"
// listen on a localhost port, which we set when we
// initialise NodeConfig (usually a random port)
conf.Stack.P2P.ListenAddr = fmt.Sprintf("127.0.0.1:%d", config.Port)
node := &ExecNode{
ID: config.ID,

19
p2p/simulations/adapters/inproc.go
View file

@ -34,11 +34,6 @@ import (
"github.com/ethereum/go-ethereum/rpc"
)
const (
socketReadBuffer = 5000 * 1024
socketWriteBuffer = 5000 * 1024
)
// SimAdapter is a NodeAdapter which creates in-memory simulation nodes and
// connects them using net.Pipe or OS socket connections
type SimAdapter struct {
@ -112,7 +107,7 @@ func (s *SimAdapter) NewNode(config *NodeConfig) (Node, error) {
MaxPeers: math.MaxInt32,
NoDiscovery: true,
Dialer: s,
EnableMsgEvents: true,
EnableMsgEvents: config.EnableMsgEvents,
},
NoUSB: true,
Logger: log.New("node.id", id.String()),
@ -378,20 +373,10 @@ func socketPipe() (net.Conn, net.Conn, error) {
return nil, nil, err
}
err = setSocketBuffer(pipe1)
if err != nil {
return nil, nil, err
}
err = setSocketBuffer(pipe2)
if err != nil {
return nil, nil, err
}
return pipe1, pipe2, nil
}
func setSocketBuffer(conn net.Conn) error {
func setSocketBuffer(conn net.Conn, socketReadBuffer int, socketWriteBuffer int) error {
switch v := conn.(type) {
case *net.UnixConn:
err := v.SetReadBuffer(socketReadBuffer)

50
p2p/simulations/adapters/inproc_test.go
View file

@ -27,7 +27,7 @@ import (
func TestSocketPipe(t *testing.T) {
c1, c2, err := socketPipe()
if err != nil {
t.Skip("system limit is less than desired. no buffer space available for socket. skipping test... err: ", err)
t.Fatal(err)
}
done := make(chan struct{})
@ -35,15 +35,19 @@ func TestSocketPipe(t *testing.T) {
go func() {
msgs := 20
size := 8
for i := 0; i < msgs; i++ {
msg := make([]byte, size)
_ = binary.PutUvarint(msg, uint64(i))
_, err := c1.Write(msg)
if err != nil {
t.Fatal(err)
// OS socket pipe is blocking (depending on buffer size on OS), so writes are emitted asynchronously
go func() {
for i := 0; i < msgs; i++ {
msg := make([]byte, size)
_ = binary.PutUvarint(msg, uint64(i))
_, err := c1.Write(msg)
if err != nil {
t.Fatal(err)
}
}
}
}()
for i := 0; i < msgs; i++ {
msg := make([]byte, size)
@ -64,7 +68,7 @@ func TestSocketPipe(t *testing.T) {
select {
case <-done:
case <-time.After(1 * time.Second):
case <-time.After(5 * time.Second):
t.Fatal("test timeout")
}
}
@ -72,7 +76,7 @@ func TestSocketPipe(t *testing.T) {
func TestSocketPipeBidirections(t *testing.T) {
c1, c2, err := socketPipe()
if err != nil {
t.Skip("system limit is less than desired. no buffer space available for socket. skipping test... err: ", err)
t.Fatal(err)
}
done := make(chan struct{})
@ -80,14 +84,18 @@ func TestSocketPipeBidirections(t *testing.T) {
go func() {
msgs := 100
size := 4
for i := 0; i < msgs; i++ {
msg := []byte(`ping`)
_, err := c1.Write(msg)
if err != nil {
t.Fatal(err)
// OS socket pipe is blocking (depending on buffer size on OS), so writes are emitted asynchronously
go func() {
for i := 0; i < msgs; i++ {
msg := []byte(`ping`)
_, err := c1.Write(msg)
if err != nil {
t.Fatal(err)
}
}
}
}()
for i := 0; i < msgs; i++ {
out := make([]byte, size)
@ -124,7 +132,7 @@ func TestSocketPipeBidirections(t *testing.T) {
select {
case <-done:
case <-time.After(1 * time.Second):
case <-time.After(5 * time.Second):
t.Fatal("test timeout")
}
}
@ -169,7 +177,7 @@ func TestTcpPipe(t *testing.T) {
select {
case <-done:
case <-time.After(1 * time.Second):
case <-time.After(5 * time.Second):
t.Fatal("test timeout")
}
}
@ -232,7 +240,7 @@ func TestTcpPipeBidirections(t *testing.T) {
select {
case <-done:
case <-time.After(1 * time.Second):
case <-time.After(5 * time.Second):
t.Fatal("test timeout")
}
}
@ -281,7 +289,7 @@ func TestNetPipe(t *testing.T) {
select {
case <-done:
case <-time.After(1 * time.Second):
case <-time.After(5 * time.Second):
t.Fatal("test timeout")
}
}
@ -356,7 +364,7 @@ func TestNetPipeBidirections(t *testing.T) {
select {
case <-done:
case <-time.After(1 * time.Second):
case <-time.After(5 * time.Second):
t.Fatal("test timeout")
}
}

58
p2p/simulations/adapters/types.go
View file

@ -23,6 +23,7 @@ import (
"fmt"
"net"
"os"
"strconv"
"github.com/docker/docker/pkg/reexec"
"github.com/ethereum/go-ethereum/crypto"
@ -97,24 +98,30 @@ type NodeConfig struct {
// function to sanction or prevent suggesting a peer
Reachable func(id discover.NodeID) bool
Port uint16
}
// nodeConfigJSON is used to encode and decode NodeConfig as JSON by encoding
// all fields as strings
type nodeConfigJSON struct {
ID string `json:"id"`
PrivateKey string `json:"private_key"`
Name string `json:"name"`
Services []string `json:"services"`
ID string `json:"id"`
PrivateKey string `json:"private_key"`
Name string `json:"name"`
Services []string `json:"services"`
EnableMsgEvents bool `json:"enable_msg_events"`
Port uint16 `json:"port"`
}
// MarshalJSON implements the json.Marshaler interface by encoding the config
// fields as strings
func (n *NodeConfig) MarshalJSON() ([]byte, error) {
confJSON := nodeConfigJSON{
ID: n.ID.String(),
Name: n.Name,
Services: n.Services,
ID: n.ID.String(),
Name: n.Name,
Services: n.Services,
Port: n.Port,
EnableMsgEvents: n.EnableMsgEvents,
}
if n.PrivateKey != nil {
confJSON.PrivateKey = hex.EncodeToString(crypto.FromECDSA(n.PrivateKey))
@ -152,6 +159,8 @@ func (n *NodeConfig) UnmarshalJSON(data []byte) error {
n.Name = confJSON.Name
n.Services = confJSON.Services
n.Port = confJSON.Port
n.EnableMsgEvents = confJSON.EnableMsgEvents
return nil
}
@ -163,13 +172,36 @@ func RandomNodeConfig() *NodeConfig {
if err != nil {
panic("unable to generate key")
}
var id discover.NodeID
pubkey := crypto.FromECDSAPub(&key.PublicKey)
copy(id[:], pubkey[1:])
return &NodeConfig{
ID: id,
PrivateKey: key,
id := discover.PubkeyID(&key.PublicKey)
port, err := assignTCPPort()
if err != nil {
panic("unable to assign tcp port")
}
return &NodeConfig{
ID: id,
Name: fmt.Sprintf("node_%s", id.String()),
PrivateKey: key,
Port: port,
EnableMsgEvents: true,
}
}
func assignTCPPort() (uint16, error) {
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
return 0, err
}
l.Close()
_, port, err := net.SplitHostPort(l.Addr().String())
if err != nil {
return 0, err
}
p, err := strconv.ParseInt(port, 10, 32)
if err != nil {
return 0, err
}
return uint16(p), nil
}
// ServiceContext is a collection of options and methods which can be utilised

3
p2p/simulations/http.go
View file

@ -561,7 +561,8 @@ func (s *Server) LoadSnapshot(w http.ResponseWriter, req *http.Request) {
// CreateNode creates a node in the network using the given configuration
func (s *Server) CreateNode(w http.ResponseWriter, req *http.Request) {
config := adapters.RandomNodeConfig()
config := &adapters.NodeConfig{}
err := json.NewDecoder(req.Body).Decode(config)
if err != nil && err != io.EOF {
http.Error(w, err.Error(), http.StatusBadRequest)

10
p2p/simulations/http_test.go
View file

@ -348,7 +348,8 @@ func startTestNetwork(t *testing.T, client *Client) []string {
nodeCount := 2
nodeIDs := make([]string, nodeCount)
for i := 0; i < nodeCount; i++ {
node, err := client.CreateNode(nil)
config := adapters.RandomNodeConfig()
node, err := client.CreateNode(config)
if err != nil {
t.Fatalf("error creating node: %s", err)
}
@ -527,7 +528,9 @@ func TestHTTPNodeRPC(t *testing.T) {
// start a node in the network
client := NewClient(s.URL)
node, err := client.CreateNode(nil)
config := adapters.RandomNodeConfig()
node, err := client.CreateNode(config)
if err != nil {
t.Fatalf("error creating node: %s", err)
}
@ -589,7 +592,8 @@ func TestHTTPSnapshot(t *testing.T) {
nodeCount := 2
nodes := make([]*p2p.NodeInfo, nodeCount)
for i := 0; i < nodeCount; i++ {
node, err := client.CreateNode(nil)
config := adapters.RandomNodeConfig()
node, err := client.CreateNode(config)
if err != nil {
t.Fatalf("error creating node: %s", err)
}

4
p2p/simulations/mocker.go
View file

@ -26,6 +26,7 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/p2p/discover"
"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
)
//a map of mocker names to its function
@ -165,7 +166,8 @@ func probabilistic(net *Network, quit chan struct{}, nodeCount int) {
func connectNodesInRing(net *Network, nodeCount int) ([]discover.NodeID, error) {
ids := make([]discover.NodeID, nodeCount)
for i := 0; i < nodeCount; i++ {
node, err := net.NewNode()
conf := adapters.RandomNodeConfig()
node, err := net.NewNodeWithConfig(conf)
if err != nil {
log.Error("Error creating a node! %s", err)
return nil, err

27
p2p/simulations/network.go
View file

@ -78,26 +78,12 @@ func (self *Network) Events() *event.Feed {
return &self.events
}
// NewNode adds a new node to the network with a random ID
func (self *Network) NewNode() (*Node, error) {
conf := adapters.RandomNodeConfig()
conf.Services = []string{self.DefaultService}
return self.NewNodeWithConfig(conf)
}
// NewNodeWithConfig adds a new node to the network with the given config,
// returning an error if a node with the same ID or name already exists
func (self *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error) {
self.lock.Lock()
defer self.lock.Unlock()
// create a random ID and PrivateKey if not set
if conf.ID == (discover.NodeID{}) {
c := adapters.RandomNodeConfig()
conf.ID = c.ID
conf.PrivateKey = c.PrivateKey
}
id := conf.ID
if conf.Reachable == nil {
conf.Reachable = func(otherID discover.NodeID) bool {
_, err := self.InitConn(conf.ID, otherID)
@ -105,14 +91,9 @@ func (self *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error)
}
}
// assign a name to the node if not set
if conf.Name == "" {
conf.Name = fmt.Sprintf("node%02d", len(self.Nodes)+1)
}
// check the node doesn't already exist
if node := self.getNode(id); node != nil {
return nil, fmt.Errorf("node with ID %q already exists", id)
if node := self.getNode(conf.ID); node != nil {
return nil, fmt.Errorf("node with ID %q already exists", conf.ID)
}
if node := self.getNodeByName(conf.Name); node != nil {
return nil, fmt.Errorf("node with name %q already exists", conf.Name)
@ -132,8 +113,8 @@ func (self *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error)
Node: adapterNode,
Config: conf,
}
log.Trace(fmt.Sprintf("node %v created", id))
self.nodeMap[id] = len(self.Nodes)
log.Trace(fmt.Sprintf("node %v created", conf.ID))
self.nodeMap[conf.ID] = len(self.Nodes)
self.Nodes = append(self.Nodes, node)
// emit a "control" event

3
p2p/simulations/network_test.go
View file

@ -41,7 +41,8 @@ func TestNetworkSimulation(t *testing.T) {
nodeCount := 20
ids := make([]discover.NodeID, nodeCount)
for i := 0; i < nodeCount; i++ {
node, err := network.NewNode()
conf := adapters.RandomNodeConfig()
node, err := network.NewNodeWithConfig(conf)
if err != nil {
t.Fatalf("error creating node: %s", err)
}

22
swarm/api/api.go
View file

@ -23,7 +23,6 @@ import (
"net/http"
"regexp"
"strings"
"sync"
"bytes"
"mime"
@ -74,8 +73,8 @@ func (self *Api) Retrieve(key storage.Key) storage.LazySectionReader {
return self.dpa.Retrieve(key)
}
func (self *Api) Store(data io.Reader, size int64, wg *sync.WaitGroup) (key storage.Key, err error) {
return self.dpa.Store(data, size, wg, nil)
func (self *Api) Store(data io.Reader, size int64) (key storage.Key, wait func(), err error) {
return self.dpa.Store(data, size)
}
type ErrResolve error
@ -112,21 +111,22 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) {
}
// Put provides singleton manifest creation on top of dpa store
func (self *Api) Put(content, contentType string) (storage.Key, error) {
func (self *Api) Put(content, contentType string) (k storage.Key, wait func(), err error) {
r := strings.NewReader(content)
wg := &sync.WaitGroup{}
key, err := self.dpa.Store(r, int64(len(content)), wg, nil)
key, waitContent, err := self.dpa.Store(r, int64(len(content)))
if err != nil {
return nil, err
return nil, nil, err
}
manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType)
r = strings.NewReader(manifest)
key, err = self.dpa.Store(r, int64(len(manifest)), wg, nil)
key, waitManifest, err := self.dpa.Store(r, int64(len(manifest)))
if err != nil {
return nil, err
return nil, nil, err
}
wg.Wait()
return key, nil
return key, func() {
waitContent()
waitManifest()
}, nil
}
// Get uses iterative manifest retrieval and prefix matching

8
swarm/api/api_test.go
View file

@ -34,9 +34,8 @@ func testApi(t *testing.T, f func(*Api)) {
if err != nil {
t.Fatalf("unable to create temp dir: %v", err)
}
os.RemoveAll(datadir)
defer os.RemoveAll(datadir)
dpa, err := storage.NewLocalDPA(datadir)
dpa, err := storage.NewLocalDPA(datadir, make([]byte, 32))
if err != nil {
return
}
@ -110,11 +109,12 @@ func TestApiPut(t *testing.T) {
content := "hello"
exp := expResponse(content, "text/plain", 0)
// exp := expResponse([]byte(content), "text/plain", 0)
key, err := api.Put(content, exp.MimeType)
key, wait, err := api.Put(content, exp.MimeType)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
resp := testGet(t, api, key.String(), "")
wait()
resp := testGet(t, api, key.Hex(), "")
checkResponse(t, resp, exp)
})
}

11
swarm/api/filesystem.go
View file

@ -43,6 +43,7 @@ func NewFileSystem(api *Api) *FileSystem {
// Upload replicates a local directory as a manifest file and uploads it
// using dpa store
// This function waits the chunks to be stored.
// TODO: localpath should point to a manifest
//
// DEPRECATED: Use the HTTP API instead
@ -112,12 +113,12 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
if err == nil {
stat, _ := f.Stat()
var hash storage.Key
wg := &sync.WaitGroup{}
hash, err = self.api.dpa.Store(f, stat.Size(), wg, nil)
var wait func()
hash, wait, err = self.api.dpa.Store(f, stat.Size())
if hash != nil {
list[i].Hash = hash.String()
list[i].Hash = hash.Hex()
}
wg.Wait()
wait()
awg.Done()
if err == nil {
first512 := make([]byte, 512)
@ -163,7 +164,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
err2 := trie.recalcAndStore()
var hs string
if err2 == nil {
hs = trie.hash.String()
hs = trie.hash.Hex()
}
awg.Wait()
return hs, err2

8
swarm/api/filesystem_test.go
View file

@ -21,7 +21,6 @@ import (
"io/ioutil"
"os"
"path/filepath"
"sync"
"testing"
"github.com/ethereum/go-ethereum/common"
@ -105,9 +104,8 @@ func TestApiDirUploadModify(t *testing.T) {
t.Errorf("unexpected error: %v", err)
return
}
wg := &sync.WaitGroup{}
hash, err := api.Store(bytes.NewReader(index), int64(len(index)), wg)
wg.Wait()
hash, wait, err := api.Store(bytes.NewReader(index), int64(len(index)))
wait()
if err != nil {
t.Errorf("unexpected error: %v", err)
return
@ -122,7 +120,7 @@ func TestApiDirUploadModify(t *testing.T) {
t.Errorf("unexpected error: %v", err)
return
}
bzzhash = key.String()
bzzhash = key.Hex()
content := readPath(t, "testdata", "test0", "index.html")
resp := testGet(t, api, bzzhash, "index2.html")

3
swarm/api/http/error.go
View file

@ -110,7 +110,8 @@ func ShowMultipleChoices(w http.ResponseWriter, r *http.Request, list api.Manife
//(and return the correct HTTP status code)
func ShowError(w http.ResponseWriter, r *http.Request, msg string, code int) {
if code == http.StatusInternalServerError {
log.Error(msg)
//log.Error(msg)
log.Output(msg, log.LvlError, 3)
}
respond(w, r, &ErrorParams{
Code: code,

110
swarm/api/http/server.go
View file

@ -91,21 +91,21 @@ type Request struct {
// body in swarm and returns the resulting storage key as a text/plain response
func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) {
if r.uri.Path != "" {
s.BadRequest(w, r, "raw POST request cannot contain a path")
ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, "raw POST request cannot contain a path"), http.StatusBadRequest)
return
}
if r.Header.Get("Content-Length") == "" {
s.BadRequest(w, r, "missing Content-Length header in request")
ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, "missing Content-Length header in request"), http.StatusBadRequest)
return
}
key, err := s.api.Store(r.Body, r.ContentLength, nil)
key, _, err := s.api.Store(r.Body, r.ContentLength)
if err != nil {
s.Error(w, r, err)
ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError)
return
}
s.logDebug("content for %s stored", key.Log())
log.Debug(fmt.Sprintf("content for %s stored", key.Log()))
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
@ -120,7 +120,7 @@ func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) {
func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) {
contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
if err != nil {
s.BadRequest(w, r, err.Error())
ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, err), http.StatusBadRequest)
return
}
@ -128,13 +128,13 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) {
if r.uri.Addr != "" {
key, err = s.api.Resolve(r.uri)
if err != nil {
s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)), http.StatusInternalServerError)
return
}
} else {
key, err = s.api.NewManifest()
if err != nil {
s.Error(w, r, err)
ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError)
return
}
}
@ -153,7 +153,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) {
}
})
if err != nil {
s.Error(w, r, fmt.Errorf("error creating manifest: %s", err))
ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("error creating manifest: %s", err)), http.StatusInternalServerError)
return
}
@ -186,12 +186,12 @@ func (s *Server) handleTarUpload(req *Request, mw *api.ManifestWriter) error {
Size: hdr.Size,
ModTime: hdr.ModTime,
}
s.logDebug("adding %s (%d bytes) to new manifest", entry.Path, entry.Size)
log.Debug(fmt.Sprintf("adding %s (%d bytes) to new manifest", entry.Path, entry.Size))
contentKey, err := mw.AddEntry(tr, entry)
if err != nil {
return fmt.Errorf("error adding manifest entry from tar stream: %s", err)
}
s.logDebug("content for %s stored", contentKey.Log())
log.Debug(fmt.Sprintf("content for %s stored", contentKey.Log()))
}
}
@ -243,12 +243,12 @@ func (s *Server) handleMultipartUpload(req *Request, boundary string, mw *api.Ma
Size: size,
ModTime: time.Now(),
}
s.logDebug("adding %s (%d bytes) to new manifest", entry.Path, entry.Size)
log.Debug(fmt.Sprintf("adding %s (%d bytes) to new manifest", entry.Path, entry.Size))
contentKey, err := mw.AddEntry(reader, entry)
if err != nil {
return fmt.Errorf("error adding manifest entry from multipart form: %s", err)
}
s.logDebug("content for %s stored", contentKey.Log())
log.Debug(fmt.Sprintf("content for %s stored", contentKey.Log()))
}
}
@ -263,7 +263,7 @@ func (s *Server) handleDirectUpload(req *Request, mw *api.ManifestWriter) error
if err != nil {
return err
}
s.logDebug("content for %s stored", key.Log())
log.Debug(fmt.Sprintf("content for %s stored", key.Log()))
return nil
}
@ -273,16 +273,16 @@ func (s *Server) handleDirectUpload(req *Request, mw *api.ManifestWriter) error
func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) {
key, err := s.api.Resolve(r.uri)
if err != nil {
s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)), http.StatusInternalServerError)
return
}
newKey, err := s.updateManifest(key, func(mw *api.ManifestWriter) error {
s.logDebug("removing %s from manifest %s", r.uri.Path, key.Log())
log.Debug(fmt.Sprintf("removing %s from manifest %s", r.uri.Path, key.Log()))
return mw.RemoveEntry(r.uri.Path)
})
if err != nil {
s.Error(w, r, fmt.Errorf("error updating manifest: %s", err))
ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("error updating manifest: %s", err)), http.StatusInternalServerError)
return
}
@ -296,12 +296,12 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
if r.uri.Path != "" {
frequency, err := strconv.ParseUint(r.uri.Path, 10, 64)
if err != nil {
s.BadRequest(w, r, fmt.Sprintf("Cannot parse frequency parameter: %v", err))
ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, fmt.Sprintf("Cannot parse frequency parameter: %v", err)), http.StatusBadRequest)
return
}
key, err := s.api.ResourceCreate(r.Context(), r.uri.Addr, frequency)
if err != nil {
s.Error(w, r, fmt.Errorf("Resource creation failed: %v", err))
ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("Resource creation failed: %v", err)), http.StatusInternalServerError)
return
}
outdata = key.Hex()
@ -309,12 +309,12 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
data, err := ioutil.ReadAll(r.Body)
if err != nil {
s.Error(w, r, err)
ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError)
return
}
_, _, _, err = s.api.ResourceUpdate(r.Context(), r.uri.Addr, data)
if err != nil {
s.Error(w, r, fmt.Errorf("Update resource failed: %v", err))
ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("Update resource failed: %v", err)), http.StatusInternalServerError)
return
}
@ -368,11 +368,11 @@ func (s *Server) handleGetResource(w http.ResponseWriter, r *Request, name strin
}
updateKey, data, err = s.api.ResourceLookup(r.Context(), name, uint32(period), uint32(version))
default:
s.BadRequest(w, r, "Invalid mutable resource request")
ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, "Invalid mutable resource request"), http.StatusBadRequest)
return
}
if err != nil {
s.Error(w, r, fmt.Errorf("Mutable resource lookup failed: %v", err))
ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("Mutable resource lookup failed: %v", err)), http.StatusInternalServerError)
return
}
log.Debug("Found update", "key", updateKey)
@ -388,7 +388,7 @@ func (s *Server) handleGetResource(w http.ResponseWriter, r *Request, name strin
func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
key, err := s.api.Resolve(r.uri)
if err != nil {
s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)), http.StatusInternalServerError)
return
}
@ -397,7 +397,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
if r.uri.Path != "" {
walker, err := s.api.NewManifestWalker(key, nil)
if err != nil {
s.BadRequest(w, r, fmt.Sprintf("%s is not a manifest", key))
ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, fmt.Sprintf("%s is not a manifest", key)), http.StatusBadRequest)
return
}
var entry *api.ManifestEntry
@ -425,7 +425,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
return api.SkipManifest
})
if entry == nil {
s.NotFound(w, r, errors.New("Manifest entry could not be loaded"))
ShowError(w, &r.Request, fmt.Sprintf("NOT FOUND error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("Manifest entry could not be loaded")), http.StatusNotFound)
return
}
key = storage.Key(common.Hex2Bytes(entry.Hash))
@ -434,7 +434,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
// check the root chunk exists by retrieving the file's size
reader := s.api.Retrieve(key)
if _, err := reader.Size(nil); err != nil {
s.NotFound(w, r, fmt.Errorf("Root chunk not found %s: %s", key, err))
ShowError(w, &r.Request, fmt.Sprintf("NOT FOUND error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("Root chunk not found %s: %s", key, err)), http.StatusNotFound)
return
}
@ -460,19 +460,19 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
// contained in the manifest
func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) {
if r.uri.Path != "" {
s.BadRequest(w, r, "files request cannot contain a path")
ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, "files request cannot contain a path"), http.StatusBadRequest)
return
}
key, err := s.api.Resolve(r.uri)
if err != nil {
s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)), http.StatusInternalServerError)
return
}
walker, err := s.api.NewManifestWalker(key, nil)
if err != nil {
s.Error(w, r, err)
ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError)
return
}
@ -519,7 +519,7 @@ func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) {
return nil
})
if err != nil {
s.logError("error generating tar stream: %s", err)
log.Error(fmt.Sprintf("error generating tar stream: %s", err))
}
}
@ -535,14 +535,14 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
key, err := s.api.Resolve(r.uri)
if err != nil {
s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)), http.StatusInternalServerError)
return
}
list, err := s.getManifestList(key, r.uri.Path)
if err != nil {
s.Error(w, r, err)
ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError)
return
}
@ -559,7 +559,7 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
List: &list,
})
if err != nil {
s.logError("error rendering list HTML: %s", err)
log.Error(fmt.Sprintf("error rendering list HTML: %s", err))
}
return
}
@ -635,7 +635,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
key, err := s.api.Resolve(r.uri)
if err != nil {
s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)), http.StatusInternalServerError)
return
}
@ -643,9 +643,9 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
if err != nil {
switch status {
case http.StatusNotFound:
s.NotFound(w, r, err)
ShowError(w, &r.Request, fmt.Sprintf("NOT FOUND error serving %s %s: %s", r.Method, r.uri, err), http.StatusNotFound)
default:
s.Error(w, r, err)
ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError)
}
return
}
@ -656,11 +656,11 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
list, err := s.getManifestList(key, r.uri.Path)
if err != nil {
s.Error(w, r, err)
ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError)
return
}
s.logDebug(fmt.Sprintf("Multiple choices! --> %v", list))
log.Debug(fmt.Sprintf("Multiple choices! --> %v", list))
//show a nice page links to available entries
ShowMultipleChoices(w, &r.Request, list)
return
@ -668,7 +668,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
// check the root chunk exists by retrieving the file's size
if _, err := reader.Size(nil); err != nil {
s.NotFound(w, r, fmt.Errorf("File not found %s: %s", r.uri, err))
ShowError(w, &r.Request, fmt.Sprintf("NOT FOUND error serving %s %s: %s", r.Method, r.uri, fmt.Errorf("File not found %s: %s", r.uri, err)), http.StatusNotFound)
return
}
@ -678,16 +678,16 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
}
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
s.logDebug("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, r.URL.Host, r.URL.Path, r.Referer(), r.Header.Get("Accept"))
log.Debug(fmt.Sprintf("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, r.URL.Host, r.URL.Path, r.Referer(), r.Header.Get("Accept")))
uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/"))
req := &Request{Request: *r, uri: uri}
if err != nil {
s.logError("Invalid URI %q: %s", r.URL.Path, err)
s.BadRequest(w, req, fmt.Sprintf("Invalid URI %q: %s", r.URL.Path, err))
log.Error(fmt.Sprintf("Invalid URI %q: %s", r.URL.Path, err))
ShowError(w, r, fmt.Sprintf("Bad request %s %s: %s", r.Method, uri, fmt.Sprintf("Invalid URI %q: %s", r.URL.Path, err)), http.StatusBadRequest)
return
}
s.logDebug("%s request received for %s", r.Method, uri)
log.Debug(fmt.Sprintf("%s request received for %s", r.Method, uri))
switch r.Method {
case "POST":
@ -763,26 +763,6 @@ func (s *Server) updateManifest(key storage.Key, update func(mw *api.ManifestWri
if err != nil {
return nil, err
}
s.logDebug("generated manifest %s", key)
log.Debug(fmt.Sprintf("generated manifest %s", key))
return key, nil
}
func (s *Server) logDebug(format string, v ...interface{}) {
log.Debug(fmt.Sprintf("[BZZ] HTTP: "+format, v...))
}
func (s *Server) logError(format string, v ...interface{}) {
log.Error(fmt.Sprintf("[BZZ] HTTP: "+format, v...))
}
func (s *Server) BadRequest(w http.ResponseWriter, r *Request, reason string) {
ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, reason), http.StatusBadRequest)
}
func (s *Server) Error(w http.ResponseWriter, r *Request, err error) {
ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError)
}
func (s *Server) NotFound(w http.ResponseWriter, r *Request, err error) {
ShowError(w, &r.Request, fmt.Sprintf("NOT FOUND error serving %s %s: %s", r.Method, r.uri, err), http.StatusNotFound)
}

10
swarm/api/http/server_test.go
View file

@ -24,7 +24,6 @@ import (
"io/ioutil"
"net/http"
"strings"
"sync"
"testing"
"github.com/ethereum/go-ethereum/common"
@ -182,15 +181,14 @@ func TestBzzGetPath(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
defer srv.Close()
wg := &sync.WaitGroup{}
for i, mf := range testmanifest {
reader[i] = bytes.NewReader([]byte(mf))
key[i], err = srv.Dpa.Store(reader[i], int64(len(mf)), wg, nil)
var wait func()
key[i], wait, err = srv.Dpa.Store(reader[i], int64(len(mf)))
if err != nil {
t.Fatal(err)
}
wg.Wait()
wait()
}
_, err = http.Get(srv.URL + "/bzz-raw:/" + common.ToHex(key[0])[2:] + "/a")
@ -245,7 +243,7 @@ func TestBzzGetPath(t *testing.T) {
t.Fatalf("Read request body: %v", err)
}
if string(respbody) != key[v].String() {
if string(respbody) != key[v].Hex() {
isexpectedfailrequest := false
for _, r := range expectedfailrequests {

16
swarm/api/manifest.go
View file

@ -24,7 +24,6 @@ import (
"io"
"net/http"
"strings"
"sync"
"time"
"github.com/ethereum/go-ethereum/common"
@ -66,7 +65,9 @@ func (a *Api) NewManifest() (storage.Key, error) {
if err != nil {
return nil, err
}
return a.Store(bytes.NewReader(data), int64(len(data)), &sync.WaitGroup{})
key, wait, err := a.Store(bytes.NewReader(data), int64(len(data)))
wait()
return key, err
}
// ManifestWriter is used to add and remove entries from an underlying manifest
@ -86,12 +87,12 @@ func (a *Api) NewManifestWriter(key storage.Key, quitC chan bool) (*ManifestWrit
// AddEntry stores the given data and adds the resulting key to the manifest
func (m *ManifestWriter) AddEntry(data io.Reader, e *ManifestEntry) (storage.Key, error) {
key, err := m.api.Store(data, e.Size, nil)
key, _, err := m.api.Store(data, e.Size)
if err != nil {
return nil, err
}
entry := newManifestTrieEntry(e, nil)
entry.Hash = key.String()
entry.Hash = key.Hex()
m.trie.addEntry(entry, m.quitC)
return key, nil
}
@ -339,7 +340,7 @@ func (self *manifestTrie) recalcAndStore() error {
if err != nil {
return err
}
entry.Hash = entry.subtrie.hash.String()
entry.Hash = entry.subtrie.hash.Hex()
}
list.Entries = append(list.Entries, entry.ManifestEntry)
}
@ -352,9 +353,8 @@ func (self *manifestTrie) recalcAndStore() error {
}
sr := bytes.NewReader(manifest)
wg := &sync.WaitGroup{}
key, err2 := self.dpa.Store(sr, int64(len(manifest)), wg, nil)
wg.Wait()
key, wait, err2 := self.dpa.Store(sr, int64(len(manifest)))
wait()
self.hash = key
return err2
}

16
swarm/api/storage.go
View file

@ -16,7 +16,11 @@
package api
import "path"
import (
"path"
"github.com/ethereum/go-ethereum/swarm/storage"
)
type Response struct {
MimeType string
@ -41,12 +45,8 @@ func NewStorage(api *Api) *Storage {
// its content type
//
// DEPRECATED: Use the HTTP API instead
func (self *Storage) Put(content, contentType string) (string, error) {
key, err := self.api.Put(content, contentType)
if err != nil {
return "", err
}
return key.String(), err
func (self *Storage) Put(content, contentType string) (storage.Key, func(), error) {
return self.api.Put(content, contentType)
}
// Get retrieves the content from bzzpath and reads the response in full
@ -100,5 +100,5 @@ func (self *Storage) Modify(rootHash, path, contentHash, contentType string) (ne
if err != nil {
return "", err
}
return key.String(), nil
return key.Hex(), nil
}

4
swarm/api/storage_test.go
View file

@ -31,10 +31,12 @@ func TestStoragePutGet(t *testing.T) {
content := "hello"
exp := expResponse(content, "text/plain", 0)
// exp := expResponse([]byte(content), "text/plain", 0)
bzzhash, err := api.Put(content, exp.MimeType)
bzzkey, wait, err := api.Put(content, exp.MimeType)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
wait()
bzzhash := bzzkey.Hex()
// to check put against the Api#Get
resp0 := testGet(t, api.api, bzzhash, "")
checkResponse(t, resp0, exp)

2
swarm/fuse/swarmfs_test.go
View file

@ -808,7 +808,7 @@ func TestFUSE(t *testing.T) {
}
os.RemoveAll(datadir)
dpa, err := storage.NewLocalDPA(datadir)
dpa, err := storage.NewLocalDPA(datadir, make([]byte, 32))
if err != nil {
t.Fatal(err)
}

152
swarm/network/README.md Normal file
View file

@ -0,0 +1,152 @@
## Streaming
Streaming is a new protocol of the swarm bzz bundle of protocols.
This protocol provides the basic logic for chunk-based data flow.
It implements simple retrieve requests and delivery using priority queue.
A data exchange stream is a directional flow of chunks between peers.
The source of datachunks is the upstream, the receiver is called the
downstream peer. Each streaming protocol defines an outgoing streamer
and an incoming streamer, the former installing on the upstream,
the latter on the downstream peer.
Subscribe on StreamerPeer launches an incoming streamer that sends
a subscribe msg upstream. The streamer on the upstream peer
handles the subscribe msg by installing the relevant outgoing streamer
. The modules now engage in a process of upstream sending a sequence of hashes of
chunks downstream (OfferedHashesMsg). The downstream peer evaluates which hashes are needed
and get it delivered by sending back a msg (WantedHashesMsg).
Historical syncing is supported - currently not the right abstraction --
state kept across sessions by saving a series of intervals after their last
batch actually arrived.
Live streaming is also supported, by starting session from the first item
after the subscription.
Provable data exchange. In case a stream represents a swarm document's data layer
or higher level chunks, streaming up to a certain index is always provable. It saves on
sending intermediate chunks.
Using the streamer logic, various stream types are easy to implement:
* light node requests:
* url lookup with offset
* document download
* document upload
* syncing
* live session syncing
* historical syncing
* simple retrieve requests and deliveries
* mutable resource updates streams
* receipting for finger pointing
## Syncing
Syncing is the process that makes sure storer nodes end up storing all and only the chunks that are requested from them.
### Requirements
- eventual consistency: so each chunk historical should be syncable
- since the same chunk can and will arrive from many peers, (network traffic should be
optimised, only one transfer of data per chunk)
- explicit request deliveries should be prioritised higher than recent chunks received
during the ongoing session which in turn should be higher than historical chunks.
- insured chunks should get receipted for finger pointing litigation, the receipts storage
should be organised efficiently, upstream peer should also be able to find these
receipts for a deleted chunk easily to refute their challenge.
- syncing should be resilient to cut connections, metadata should be persisted that
keep track of syncing state across sessions, historical syncing state should survive restart
- extra data structures to support syncing should be kept at minimum
- syncing is organized separately for chunk types (resource update v content chunk)
- various types of streams should have common logic abstracted
Syncing is now entirely mediated by the localstore, ie., no processes or memory leaks due to network contention.
When a new chunk is stored, its chunk hash is index by proximity bin
peers syncronise by getting the chunks closer to the downstream peer than to the upstream one.
Consequently peers just sync all stored items for the kad bin the receiving peer falls into.
The special case of nearest neighbour sets is handled by the downstream peer
indicating they want to sync all kademlia bins with proximity equal to or higher
than their depth.
This sync state represents the initial state of a sync connection session.
Retrieval is dictated by downstream peers simply using a special streamer protocol.
Syncing chunks created during the session by the upstream peer is called live session syncing
while syncing of earlier chunks is historical syncing.
Once the relevant chunk is retrieved, downstream peer looks up all hash segments in its localstore
and sends to the upstream peer a message with a a bitvector to indicate
missing chunks (e.g., for chunk `k`, hash with chunk internal index which case )
new items. In turn upstream peer sends the relevant chunk data alongside their index.
On sending chunks there is a priority queue system. If during looking up hashes in its localstore,
downstream peer hits on an open request then a retrieve request is sent immediately to the upstream peer indicating
that no extra round of checks is needed. If another peers syncer hits the same open request, it is slightly unsafe to not ask
that peer too: if the first one disconnects before delivering or fails to deliver and therefore gets
disconnected, we should still be able to continue with the other. The minimum redundant traffic coming from such simultaneous
eventualities should be sufficiently rare not to warrant more complex treatment.
Session syncing involves downstream peer to request a new state on a bin from upstream.
using the new state, the range (of chunks) between the previous state and the new one are retrieved
and chunks are requested identical to the historical case. After receiving all the missing chunks
from the new hashes, downstream peer will request a new range. If this happens before upstream peer updates a new state,
we say that session syncing is live or the two peers are in sync. In general the time interval passed since downstream peer request up to the current session cursor is a good indication of a permanent (probably increasing) lag.
If there is no historical backlog, and downstream peer has an acceptable 'last synced' tag, then it is said to be fully synced with the upstream peer.
If a peer is fully synced with all its storer peers, it can advertise itself as globally fully synced.
The downstream peer persists the record of the last synced offset. When the two peers disconnect and
reconnect syncing can start from there.
This situation however can also happen while historical syncing is not yet complete.
Effectively this means that the peer needs to persist a record of an arbitrary array of offset ranges covered.
### Delivery requests
once the appropriate ranges of the hashstream are retrieved and buffered, downstream peer just scans the hashes, looks them up in localstore, if not found, create a request entry.
The range is referenced by the chunk index. Alongside the name (indicating the stream, e.g., content chunks for bin 6) and the range
downstream peer sends a 128 long bitvector indicating which chunks are needed.
Newly created requests are satisfied bound together in a waitgroup which when done, will promptt sending the next one.
to be able to do check and storage concurrently, we keep a buffer of one, we start with two batches of hashes.
If there is nothing to give, upstream peers SetNextBatch is blocking. Subscription ends with an unsubscribe. which removes the syncer from the map.
Canceling requests (for instance the late chunks of an erasure batch) should be a chan closed
on the request
Simple request is also a subscribe
different streaming protocols are different p2p protocols with same message types.
the constructor is the Run function itself. which takes a streamerpeer as argument
### provable streams
The swarm hash over the hash stream has many advantages. It implements a provable data transfer
and provide efficient storage for receipts in the form of inclusion proofs useable for finger pointing litigation.
When challenged on a missing chunk, upstream peer will provide an inclusion proof of a chunk hash against the state of the
sync stream. In order to be able to generate such an inclusion proof, upstream peer needs to store the hash index (counting consecutive hash-size segments) alongside the chunk data and preserve it even when the chunk data is deleted until the chunk is no longer insured.
if there is no valid insurance on the files the entry may be deleted.
As long as the chunk is preserved, no takeover proof will be needed since the node can respond to any challenge.
However, once the node needs to delete an insured chunk for capacity reasons, a receipt should be available to
refute the challenge by finger pointing to a downstream peer.
As part of the deletion protocol then, hashes of insured chunks to be removed are pushed to an infinite stream for every bin.
Downstream peer on the other hand needs to make sure that they can only be finger pointed about a chunk they did receive and store.
For this the check of a state should be exhaustive. If historical syncing finishes on one state, all hashes before are covered, no
surprises. In other words historical syncing this process is self verifying. With session syncing however, it is not enough to check going back covering the range from old offset to new. Continuity (i.e., that the new state is extension of the old) needs to be verified: after downstream peer reads the range into a buffer, it appends the buffer the last known state at the last known offset and verifies the resulting hash matches
the latest state. Past intervals of historical syncing are checked via the the session root.
Upstream peer signs the states, downstream peers can use as handover proofs.
Downstream peers sign off on a state together with an initial offset.
Once historical syncing is complete and the session does not lag, downstream peer only preserves the latest upstream state and store the signed version.
Upstream peer needs to keep the latest takeover states: each deleted chunk's hash should be covered by takeover proof of at least one peer. If historical syncing is complete, upstream peer typically will store only the latest takeover proof from downstream peer.
Crucially, the structure is totally independent of the number of peers in the bin, so it scales extremely well.
## implementation
The simplest protocol just involves upstream peer to prefix the key with the kademlia proximity order (say 0-15 or 0-31)
and simply iterate on index per bin when syncing with a peer.
priority queues are used for sending chunks so that user triggered requests should be responded to first, session syncing second, and historical with lower priority.
The request on chunks remains implemented as a dataless entry in the memory store.
The lifecycle of this object should be more carefully thought through, ie., when it fails to retrieve it should be removed.

50
swarm/network/bitvector/bitvector.go Normal file
View file

@ -0,0 +1,50 @@
package bitvector
import (
"errors"
)
var errInvalidLength = errors.New("invalid length")
type BitVector struct {
len int
b []byte
}
func New(l int) (bv *BitVector, err error) {
return NewFromBytes(make([]byte, l/8+1), l)
}
func NewFromBytes(b []byte, l int) (bv *BitVector, err error) {
if l <= 0 {
return nil, errInvalidLength
}
if len(b)*8 < l {
return nil, errInvalidLength
}
return &BitVector{
len: l,
b: b,
}, nil
}
func (bv *BitVector) Get(i int) bool {
bi := i / 8
return bv.b[bi]&(0x1<<uint(i%8)) != 0
}
func (bv *BitVector) Set(i int, v bool) {
bi := i / 8
cv := bv.Get(i)
if cv != v {
bv.b[bi] ^= 0x1 << uint8(i%8)
}
}
func (bv *BitVector) Bytes() []byte {
return bv.b
}
func (bv *BitVector) Length() int {
return bv.len
}

88
swarm/network/bitvector/bitvector_test.go Normal file
View file

@ -0,0 +1,88 @@
package bitvector
import "testing"
func TestBitvectorNew(t *testing.T) {
_, err := New(0)
if err != errInvalidLength {
t.Errorf("expected err %v, got %v", errInvalidLength, err)
}
_, err = NewFromBytes(nil, 0)
if err != errInvalidLength {
t.Errorf("expected err %v, got %v", errInvalidLength, err)
}
_, err = NewFromBytes([]byte{0}, 9)
if err != errInvalidLength {
t.Errorf("expected err %v, got %v", errInvalidLength, err)
}
_, err = NewFromBytes(make([]byte, 8), 8)
if err != nil {
t.Error(err)
}
}
func TestBitvectorGetSet(t *testing.T) {
for _, length := range []int{
1,
2,
4,
8,
9,
15,
16,
} {
bv, err := New(length)
if err != nil {
t.Errorf("error for length %v: %v", length, err)
}
for i := 0; i < length; i++ {
if bv.Get(i) {
t.Errorf("expected false for element on index %v", i)
}
}
func() {
defer func() {
if err := recover(); err == nil {
t.Errorf("expecting panic")
}
}()
bv.Get(length + 8)
}()
for i := 0; i < length; i++ {
bv.Set(i, true)
for j := 0; j < length; j++ {
if j == i {
if !bv.Get(j) {
t.Errorf("element on index %v is not set to true", i)
}
} else {
if bv.Get(j) {
t.Errorf("element on index %v is not false", i)
}
}
}
bv.Set(i, false)
if bv.Get(i) {
t.Errorf("element on index %v is not set to false", i)
}
}
}
}
func TestBitvectorNewFromBytesGet(t *testing.T) {
bv, err := NewFromBytes([]byte{8}, 8)
if err != nil {
t.Error(err)
}
if !bv.Get(3) {
t.Fatalf("element 3 is not set to true: state %08b", bv.b[0])
}
}

8
swarm/network/discovery.go
View file

@ -25,9 +25,9 @@ import (
// discovery bzz extension for requesting and relaying node address records
// discPeer wraps bzzPeer and embeds an Overlay connectivity driver
// discPeer wraps BzzPeer and embeds an Overlay connectivity driver
type discPeer struct {
*bzzPeer
*BzzPeer
overlay Overlay
sentPeers bool // whether we already sent peer closer to this address
mtx sync.Mutex
@ -36,10 +36,10 @@ type discPeer struct {
}
// NewDiscovery constructs a discovery peer
func newDiscovery(p *bzzPeer, o Overlay) *discPeer {
func newDiscovery(p *BzzPeer, o Overlay) *discPeer {
d := &discPeer{
overlay: o,
bzzPeer: p,
BzzPeer: p,
peers: make(map[string]bool),
}
// record remote as seen so we never send a peer its own record

2
swarm/network/discovery_test.go
View file

@ -33,7 +33,7 @@ func TestDiscovery(t *testing.T) {
addr := RandomAddr()
to := NewKademlia(addr.OAddr, NewKadParams())
run := func(p *bzzPeer) error {
run := func(p *BzzPeer) error {
dp := newDiscovery(p, to)
to.On(p)
defer to.Off(p)

4
swarm/network/hive.go
View file

@ -159,7 +159,7 @@ func (h *Hive) connect() {
}
// Run protocol run function
func (h *Hive) Run(p *bzzPeer) error {
func (h *Hive) Run(p *BzzPeer) error {
dp := newDiscovery(p, h)
depth, changed := h.On(dp)
// if we want discovery, advertise changed depth of depth
@ -191,7 +191,7 @@ func ToAddr(pa OverlayPeer) *BzzAddr {
if p, ok := pa.(*discPeer); ok {
return p.BzzAddr
}
return pa.(*bzzPeer).BzzAddr
return pa.(*BzzPeer).BzzAddr
}
// loadPeers, savePeer implement persistence callback/

16
swarm/network/hive_test.go
View file

@ -43,14 +43,12 @@ func TestRegisterAndConnect(t *testing.T) {
pp.Start(s.Server)
defer pp.Stop()
// retrieve and broadcast
s.TestExchanges(p2ptest.Exchange{
Label: "getPeersMsg message",
Expects: []p2ptest.Expect{
{
Code: 2,
Msg: &subPeersMsg{0},
Peer: id,
},
},
err := s.TestDisconnected(&p2ptest.Disconnect{
Peer: s.IDs[0],
Error: nil,
})
if err == nil || err.Error() != "timed out waiting for peers to disconnect" {
t.Fatalf("expected peer to connect")
}
}

24
swarm/network/kademlia.go
View file

@ -54,14 +54,14 @@ var pof = pot.DefaultPof(256)
// KadParams holds the config params for Kademlia
type KadParams struct {
// adjustable parameters
MaxProxDisplay int // number of rows the table shows
MinProxBinSize int // nearest neighbour core minimum cardinality
MinBinSize int // minimum number of peers in a row
MaxBinSize int // maximum number of peers in a row before pruning
RetryInterval uint // initial interval before a peer is first redialed
RetryExponent uint // exponent to multiply retry intervals with
MaxRetries int // maximum number of redial attempts
PruneInterval int // interval between peer pruning cycles
MaxProxDisplay int // number of rows the table shows
MinProxBinSize int // nearest neighbour core minimum cardinality
MinBinSize int // minimum number of peers in a row
MaxBinSize int // maximum number of peers in a row before pruning
RetryInterval int64 // initial interval before a peer is first redialed
RetryExponent int // exponent to multiply retry intervals with
MaxRetries int // maximum number of redial attempts
PruneInterval int // interval between peer pruning cycles
// function to sanction or prevent suggesting a peer
Reachable func(OverlayAddr) bool
}
@ -399,11 +399,11 @@ func (k *Kademlia) callable(val pot.Val) OverlayAddr {
return nil
}
// calculate the allowed number of retries based on time lapsed since last seen
timeAgo := int(time.Since(e.seenAt))
div := int(k.RetryExponent)
div += (150000 - rand.Intn(300000)) * div / 1000000
timeAgo := int64(time.Since(e.seenAt))
div := int64(k.RetryExponent)
div += (150000 - rand.Int63n(300000)) * div / 1000000
var retries int
for delta := timeAgo; uint(delta) > k.RetryInterval; delta /= div {
for delta := timeAgo; delta > k.RetryInterval; delta /= div {
retries++
}

23
swarm/network/kademlia_test.go
View file

@ -70,7 +70,7 @@ func newTestKademlia(b string) *testKademlia {
}
func (k *testKademlia) newTestKadPeer(s string) Peer {
return &testDropPeer{&bzzPeer{BzzAddr: testKadPeerAddr(s)}, k.dropc}
return &testDropPeer{&BzzPeer{BzzAddr: testKadPeerAddr(s)}, k.dropc}
}
func (k *testKademlia) On(ons ...string) *testKademlia {
@ -283,16 +283,15 @@ func TestSuggestPeerFindPeers(t *testing.T) {
func TestSuggestPeerRetries(t *testing.T) {
// 2 row gap, unsaturated proxbin, no callables -> want PO 0
k := newTestKademlia("00000000")
cycle := time.Second
k.RetryInterval = uint(cycle)
k.RetryInterval = int64(time.Second) // cycle
k.MaxRetries = 50
k.RetryExponent = 2
sleep := func(n int) {
t := k.RetryInterval
ts := k.RetryInterval
for i := 1; i < n; i++ {
t *= k.RetryExponent
ts *= int64(k.RetryExponent)
}
time.Sleep(time.Duration(t))
time.Sleep(time.Duration(ts))
}
k.Register("01000000")
@ -399,14 +398,10 @@ func TestPruning(t *testing.T) {
func TestKademliaHiveString(t *testing.T) {
k := newTestKademlia("00000000").On("01000000", "00100000").Register("10000000", "10000001")
k.MaxProxDisplay = 8
h := k.String()
expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), MinProxBinSize: 2, MinBinSize: 1, MaxBinSize: 4\n000 0 | 2 8100 (0) 8000 (0)\n============ DEPTH: 1 ==========================================\n001 1 4000 | 1 4000 (0)\n002 1 2000 | 1 2000 (0)\n"
for i := 3; i < 16; i++ {
expH += fmt.Sprintf("%03d 0 | 0\n", i)
}
expH += "========================================================================="
if expH[106:] != h[106:] {
t.Errorf("incorrect hive output. full - expected %v, got %v", expH, h)
t.Fatalf("incorrect hive output. substr - expected %v, got %v", expH[100:], h[100:])
expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), MinProxBinSize: 2, MinBinSize: 1, MaxBinSize: 4\n000 0 | 2 8100 (0) 8000 (0)\n============ DEPTH: 1 ==========================================\n001 1 4000 | 1 4000 (0)\n002 1 2000 | 1 2000 (0)\n003 0 | 0\n004 0 | 0\n005 0 | 0\n006 0 | 0\n007 0 | 0\n========================================================================="
if expH[104:] != h[104:] {
t.Fatalf("incorrect hive output. expected %v, got %v", expH, h)
}
}

190
swarm/network/light/lightnode.go Normal file
View file

@ -0,0 +1,190 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.d
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package light
import (
"errors"
"github.com/ethereum/go-ethereum/swarm/network/stream"
"github.com/ethereum/go-ethereum/swarm/storage"
)
// RemoteReader implements IncomingStreamer
type RemoteSectionReader struct {
db *storage.DBAPI
start uint64
end uint64
hashes chan []byte
currentHashes []byte
currentData []byte
quit chan struct{}
root []byte
}
// NewRemoteReader is the constructor for RemoteReader
func NewRemoteSectionReader(root []byte, db *storage.DBAPI) *RemoteSectionReader {
return &RemoteSectionReader{
db: db,
root: root,
hashes: make(chan []byte),
quit: make(chan struct{}),
}
}
func (r *RemoteSectionReader) NeedData(key []byte) func() {
chunk, created := r.db.GetOrCreateRequest(storage.Key(key))
// TODO: we may want to request from this peer anyway even if the request exists
if chunk.ReqC == nil || !created {
return nil
}
return func() {
select {
case <-chunk.ReqC:
case <-r.quit:
}
}
}
func (r *RemoteSectionReader) BatchDone(s string, from uint64, hashes []byte, root []byte) func() (*stream.TakeoverProof, error) {
r.hashes <- hashes
return nil
}
func (r *RemoteSectionReader) Read(b []byte) (n int64, err error) {
l := int64(len(b))
m := int64(len(r.currentData))
if m > l {
m = l
}
copy(b, r.currentData[:m])
if m == l {
r.currentData = r.currentData[m:]
return l, nil
}
var end bool
for i := 0; !end && i < len(r.currentHashes); i += stream.HashSize {
hash := r.currentHashes[i : i+stream.HashSize]
chunk, err := r.db.Get(hash)
if err != nil {
return n, err
}
m := chunk.Size
if n+m > l {
m = l - n
end = true
}
copy(b[n:], chunk.SData[:m])
n += m
}
for {
select {
case <-r.quit:
return n, errors.New("aborted")
case hashes := <-r.hashes:
var i int
for ; !end && i < len(hashes); i += stream.HashSize {
hash := hashes[i : i+stream.HashSize]
chunk, err := r.db.Get(hash)
if err != nil {
return n, err
}
m := chunk.Size
if n+m > l {
m = l - n
end = true
}
copy(b[n:], chunk.SData[:m])
n += m
}
hashes = hashes[i:]
}
}
}
func (r *RemoteSectionReader) Close() {}
// RemoteSectionServer implements OutgoingStreamer
type RemoteSectionServer struct {
// quit chan struct{}
root []byte
db *storage.DBAPI
r *storage.LazyChunkReader
}
// NewRemoteReader is the constructor for RemoteReader
func NewRemoteSectionServer(db *storage.DBAPI, r *storage.LazyChunkReader) *RemoteSectionServer {
return &RemoteSectionServer{
db: db,
r: r,
}
}
// GetData retrieves the actual chunk from localstore
func (s *RemoteSectionServer) GetData(key []byte) ([]byte, error) {
chunk, err := s.db.Get(storage.Key(key))
if err != nil {
return nil, err
}
return chunk.SData, nil
}
// GetBatch retrieves the next batch of hashes from the dbstore
func (s *RemoteSectionServer) SetNextBatch(from, to uint64) ([]byte, uint64, uint64, *stream.HandoverProof, error) {
if to > from+stream.BatchSize {
to = from + stream.BatchSize
}
batch := make([]byte, (to-from)*stream.HashSize)
s.r.ReadAt(batch, int64(from))
return batch, from, to, nil, nil
}
func (s *RemoteSectionServer) Close() {}
// RegisterRemoteSectionReader registers RemoteSectionReader on light downstream node
func RegisterRemoteSectionReader(s *stream.Registry, db *storage.DBAPI) {
s.RegisterClientFunc("REMOTE_SECTION", func(p *stream.Peer, t []byte) (stream.Client, error) {
return NewRemoteSectionReader(t, db), nil
})
}
// RegisterRemoteSectionServer registers RemoteSectionServer outgoing streamer on
// upstream light server node
func RegisterRemoteSectionServer(s *stream.Registry, db *storage.DBAPI, rf func([]byte) *storage.LazyChunkReader) {
s.RegisterServerFunc("REMOTE_SECTION", func(p *stream.Peer, t []byte) (stream.Server, error) {
r := rf(t)
return NewRemoteSectionServer(db, r), nil
})
}
// RegisterRemoteDownloader registers RemoteDownloader incoming streamer
// on downstream light node
// func RegisterRemoteDownloader(s *Streamer, db *storage.DBAPI) {
// s.RegisterIncomingStreamer("REMOTE_DOWNLOADER", func(p *stream.Peer, t []byte) (IncomingStreamer, error) {
// return NewRemoteDownloader(t, db), nil
// })
// }
//
// // RegisterRemoteDownloadServer registers RemoteDownloadServer outgoing streamer on
// // upstream light server node
// func RegisterRemoteDownloadServer(s *Streamer, db *storage.DBAPI, rf func([]byte) *storage.LazyChunkReader) {
// s.RegisterOutgoingStreamer("REMOTE_DOWNLOADER", func(p *stream.Peer, t []byte) (OutgoingStreamer, error) {
// r := rf(t)
// return NewRemoteDownloadServer(db, r), nil
// })
// }

95
swarm/network/priorityqueue/priorityqueue.go Normal file
View file

@ -0,0 +1,95 @@
// package priority_queue implement a channel based priority queue
// over arbitrary types. It provides an
// an autopop loop applying a function to the items always respecting
// their priority. The structure is only quasi consistent ie., if a lower
// priority item is autopopped, it is guaranteed that there was a point
// when no higher priority item was present, ie. it is not guaranteed
// that there was any point where the lower priority item was present
// but the higher was not
package priorityqueue
import (
"context"
"errors"
)
var (
errContention = errors.New("queue contention")
errBadPriority = errors.New("bad priority")
wakey = struct{}{}
)
// PriorityQueue is the basic structure
type PriorityQueue struct {
queues []chan interface{}
wakeup chan struct{}
}
// New is the constructor for PriorityQueue
func New(n int, l int) *PriorityQueue {
var queues = make([]chan interface{}, n)
for i := range queues {
queues[i] = make(chan interface{}, l)
}
return &PriorityQueue{
queues: queues,
wakeup: make(chan struct{}, 1),
}
}
// Run is a forever loop popping items from the queues
func (pq *PriorityQueue) Run(ctx context.Context, f func(interface{})) {
top := len(pq.queues) - 1
p := top
READ:
for {
q := pq.queues[p]
select {
case <-ctx.Done():
return
case x := <-q:
f(x)
p = top
default:
if p > 0 {
p--
continue READ
}
p = top
select {
case <-ctx.Done():
return
case <-pq.wakeup:
}
}
}
}
// Push pushes an item to the appropriate queue specified in the priority argument
// if context is given it waits until either the item is pushed or the Context aborts
// otherwise returns errContention if the queue is full
func (pq *PriorityQueue) Push(ctx context.Context, x interface{}, p int) error {
if p < 0 || p >= len(pq.queues) {
return errBadPriority
}
if ctx == nil {
select {
case pq.queues[p] <- x:
default:
return errContention
}
} else {
select {
case pq.queues[p] <- x:
case <-ctx.Done():
return ctx.Err()
}
}
select {
case pq.wakeup <- wakey:
default:
}
return nil
}

82
swarm/network/priorityqueue/priorityqueue_test.go Normal file
View file

@ -0,0 +1,82 @@
package priorityqueue
import (
"context"
"sync"
"testing"
)
func TestPriorityQueue(t *testing.T) {
var results []string
wg := sync.WaitGroup{}
pq := New(3, 2)
wg.Add(1)
go pq.Run(context.Background(), func(v interface{}) {
results = append(results, v.(string))
wg.Done()
})
pq.Push(context.Background(), "2.0", 2)
wg.Wait()
if results[0] != "2.0" {
t.Errorf("expected first result %q, got %q", "2.0", results[0])
}
Loop:
for i, tc := range []struct {
priorities []int
values []string
results []string
errors []error
}{
{
priorities: []int{0},
values: []string{""},
results: []string{""},
},
{
priorities: []int{0, 1},
values: []string{"0.0", "1.0"},
results: []string{"1.0", "0.0"},
},
{
priorities: []int{1, 0},
values: []string{"1.0", "0.0"},
results: []string{"1.0", "0.0"},
},
{
priorities: []int{0, 1, 1},
values: []string{"0.0", "1.0", "1.1"},
results: []string{"1.0", "1.1", "0.0"},
},
{
priorities: []int{0, 0, 0},
values: []string{"0.0", "0.0", "0.1"},
errors: []error{nil, nil, errContention},
},
} {
var results []string
wg := sync.WaitGroup{}
pq := New(3, 2)
wg.Add(len(tc.values))
for j, value := range tc.values {
err := pq.Push(nil, value, tc.priorities[j])
if tc.errors != nil && err != tc.errors[j] {
t.Errorf("expected push error %v, got %v", tc.errors[j], err)
continue Loop
}
if err != nil {
continue Loop
}
}
go pq.Run(context.Background(), func(v interface{}) {
results = append(results, v.(string))
wg.Done()
})
wg.Wait()
for k, result := range tc.results {
if results[k] != result {
t.Errorf("test case %v: expected %v element %q, got %q", i, k, result, results[k])
}
}
}
}

41
swarm/network/protocol.go
View file

@ -175,12 +175,12 @@ func (b *Bzz) APIs() []rpc.API {
// returns a p2p protocol run function that can be assigned to p2p.Protocol#Run field
// arguments:
// * p2p protocol spec
// * run function taking bzzPeer as argument
// * run function taking BzzPeer as argument
// this run function is meant to block for the duration of the protocol session
// on return the session is terminated and the peer is disconnected
// the protocol waits for the bzz handshake is negotiated
// the overlay address on the bzzPeer is set from the remote handshake
func (b *Bzz) RunProtocol(spec *protocols.Spec, run func(*bzzPeer) error) func(*p2p.Peer, p2p.MsgReadWriter) error {
// the overlay address on the BzzPeer is set from the remote handshake
func (b *Bzz) RunProtocol(spec *protocols.Spec, run func(*BzzPeer) error) func(*p2p.Peer, p2p.MsgReadWriter) error {
return func(p *p2p.Peer, rw p2p.MsgReadWriter) error {
// wait for the bzz protocol to perform the handshake
handshake, _ := b.GetHandshake(p.ID())
@ -193,8 +193,8 @@ func (b *Bzz) RunProtocol(spec *protocols.Spec, run func(*bzzPeer) error) func(*
if handshake.err != nil {
return fmt.Errorf("%08x: %s protocol closed: %v", b.BaseAddr()[:4], spec.Name, handshake.err)
}
// the handshake has succeeded so construct the bzzPeer and run the protocol
peer := &bzzPeer{
// the handshake has succeeded so construct the BzzPeer and run the protocol
peer := &BzzPeer{
Peer: protocols.NewPeer(p, rw, spec),
localAddr: b.localAddr,
BzzAddr: handshake.peerAddr,
@ -208,8 +208,10 @@ func (b *Bzz) RunProtocol(spec *protocols.Spec, run func(*bzzPeer) error) func(*
// shared among swarm subprotocols
func performHandshake(p *protocols.Peer, handshake *HandshakeMsg) error {
ctx, cancel := context.WithTimeout(context.Background(), bzzHandshakeTimeout)
defer cancel()
defer close(handshake.done)
defer func() {
close(handshake.done)
cancel()
}()
rsh, err := p.Handshake(ctx, handshake, checkHandshake)
if err != nil {
handshake.err = err
@ -243,22 +245,30 @@ func (b *Bzz) runBzz(p *p2p.Peer, rw p2p.MsgReadWriter) error {
return errors.New("received multiple handshakes")
}
// bzzPeer is the bzz protocol view of a protocols.Peer (itself an extension of p2p.Peer)
// BzzPeer is the bzz protocol view of a protocols.Peer (itself an extension of p2p.Peer)
// implements the Peer interface and all interfaces Peer implements: Addr, OverlayPeer
type bzzPeer struct {
type BzzPeer struct {
*protocols.Peer // represents the connection for online peers
localAddr *BzzAddr // local Peers address
*BzzAddr // remote address -> implements Addr interface = protocols.Peer
lastActive time.Time // time is updated whenever mutexes are releasing
}
func NewBzzTestPeer(p *protocols.Peer, addr *BzzAddr) *BzzPeer {
return &BzzPeer{
Peer: p,
localAddr: addr,
BzzAddr: NewAddrFromNodeID(p.ID()),
}
}
// Off returns the overlay peer record for offline persistence
func (p *bzzPeer) Off() OverlayAddr {
func (p *BzzPeer) Off() OverlayAddr {
return p.BzzAddr
}
// LastActive returns the time the peer was last active
func (p *bzzPeer) LastActive() time.Time {
func (p *BzzPeer) LastActive() time.Time {
return p.lastActive
}
@ -393,6 +403,15 @@ func NewAddrFromNodeID(id discover.NodeID) *BzzAddr {
}
}
// NewAddrFromNodeIDAndPort constucts a BzzAddr from a discover.NodeID and port uint16
// the overlay address is derived as the hash of the nodeID
func NewAddrFromNodeIDAndPort(id discover.NodeID, port uint16) *BzzAddr {
return &BzzAddr{
OAddr: ToOverlayAddr(id.Bytes()),
UAddr: []byte(discover.NewNode(id, net.IP{127, 0, 0, 1}, port, port).String()),
}
}
// ToOverlayAddr creates an overlayaddress from a byte slice
func ToOverlayAddr(id []byte) []byte {
return crypto.Keccak256(id)

21
swarm/network/protocol_test.go
View file

@ -17,16 +17,29 @@
package network
import (
"flag"
"fmt"
"os"
"sync"
"testing"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/discover"
"github.com/ethereum/go-ethereum/p2p/protocols"
p2ptest "github.com/ethereum/go-ethereum/p2p/testing"
)
var (
adapter = flag.String("adapter", "sim", "type of simulation: sim|socket|exec|docker")
loglevel = flag.Int("loglevel", 2, "verbosity of logs")
)
func init() {
flag.Parse()
log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
}
type testStore struct {
sync.Mutex
@ -78,16 +91,16 @@ func HandshakeMsgExchange(lhs, rhs *HandshakeMsg, id discover.NodeID) []p2ptest.
}
}
func newBzzBaseTester(t *testing.T, n int, addr *BzzAddr, spec *protocols.Spec, run func(*bzzPeer) error) *bzzTester {
func newBzzBaseTester(t *testing.T, n int, addr *BzzAddr, spec *protocols.Spec, run func(*BzzPeer) error) *bzzTester {
cs := make(map[string]chan bool)
srv := func(p *bzzPeer) error {
srv := func(p *BzzPeer) error {
defer close(cs[p.ID().String()])
return run(p)
}
protocall := func(p *p2p.Peer, rw p2p.MsgReadWriter) error {
return srv(&bzzPeer{
return srv(&BzzPeer{
Peer: protocols.NewPeer(p, rw, spec),
localAddr: addr,
BzzAddr: NewAddrFromNodeID(p.ID()),
@ -115,7 +128,7 @@ type bzzTester struct {
func newBzzTester(t *testing.T, n int, addr *BzzAddr, pp *p2ptest.TestPeerPool, spec *protocols.Spec, services func(Peer) error) *bzzTester {
extraservices := func(p *bzzPeer) error {
extraservices := func(p *BzzPeer) error {
pp.Add(p)
defer pp.Remove(p)
if services == nil {

30
swarm/network/simulations/discovery/discovery_test.go
View file

@ -1,4 +1,4 @@
package discovery_test
package discovery
import (
"context"
@ -70,15 +70,18 @@ func BenchmarkDiscovery_64_4(b *testing.B) { benchmarkDiscovery(b, 64, 4) }
func BenchmarkDiscovery_128_4(b *testing.B) { benchmarkDiscovery(b, 128, 4) }
func BenchmarkDiscovery_256_4(b *testing.B) { benchmarkDiscovery(b, 256, 4) }
func TestDiscoverySimulationDockerAdapter(t *testing.T) {
t.Skip("broken (cannot build image)")
func XTestDiscoverySimulationDockerAdapter(t *testing.T) {
testDiscoverySimulationDockerAdapter(t, *nodeCount, *initCount)
}
func testDiscoverySimulationDockerAdapter(t *testing.T, nodes, conns int) {
adapter, err := adapters.NewDockerAdapter()
if err != nil {
t.Fatal(err)
if err == adapters.ErrLinuxOnly {
t.Skip(err)
} else {
t.Fatal(err)
}
}
testDiscoverySimulation(t, nodes, conns, adapter)
}
@ -97,8 +100,20 @@ func testDiscoverySimulationExecAdapter(t *testing.T, nodes, conns int) {
testDiscoverySimulation(t, nodes, conns, adapters.NewExecAdapter(baseDir))
}
func TestDiscoverySimulationSocketAdapter(t *testing.T) {
testDiscoverySimulationSocketAdapter(t, *nodeCount, *initCount)
}
func TestDiscoverySimulationSimAdapter(t *testing.T) {
testDiscoverySimulation(t, *nodeCount, *initCount, adapters.NewSimAdapter(services))
testDiscoverySimulationSimAdapter(t, *nodeCount, *initCount)
}
func testDiscoverySimulationSimAdapter(t *testing.T, nodes, conns int) {
testDiscoverySimulation(t, nodes, conns, adapters.NewSimAdapter(services))
}
func testDiscoverySimulationSocketAdapter(t *testing.T, nodes, conns int) {
testDiscoverySimulation(t, nodes, conns, adapters.NewSocketAdapter(services))
}
func testDiscoverySimulation(t *testing.T, nodes, conns int, adapter adapters.NodeAdapter) {
@ -150,7 +165,8 @@ func discoverySimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simul
trigger := make(chan discover.NodeID)
ids := make([]discover.NodeID, nodes)
for i := 0; i < nodes; i++ {
node, err := net.NewNode()
conf := adapters.RandomNodeConfig()
node, err := net.NewNodeWithConfig(conf)
if err != nil {
return nil, fmt.Errorf("error starting node: %s", err)
}
@ -290,7 +306,7 @@ func triggerChecks(trigger chan discover.NodeID, net *simulations.Network, id di
}
func newService(ctx *adapters.ServiceContext) (node.Service, error) {
addr := network.NewAddrFromNodeID(ctx.Config.ID)
addr := network.NewAddrFromNodeIDAndPort(ctx.Config.ID, ctx.Config.Port)
kp := network.NewKadParams()
kp.MinProxBinSize = testMinProxBinSize

152
swarm/network/stream/common_test.go Normal file
View file

@ -0,0 +1,152 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package stream
import (
"errors"
"flag"
"io/ioutil"
"os"
"sync/atomic"
"testing"
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
p2ptest "github.com/ethereum/go-ethereum/p2p/testing"
"github.com/ethereum/go-ethereum/swarm/network"
"github.com/ethereum/go-ethereum/swarm/storage"
)
var (
adapter = flag.String("adapter", "sim", "type of simulation: sim|socket|exec|docker")
loglevel = flag.Int("loglevel", 2, "verbosity of logs")
)
var (
defaultSkipCheck bool
waitPeerErrC chan error
chunkSize = 4096
)
var services = adapters.Services{
"streamer": NewStreamerService,
}
func init() {
flag.Parse()
// register the Delivery service which will run as a devp2p
// protocol when using the exec adapter
adapters.RegisterServices(services)
log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(false))))
}
// NewStreamerService
func NewStreamerService(ctx *adapters.ServiceContext) (node.Service, error) {
id := ctx.Config.ID
addr := toAddr(id)
kad := network.NewKademlia(addr.Over(), network.NewKadParams())
store := stores[id].(*storage.LocalStore)
db := storage.NewDBAPI(store)
delivery := NewDelivery(kad, db)
deliveries[id] = delivery
netStore := storage.NewNetStore(store, nil)
r := NewRegistry(addr, delivery, netStore, defaultSkipCheck)
RegisterSwarmSyncerServer(r, db)
RegisterSwarmSyncerClient(r, db)
go func() {
waitPeerErrC <- waitForPeers(r, 1*time.Second, peerCount(id))
}()
return r, nil
}
func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Registry, *storage.LocalStore, func(), error) {
// setup
addr := network.RandomAddr() // tested peers peer address
to := network.NewKademlia(addr.OAddr, network.NewKadParams())
// temp datadir
datadir, err := ioutil.TempDir("", "streamer")
if err != nil {
return nil, nil, nil, func() {}, err
}
teardown := func() {
os.RemoveAll(datadir)
}
localStore, err := storage.NewTestLocalStoreForAddr(datadir, addr.Over())
if err != nil {
return nil, nil, nil, teardown, err
}
db := storage.NewDBAPI(localStore)
delivery := NewDelivery(to, db)
streamer := NewRegistry(addr, delivery, localStore, defaultSkipCheck)
protocolTester := p2ptest.NewProtocolTester(t, network.NewNodeIDFromAddr(addr), 1, streamer.runProtocol)
err = waitForPeers(streamer, 1*time.Second, 1)
if err != nil {
return nil, nil, nil, nil, errors.New("timeout: peer is not created")
}
return protocolTester, streamer, localStore, teardown, nil
}
func waitForPeers(streamer *Registry, timeout time.Duration, expectedPeers int) error {
ticker := time.NewTicker(10 * time.Millisecond)
timeoutTimer := time.NewTimer(timeout)
for {
select {
case <-ticker.C:
if streamer.peersCount() >= expectedPeers {
return nil
}
case <-timeoutTimer.C:
return errors.New("timeout")
}
}
}
type roundRobinStore struct {
index uint32
stores []storage.ChunkStore
}
func newRoundRobinStore(stores ...storage.ChunkStore) *roundRobinStore {
return &roundRobinStore{
stores: stores,
}
}
func (rrs *roundRobinStore) Get(key storage.Key) (*storage.Chunk, error) {
return nil, errors.New("get not well defined on round robin store")
}
func (rrs *roundRobinStore) Put(chunk *storage.Chunk) {
i := atomic.AddUint32(&rrs.index, 1)
idx := int(i) % len(rrs.stores)
rrs.stores[idx].Put(chunk)
}
func (rrs *roundRobinStore) Close() {
for _, store := range rrs.stores {
store.Close()
}
}

244
swarm/network/stream/delivery.go Normal file
View file

@ -0,0 +1,244 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package stream
import (
"bytes"
"errors"
"fmt"
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/p2p/discover"
"github.com/ethereum/go-ethereum/swarm/network"
"github.com/ethereum/go-ethereum/swarm/storage"
)
const (
swarmChunkServerStreamName = "RETRIEVE_REQUEST"
deliveryCap = 32
)
type Delivery struct {
db *storage.DBAPI
overlay network.Overlay
receiveC chan *ChunkDeliveryMsg
getPeer func(discover.NodeID) *Peer
quit chan struct{}
}
func NewDelivery(overlay network.Overlay, db *storage.DBAPI) *Delivery {
d := &Delivery{
db: db,
overlay: overlay,
receiveC: make(chan *ChunkDeliveryMsg, deliveryCap),
}
go d.processReceivedChunks()
return d
}
// SwarmChunkServer implements Server
type SwarmChunkServer struct {
deliveryC chan []byte
batchC chan []byte
db *storage.DBAPI
currentLen uint64
quit chan struct{}
}
// NewSwarmChunkServer is SwarmChunkServer constructor
func NewSwarmChunkServer(db *storage.DBAPI) *SwarmChunkServer {
s := &SwarmChunkServer{
deliveryC: make(chan []byte, deliveryCap),
batchC: make(chan []byte),
db: db,
quit: make(chan struct{}),
}
go s.processDeliveries()
return s
}
// processDeliveries handles delivered chunk hashes
func (s *SwarmChunkServer) processDeliveries() {
var hashes []byte
var batchC chan []byte
for {
select {
case <-s.quit:
return
case hash := <-s.deliveryC:
hashes = append(hashes, hash...)
batchC = s.batchC
case batchC <- hashes:
hashes = nil
batchC = nil
}
}
}
// SetNextBatch
func (s *SwarmChunkServer) SetNextBatch(_, _ uint64) (hashes []byte, from uint64, to uint64, proof *HandoverProof, err error) {
select {
case hashes = <-s.batchC:
case <-s.quit:
return
}
from = s.currentLen
s.currentLen += uint64(len(hashes))
to = s.currentLen
return
}
// Close needs to be called on a stream server
func (s *SwarmChunkServer) Close() {
close(s.quit)
}
// GetData retrives chunk data from db store
func (s *SwarmChunkServer) GetData(key []byte) ([]byte, error) {
chunk, err := s.db.Get(storage.Key(key))
if err == storage.ErrFetching {
<-chunk.ReqC
} else if err != nil {
return nil, err
}
return chunk.SData, nil
}
// RetrieveRequestMsg is the protocol msg for chunk retrieve requests
type RetrieveRequestMsg struct {
Key storage.Key
SkipCheck bool
}
func (d *Delivery) handleRetrieveRequestMsg(sp *Peer, req *RetrieveRequestMsg) error {
log.Debug("received request", "peer", sp.ID(), "hash", req.Key)
s, err := sp.getServer(swarmChunkServerStreamName)
if err != nil {
return err
}
streamer := s.Server.(*SwarmChunkServer)
chunk, created := d.db.GetOrCreateRequest(req.Key)
if chunk.ReqC != nil {
if created {
if err := d.RequestFromPeers(chunk.Key[:], false, sp.ID()); err != nil {
log.Warn("unable to forward chunk request", "peer", sp.ID(), "key", chunk.Key, "err", err)
return nil
}
}
go func() {
t := time.NewTimer(3 * time.Minute)
defer t.Stop()
select {
case <-chunk.ReqC:
case <-d.quit:
return
case <-t.C:
return
}
if req.SkipCheck {
err := sp.Deliver(chunk, s.priority)
if err != nil {
sp.Drop(err)
}
}
streamer.deliveryC <- chunk.Key[:]
}()
return nil
}
// TODO: call the retrieve function of the outgoing syncer
if req.SkipCheck {
log.Trace("deliver", "peer", sp.ID(), "hash", chunk.Key)
return sp.Deliver(chunk, s.priority)
}
streamer.deliveryC <- chunk.Key[:]
return nil
}
type ChunkDeliveryMsg struct {
Key storage.Key
SData []byte // the stored chunk Data (incl size)
peer *Peer // set in handleChunkDeliveryMsg
}
func (d *Delivery) handleChunkDeliveryMsg(sp *Peer, req *ChunkDeliveryMsg) error {
req.peer = sp
d.receiveC <- req
return nil
}
func (d *Delivery) processReceivedChunks() {
R:
for req := range d.receiveC {
// this should be has locally
chunk, err := d.db.Get(req.Key)
if !bytes.Equal(chunk.Key, req.Key) {
panic(fmt.Errorf("processReceivedChunks: chunk key %s != req key %s (peer %s)", chunk.Key.Hex(), req.Key.Hex(), req.peer.ID()))
}
if err == nil {
continue R
}
if err != storage.ErrFetching {
panic(fmt.Sprintf("not in db? key %v chunk %v", req.Key, chunk))
}
select {
case <-chunk.ReqC:
log.Error("someone else delivered?", "hash", chunk.Key.Hex())
continue R
default:
}
chunk.SData = req.SData
d.db.Put(chunk)
chunk.WaitToStore()
close(chunk.ReqC)
}
}
// RequestFromPeers sends a chunk retrieve request to
func (d *Delivery) RequestFromPeers(hash []byte, skipCheck bool, peersToSkip ...discover.NodeID) error {
var success bool
var err error
d.overlay.EachConn(hash, 255, func(p network.OverlayConn, po int, nn bool) bool {
spId := p.(*network.BzzPeer).ID()
for _, p := range peersToSkip {
if p == spId {
log.Trace("Delivery.RequestFromPeers: skip peer", "peer", spId)
return true
}
}
sp := d.getPeer(spId)
if sp == nil {
log.Warn("Delivery.RequestFromPeers: peer not found", "id", spId)
return true
}
// TODO: skip light nodes that do not accept retrieve requests
err = sp.SendPriority(&RetrieveRequestMsg{
Key: hash,
SkipCheck: skipCheck,
}, Top)
success = true
return false
})
if success {
return err
}
return errors.New("no peer found")
}

688
swarm/network/stream/delivery_test.go Normal file
View file

@ -0,0 +1,688 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package stream
import (
"bytes"
"context"
crand "crypto/rand"
"fmt"
"io"
"testing"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/p2p/discover"
"github.com/ethereum/go-ethereum/p2p/simulations"
p2ptest "github.com/ethereum/go-ethereum/p2p/testing"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/swarm/network"
streamTesting "github.com/ethereum/go-ethereum/swarm/network/stream/testing"
"github.com/ethereum/go-ethereum/swarm/storage"
)
var (
deliveries map[discover.NodeID]*Delivery
stores map[discover.NodeID]storage.ChunkStore
toAddr func(discover.NodeID) *network.BzzAddr
peerCount func(discover.NodeID) int
)
func TestStreamerRetrieveRequest(t *testing.T) {
tester, streamer, _, teardown, err := newStreamerTester(t)
defer teardown()
if err != nil {
t.Fatal(err)
}
peerID := tester.IDs[0]
streamer.delivery.RequestFromPeers(hash0[:], true)
err = tester.TestExchanges(p2ptest.Exchange{
Label: "RetrieveRequestMsg",
Expects: []p2ptest.Expect{
{
Code: 5,
Msg: &RetrieveRequestMsg{
Key: hash0[:],
SkipCheck: true,
},
Peer: peerID,
},
},
})
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
}
func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) {
tester, streamer, _, teardown, err := newStreamerTester(t)
defer teardown()
if err != nil {
t.Fatal(err)
}
peerID := tester.IDs[0]
chunk := storage.NewChunk(storage.Key(hash0[:]), nil)
peer := streamer.getPeer(peerID)
peer.handleSubscribeMsg(&SubscribeMsg{
Stream: swarmChunkServerStreamName,
Key: nil,
From: 0,
To: 0,
Priority: Top,
})
err = tester.TestExchanges(p2ptest.Exchange{
Label: "RetrieveRequestMsg",
Triggers: []p2ptest.Trigger{
{
Code: 5,
Msg: &RetrieveRequestMsg{
Key: chunk.Key[:],
},
Peer: peerID,
},
},
Expects: []p2ptest.Expect{
{
Code: 1,
Msg: &OfferedHashesMsg{
HandoverProof: nil,
Hashes: nil,
From: 0,
To: 0,
},
Peer: peerID,
},
},
})
expectedError := "exchange 0: 'RetrieveRequestMsg' timed out"
if err == nil || err.Error() != expectedError {
t.Fatalf("Expected error %v, got %v", expectedError, err)
}
}
// upstream request server receives a retrieve Request and responds with
// offered hashes or delivery if skipHash is set to true
func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) {
tester, streamer, localStore, teardown, err := newStreamerTester(t)
defer teardown()
if err != nil {
t.Fatal(err)
}
peerID := tester.IDs[0]
peer := streamer.getPeer(peerID)
peer.handleSubscribeMsg(&SubscribeMsg{
Stream: swarmChunkServerStreamName,
Key: nil,
From: 0,
To: 0,
Priority: Top,
})
hash := storage.Key(hash0[:])
chunk := storage.NewChunk(hash, nil)
chunk.SData = hash
localStore.Put(chunk)
chunk.WaitToStore()
err = tester.TestExchanges(p2ptest.Exchange{
Label: "RetrieveRequestMsg",
Triggers: []p2ptest.Trigger{
{
Code: 5,
Msg: &RetrieveRequestMsg{
Key: hash,
},
Peer: peerID,
},
},
Expects: []p2ptest.Expect{
{
Code: 1,
Msg: &OfferedHashesMsg{
HandoverProof: &HandoverProof{
Handover: &Handover{},
},
Hashes: hash,
From: 0,
// TODO: why is this 32???
To: 32,
Key: []byte{},
Stream: swarmChunkServerStreamName,
},
Peer: peerID,
},
},
})
if err != nil {
t.Fatal(err)
}
hash = storage.Key(hash1[:])
chunk = storage.NewChunk(hash, nil)
chunk.SData = hash1[:]
localStore.Put(chunk)
chunk.WaitToStore()
err = tester.TestExchanges(p2ptest.Exchange{
Label: "RetrieveRequestMsg",
Triggers: []p2ptest.Trigger{
{
Code: 5,
Msg: &RetrieveRequestMsg{
Key: hash,
SkipCheck: true,
},
Peer: peerID,
},
},
Expects: []p2ptest.Expect{
{
Code: 6,
Msg: &ChunkDeliveryMsg{
Key: hash,
SData: hash,
},
Peer: peerID,
},
},
})
if err != nil {
t.Fatal(err)
}
}
func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) {
tester, streamer, localStore, teardown, err := newStreamerTester(t)
defer teardown()
if err != nil {
t.Fatal(err)
}
streamer.RegisterClientFunc("foo", func(p *Peer, t []byte) (Client, error) {
return &testClient{
t: t,
}, nil
})
peerID := tester.IDs[0]
err = streamer.Subscribe(peerID, "foo", []byte{}, 5, 8, Top, true)
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
chunkKey := hash0[:]
chunkData := hash1[:]
chunk, created := localStore.GetOrCreateRequest(chunkKey)
if !created {
t.Fatal("chunk already exists")
}
select {
case <-chunk.ReqC:
t.Fatal("chunk is already received")
default:
}
err = tester.TestExchanges(p2ptest.Exchange{
Label: "Subscribe message",
Expects: []p2ptest.Expect{
{
Code: 4,
Msg: &SubscribeMsg{
Stream: "foo",
Key: []byte{},
From: 5,
To: 8,
Priority: Top,
},
Peer: peerID,
},
},
},
p2ptest.Exchange{
Label: "ChunkDeliveryRequest message",
Triggers: []p2ptest.Trigger{
{
Code: 6,
Msg: &ChunkDeliveryMsg{
Key: chunkKey,
SData: chunkData,
},
Peer: peerID,
},
},
})
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
timeout := time.NewTimer(1 * time.Second)
select {
case <-timeout.C:
t.Fatal("timeout receiving chunk")
case <-chunk.ReqC:
}
storedChunk, err := localStore.Get(chunkKey)
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
if !bytes.Equal(storedChunk.SData, chunkData) {
t.Fatal("Retrieved chunk has different data than original")
}
}
func TestDeliveryFromNodes(t *testing.T) {
testDeliveryFromNodes(t, 2, 1, dataChunkCount, true)
testDeliveryFromNodes(t, 2, 1, dataChunkCount, false)
testDeliveryFromNodes(t, 4, 1, dataChunkCount, true)
testDeliveryFromNodes(t, 4, 1, dataChunkCount, false)
testDeliveryFromNodes(t, 8, 1, dataChunkCount, true)
testDeliveryFromNodes(t, 8, 1, dataChunkCount, false)
testDeliveryFromNodes(t, 16, 1, dataChunkCount, true)
testDeliveryFromNodes(t, 16, 1, dataChunkCount, false)
}
func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck bool) {
defaultSkipCheck = skipCheck
toAddr = network.NewAddrFromNodeID
conf := &streamTesting.RunConfig{
Adapter: *adapter,
NodeCount: nodes,
ConnLevel: conns,
ToAddr: toAddr,
Services: services,
EnableMsgEvents: false,
}
sim, teardown, err := streamTesting.NewSimulation(conf)
defer teardown()
if err != nil {
t.Fatal(err.Error())
}
stores = make(map[discover.NodeID]storage.ChunkStore)
deliveries = make(map[discover.NodeID]*Delivery)
for i, id := range sim.IDs {
stores[id] = sim.Stores[i]
}
peerCount = func(id discover.NodeID) int {
if sim.IDs[0] == id || sim.IDs[nodes-1] == id {
return 1
}
return 2
}
// here we distribute chunks of a random file into Stores of nodes 1 to nodes
rrdpa := storage.NewDPA(newRoundRobinStore(sim.Stores[1:]...), storage.NewChunkerParams())
rrdpa.Start()
size := chunkCount * chunkSize
fileHash, wait, err := rrdpa.Store(io.LimitReader(crand.Reader, int64(size)), int64(size))
// wait until all chunks stored
wait()
defer rrdpa.Stop()
if err != nil {
t.Fatal(err.Error())
}
errc := make(chan error, 1)
waitPeerErrC = make(chan error)
quitC := make(chan struct{})
action := func(ctx context.Context) error {
// each node Subscribes to each other's swarmChunkServerStreamName
// need to wait till an aynchronous process registers the peers in streamer.peers
// that is used by Subscribe
// using a global err channel to share betweem action and node service
i := 0
for err := range waitPeerErrC {
if err != nil {
return fmt.Errorf("error waiting for peers: %s", err)
}
i++
if i == nodes {
break
}
}
// each node subscribes to the upstream swarm chunk server stream
// which responds to chunk retrieve requests all but the last node in the chain does not
for j := 0; j < nodes-1; j++ {
id := sim.IDs[j]
err := sim.CallClient(id, func(client *rpc.Client) error {
err := streamTesting.WatchDisconnections(id, client, errc, quitC)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(ctx, 1*time.Second)
defer cancel()
sid := sim.IDs[j+1]
return client.CallContext(ctx, nil, "stream_subscribeStream", sid, swarmChunkServerStreamName, nil, 0, 0, Top, false)
})
if err != nil {
return err
}
}
// create a retriever dpa for the pivot node
delivery := deliveries[sim.IDs[0]]
retrieveFunc := func(chunk *storage.Chunk) error {
return delivery.RequestFromPeers(chunk.Key[:], skipCheck)
}
netStore := storage.NewNetStore(sim.Stores[0].(*storage.LocalStore), retrieveFunc)
dpa := storage.NewDPA(netStore, storage.NewChunkerParams())
dpa.Start()
go func() {
defer dpa.Stop()
// start the retrieval on the pivot node - this will spawn retrieve requests for missing chunks
// we must wait for the peer connections to have started before requesting
n, err := readAll(dpa, fileHash)
log.Info(fmt.Sprintf("retrieved %v", fileHash), "read", n, "err", err)
if err != nil {
errc <- fmt.Errorf("requesting chunks action error: %v", err)
}
}()
return nil
}
check := func(ctx context.Context, id discover.NodeID) (bool, error) {
select {
case err := <-errc:
return false, err
case <-ctx.Done():
return false, ctx.Err()
default:
}
var total int64
err := sim.CallClient(id, func(client *rpc.Client) error {
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
return client.CallContext(ctx, &total, "stream_readAll", common.BytesToHash(fileHash))
})
log.Info(fmt.Sprintf("check if %08x is available locally: number of bytes read %v/%v (error: %v)", fileHash, total, size, err))
if err != nil || total != int64(size) {
return false, nil
}
return true, nil
}
conf.Step = &simulations.Step{
Action: action,
Trigger: streamTesting.Trigger(10*time.Millisecond, quitC, sim.IDs[0]),
// we are only testing the pivot node (net.Nodes[0])
Expect: &simulations.Expectation{
Nodes: sim.IDs[0:1],
Check: check,
},
}
startedAt := time.Now()
timeout := 300 * time.Second
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
result, err := sim.Run(ctx, conf)
finishedAt := time.Now()
if err != nil {
t.Fatalf("Setting up simulation failed: %v", err)
}
if result.Error != nil {
t.Fatalf("Simulation failed: %s", result.Error)
}
streamTesting.CheckResult(t, result, startedAt, finishedAt)
}
func BenchmarkDeliveryFromNodesWithoutCheck(b *testing.B) {
for chunks := 32; chunks <= 128; chunks *= 2 {
for i := 2; i < 32; i *= 2 {
b.Run(
fmt.Sprintf("nodes=%v,chunks=%v", i, chunks),
func(b *testing.B) {
benchmarkDeliveryFromNodes(b, i, 1, chunks, true)
},
)
}
}
}
func BenchmarkDeliveryFromNodesWithCheck(b *testing.B) {
for chunks := 32; chunks <= 128; chunks *= 2 {
for i := 2; i < 32; i *= 2 {
b.Run(
fmt.Sprintf("nodes=%v,chunks=%v", i, chunks),
func(b *testing.B) {
benchmarkDeliveryFromNodes(b, i, 1, chunks, false)
},
)
}
}
}
func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skipCheck bool) {
defaultSkipCheck = skipCheck
toAddr = network.NewAddrFromNodeID
timeout := 300 * time.Second
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
conf := &streamTesting.RunConfig{
Adapter: *adapter,
NodeCount: nodes,
ConnLevel: conns,
ToAddr: toAddr,
Services: services,
EnableMsgEvents: false,
}
sim, teardown, err := streamTesting.NewSimulation(conf)
defer teardown()
if err != nil {
b.Fatal(err.Error())
}
stores = make(map[discover.NodeID]storage.ChunkStore)
deliveries = make(map[discover.NodeID]*Delivery)
for i, id := range sim.IDs {
stores[id] = sim.Stores[i]
}
peerCount = func(id discover.NodeID) int {
if sim.IDs[0] == id || sim.IDs[nodes-1] == id {
return 1
}
return 2
}
// wait channel for all nodes all peer connections to set up
waitPeerErrC = make(chan error)
// create a dpa for the last node in the chain which we are gonna write to
remoteDpa := storage.NewDPA(sim.Stores[nodes-1], storage.NewChunkerParams())
remoteDpa.Start()
defer remoteDpa.Stop()
// channel to signal simulation initialisation with action call complete
// or node disconnections
disconnectC := make(chan error)
quitC := make(chan struct{})
initC := make(chan error)
action := func(ctx context.Context) error {
// each node Subscribes to each other's swarmChunkServerStreamName
// need to wait till an aynchronous process registers the peers in streamer.peers
// that is used by Subscribe
// waitPeerErrC using a global err channel to share betweem action and node service
i := 0
for err := range waitPeerErrC {
if err != nil {
return fmt.Errorf("error waiting for peers: %s", err)
}
i++
if i == nodes {
break
}
}
var err error
// each node except the last one subscribes to the upstream swarm chunk server stream
// which responds to chunk retrieve requests
for j := 0; j < nodes-1; j++ {
id := sim.IDs[j]
err = sim.CallClient(id, func(client *rpc.Client) error {
err := streamTesting.WatchDisconnections(id, client, disconnectC, quitC)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(ctx, 1*time.Second)
defer cancel()
sid := sim.IDs[j+1] // the upstream peer's id
return client.CallContext(ctx, nil, "stream_subscribeStream", sid, swarmChunkServerStreamName, nil, 0, 0, Top, false)
})
if err != nil {
break
}
}
initC <- err
return nil
}
// the check function is only triggered when the benchmark finishes
trigger := make(chan discover.NodeID)
check := func(ctx context.Context, id discover.NodeID) (_ bool, err error) {
return true, nil
}
conf.Step = &simulations.Step{
Action: action,
Trigger: trigger,
// we are only testing the pivot node (net.Nodes[0])
Expect: &simulations.Expectation{
Nodes: sim.IDs[0:1],
Check: check,
},
}
// run the simulation in the background
errc := make(chan error)
go func() {
_, err := sim.Run(ctx, conf)
close(quitC)
errc <- err
}()
// wait for simulation action to complete stream subscriptions
err = <-initC
if err != nil {
b.Fatalf("simulation failed to initialise. expected no error. got %v", err)
}
// create a retriever dpa for the pivot node
// by now deliveries are set for each node by the streamer service
delivery := deliveries[sim.IDs[0]]
retrieveFunc := func(chunk *storage.Chunk) error {
return delivery.RequestFromPeers(chunk.Key[:], skipCheck)
}
netStore := storage.NewNetStore(sim.Stores[0].(*storage.LocalStore), retrieveFunc)
// benchmark loop
b.ResetTimer()
b.StopTimer()
Loop:
for i := 0; i < b.N; i++ {
// uploading chunkCount random chunks to the last node
hashes := make([]storage.Key, chunkCount)
for i := 0; i < chunkCount; i++ {
// create actual size real chunks
hash, wait, err := remoteDpa.Store(io.LimitReader(crand.Reader, int64(chunkSize)), int64(chunkSize))
// wait until all chunks stored
wait()
if err != nil {
b.Fatalf("expected no error. got %v", err)
}
// collect the hashes
hashes[i] = hash
}
// now benchmark the actual retrieval
// netstore.Get is called for each hash in a go routine and errors are collected
b.StartTimer()
errs := make(chan error)
for _, hash := range hashes {
go func(h storage.Key) {
_, err := netStore.Get(h)
log.Warn("test check netstore get", "hash", h, "err", err)
errs <- err
}(hash)
}
// count and report retrieval errors
// if there are misses then chunk timeout is too low for the distance and volume (?)
var total, misses int
for err := range errs {
if err != nil {
log.Warn(err.Error())
misses++
}
total++
if total == chunkCount {
break
}
}
b.StopTimer()
select {
case err = <-disconnectC:
if err != nil {
break Loop
}
default:
}
if misses > 0 {
err = fmt.Errorf("%v chunk not found out of %v", misses, total)
break Loop
}
}
select {
case <-quitC:
case trigger <- sim.IDs[0]:
}
if err == nil {
err = <-errc
} else {
if e := <-errc; e != nil {
b.Errorf("sim.Run function error: %v", e)
}
}
// benchmark over, trigger the check function to conclude the simulation
if err != nil {
b.Fatalf("expected no error. got %v", err)
}
}

274
swarm/network/stream/messages.go Normal file
View file

@ -0,0 +1,274 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package stream
import (
"fmt"
"sync"
"time"
"github.com/ethereum/go-ethereum/log"
bv "github.com/ethereum/go-ethereum/swarm/network/bitvector"
"github.com/ethereum/go-ethereum/swarm/storage"
)
// SubcribeMsg is the protocol msg for requesting a stream(section)
type SubscribeMsg struct {
Stream string
Key []byte
From, To uint64
Priority uint8 // delivered on priority channel
}
func (p *Peer) handleSubscribeMsg(req *SubscribeMsg) (err error) {
defer func() {
if err != nil {
if e := p.Send(SubscribeErrorMsg{
Error: err.Error(),
}); e != nil {
log.Error("send stream subscribe error message", "err", err)
}
}
}()
f, err := p.streamer.GetServerFunc(req.Stream)
if err != nil {
return err
}
s, err := f(p, req.Key)
if err != nil {
return err
}
os, err := p.setServer(req.Stream, req.Key, s, req.Priority)
if err != nil {
return err
}
log.Debug("received subscription", "peer", p.ID(), "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To)
go func() {
if err := p.SendOfferedHashes(os, req.From, req.To); err != nil {
p.Drop(err)
}
}()
return nil
}
type SubscribeErrorMsg struct {
Error string
}
func (p *Peer) handleSubscribeErrorMsg(req *SubscribeErrorMsg) (err error) {
return fmt.Errorf("subscribe to peer %s: %v", p.ID(), req.Error)
}
type UnsubscribeMsg struct {
Stream string
Key []byte
}
func (p *Peer) handleUnsubscribeMsg(req *UnsubscribeMsg) error {
p.removeServer(req.Stream, req.Key)
return nil
}
// OfferedHashesMsg is the protocol msg for offering to hand over a
// stream section
type OfferedHashesMsg struct {
Stream string // name of Stream
Key []byte // subtype or key
From, To uint64 // peer and db-specific entry count
Hashes []byte // stream of hashes (128)
*HandoverProof // HandoverProof
}
// String pretty prints OfferedHashesMsg
func (m OfferedHashesMsg) String() string {
return fmt.Sprintf("Stream '%v' [%v-%v] (%v)", m.Stream, m.From, m.To, len(m.Hashes)/HashSize)
}
// handleOfferedHashesMsg protocol msg handler calls the incoming streamer interface
// Filter method
func (p *Peer) handleOfferedHashesMsg(req *OfferedHashesMsg) error {
sk := req.Stream
sk += keyToString(req.Key)
s, err := p.getClient(sk)
if err != nil {
return err
}
hashes := req.Hashes
want, err := bv.New(len(hashes) / HashSize)
if err != nil {
return fmt.Errorf("error initiaising bitvector of length %v: %v", len(hashes)/HashSize, err)
}
wg := sync.WaitGroup{}
for i := 0; i < len(hashes); i += HashSize {
hash := hashes[i : i+HashSize]
if wait := s.NeedData(hash); wait != nil {
want.Set(i/HashSize, true)
wg.Add(1)
// create request and wait until the chunk data arrives and is stored
go func(w func()) {
w()
wg.Done()
}(wait)
}
}
// done := make(chan bool)
// go func() {
// wg.Wait()
// close(done)
// }()
// go func() {
// select {
// case <-done:
// s.next <- s.batchDone(p, req, hashes)
// case <-time.After(1 * time.Second):
// p.Drop(errors.New("timeout waiting for batch to be delivered"))
// }
// }()
go func() {
wg.Wait()
s.next <- s.batchDone(p, req, hashes)
}()
// only send wantedKeysMsg if all missing chunks of the previous batch arrived
// except
if s.live {
s.sessionAt = req.From
}
from, to := s.nextBatch(req.To)
log.Trace("received offered batch", "peer", p.ID(), "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To)
if from == to {
return nil
}
msg := &WantedHashesMsg{
Stream: req.Stream,
Key: req.Key,
Want: want.Bytes(),
From: from,
To: to,
}
go func() {
select {
case <-time.After(30 * time.Second):
p.Drop(err)
return
case err := <-s.next:
if err != nil {
p.Drop(err)
return
}
}
log.Trace("sending want batch", "peer", p.ID(), "stream", msg.Stream, "Key", msg.Key, "from", msg.From, "to", msg.To)
err := p.SendPriority(msg, s.priority)
if err != nil {
p.Drop(err)
}
}()
return nil
}
// WantedHashesMsg is the protocol msg data for signaling which hashes
// offered in OfferedHashesMsg downstream peer actually wants sent over
type WantedHashesMsg struct {
Stream string // name of stream
Key []byte // subtype or key
Want []byte // bitvector indicating which keys of the batch needed
From, To uint64 // next interval offset - empty if not to be continued
}
// String pretty prints WantedHashesMsg
func (m WantedHashesMsg) String() string {
return fmt.Sprintf("Stream '%v', Want: %x, Next: [%v-%v]", m.Stream, m.Want, m.From, m.To)
}
// handleWantedHashesMsg protocol msg handler
// * sends the next batch of unsynced keys
// * sends the actual data chunks as per WantedHashesMsg
func (p *Peer) handleWantedHashesMsg(req *WantedHashesMsg) error {
log.Trace("received wanted batch", "peer", p.ID(), "stream", req.Stream, "Key", req.Key, "from", req.From, "to", req.To)
s, err := p.getServer(req.Stream + keyToString(req.Key))
if err != nil {
return err
}
hashes := s.currentBatch
// launch in go routine since GetBatch blocks until new hashes arrive
go func() {
if err := p.SendOfferedHashes(s, req.From, req.To); err != nil {
p.Drop(err)
}
}()
// go p.SendOfferedHashes(s, req.From, req.To)
l := len(hashes) / HashSize
want, err := bv.NewFromBytes(req.Want, l)
if err != nil {
return fmt.Errorf("error initiaising bitvector of length %v: %v", l, err)
}
for i := 0; i < l; i++ {
if want.Get(i) {
hash := hashes[i*HashSize : (i+1)*HashSize]
data, err := s.GetData(hash)
if err != nil {
return fmt.Errorf("handleWantedHashesMsg get data %x: %v", hash, err)
}
chunk := storage.NewChunk(hash, nil)
chunk.SData = data
if err := p.Deliver(chunk, s.priority); err != nil {
return err
}
}
}
return nil
}
// Handover represents a statement that the upstream peer hands over the stream section
type Handover struct {
Stream string // name of stream
Start, End uint64 // index of hashes
Root []byte // Root hash for indexed segment inclusion proofs
}
// HandoverProof represents a signed statement that the upstream peer handed over the stream section
type HandoverProof struct {
Sig []byte // Sign(Hash(Serialisation(Handover)))
*Handover
}
// Takeover represents a statement that downstream peer took over (stored all data)
// handed over
type Takeover Handover
// TakeoverProof represents a signed statement that the downstream peer took over
// the stream section
type TakeoverProof struct {
Sig []byte // Sign(Hash(Serialisation(Takeover)))
*Takeover
}
// TakeoverProofMsg is the protocol msg sent by downstream peer
type TakeoverProofMsg TakeoverProof
// String pretty prints TakeoverProofMsg
func (m TakeoverProofMsg) String() string {
return fmt.Sprintf("Stream: '%v' [%v-%v], Root: %x, Sig: %x", m.Stream, m.Start, m.End, m.Root, m.Sig)
}
func (p *Peer) handleTakeoverProofMsg(req *TakeoverProofMsg) error {
_, err := p.getServer(req.Stream)
// store the strongest takeoverproof for the stream in streamer
return err
}

212
swarm/network/stream/peer.go Normal file
View file

@ -0,0 +1,212 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package stream
import (
"context"
"errors"
"fmt"
"sync"
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/p2p/protocols"
pq "github.com/ethereum/go-ethereum/swarm/network/priorityqueue"
"github.com/ethereum/go-ethereum/swarm/storage"
)
var sendTimeout = 5 * time.Second
var (
errServerNotFound = errors.New("server not found")
errClientNotFound = errors.New("client not found")
)
// Peer is the Peer extension for the streaming protocol
type Peer struct {
*protocols.Peer
streamer *Registry
pq *pq.PriorityQueue
serverMu sync.RWMutex
clientMu sync.RWMutex
servers map[string]*server
clients map[string]*client
quit chan struct{}
}
// NewPeer is the constructor for Peer
func NewPeer(peer *protocols.Peer, streamer *Registry) *Peer {
p := &Peer{
Peer: peer,
pq: pq.New(int(PriorityQueue), PriorityQueueCap),
streamer: streamer,
servers: make(map[string]*server),
clients: make(map[string]*client),
quit: make(chan struct{}),
}
ctx, cancel := context.WithCancel(context.Background())
go p.pq.Run(ctx, func(i interface{}) { p.Send(i) })
go func() {
<-p.quit
cancel()
}()
return p
}
// Deliver sends a storeRequestMsg protocol message to the peer
func (p *Peer) Deliver(chunk *storage.Chunk, priority uint8) error {
msg := &ChunkDeliveryMsg{
Key: chunk.Key,
SData: chunk.SData,
}
return p.SendPriority(msg, priority)
}
// SendPriority sends message to the peer using the outgoing priority queue
func (p *Peer) SendPriority(msg interface{}, priority uint8) error {
ctx, cancel := context.WithTimeout(context.Background(), sendTimeout)
defer cancel()
return p.pq.Push(ctx, msg, int(priority))
}
// SendOfferedHashes sends OfferedHashesMsg protocol msg
func (p *Peer) SendOfferedHashes(s *server, f, t uint64) error {
hashes, from, to, proof, err := s.SetNextBatch(f, t)
if err != nil {
return err
}
// true only when quiting
if len(hashes) == 0 {
return nil
}
if proof == nil {
proof = &HandoverProof{
Handover: &Handover{},
}
}
s.currentBatch = hashes
msg := &OfferedHashesMsg{
HandoverProof: proof,
Hashes: hashes,
From: from,
To: to,
Stream: s.stream,
Key: s.key,
}
log.Trace("Swarm syncer offer batch", "peer", p.ID(), "stream", s.stream, "key", s.key, "len", len(hashes), "from", from, "to", to)
return p.SendPriority(msg, s.priority)
}
func (p *Peer) getServer(s string) (*server, error) {
p.serverMu.RLock()
defer p.serverMu.RUnlock()
server := p.servers[s]
if server == nil {
return nil, fmt.Errorf("server '%v' not provided to peer %v", s, p.ID())
}
return server, nil
}
func (p *Peer) getClient(s string) (*client, error) {
p.clientMu.RLock()
defer p.clientMu.RUnlock()
client := p.clients[s]
if client == nil {
return nil, fmt.Errorf("client '%v' not provided to peer %v", s, p.ID())
}
return client, nil
}
func (p *Peer) setServer(s string, key []byte, o Server, priority uint8) (*server, error) {
p.serverMu.Lock()
defer p.serverMu.Unlock()
sk := s + keyToString(key)
if p.servers[sk] != nil {
return nil, fmt.Errorf("server %v already registered", sk)
}
os := &server{
Server: o,
priority: priority,
stream: s,
key: key,
}
p.servers[sk] = os
return os, nil
}
func (p *Peer) removeServer(s string, key []byte) error {
p.serverMu.Lock()
defer p.serverMu.Unlock()
sk := s + keyToString(key)
server, ok := p.servers[sk]
if !ok {
return errServerNotFound
}
server.Close()
delete(p.servers, sk)
return nil
}
func (p *Peer) setClient(s string, key []byte, i Client, priority uint8, live bool) error {
p.clientMu.Lock()
defer p.clientMu.Unlock()
sk := s + keyToString(key)
if p.clients[sk] != nil {
return fmt.Errorf("client %v already registered", sk)
}
next := make(chan error, 1)
// var intervals *Intervals
// if !live {
// key := s + p.ID().String()
// intervals = NewIntervals(key, p.streamer)
// }
p.clients[sk] = &client{
Client: i,
// intervals: intervals,
live: live,
priority: priority,
next: next,
stream: s,
key: key,
}
next <- nil // this is to allow wantedKeysMsg before first batch arrives
return nil
}
func (p *Peer) removeClient(s string, key []byte) error {
p.clientMu.Lock()
defer p.clientMu.Unlock()
sk := s + keyToString(key)
client, ok := p.clients[sk]
if !ok {
return errClientNotFound
}
client.close()
return nil
}
func (p *Peer) close() {
for _, s := range p.servers {
s.Close()
}
}

441
swarm/network/stream/stream.go Normal file
View file

@ -0,0 +1,441 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package stream
import (
"fmt"
"io"
"math"
"sync"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/p2p/discover"
"github.com/ethereum/go-ethereum/p2p/protocols"
"github.com/ethereum/go-ethereum/swarm/network"
"github.com/ethereum/go-ethereum/swarm/storage"
)
const (
Low uint8 = iota
Mid
High
Top
PriorityQueue // number of queues
PriorityQueueCap = 32 // queue capacity
HashSize = 32
)
// Registry registry for outgoing and incoming streamer constructors
type Registry struct {
api *API
addr *network.BzzAddr
skipCheck bool
clientMu sync.RWMutex
serverMu sync.RWMutex
peersMu sync.RWMutex
serverFuncs map[string]func(*Peer, []byte) (Server, error)
clientFuncs map[string]func(*Peer, []byte) (Client, error)
peers map[discover.NodeID]*Peer
delivery *Delivery
store storage.ChunkStore
}
// NewRegistry is Streamer constructor
func NewRegistry(addr *network.BzzAddr, delivery *Delivery, store storage.ChunkStore, skipCheck bool) *Registry {
streamer := &Registry{
addr: addr,
skipCheck: skipCheck,
store: store,
serverFuncs: make(map[string]func(*Peer, []byte) (Server, error)),
clientFuncs: make(map[string]func(*Peer, []byte) (Client, error)),
peers: make(map[discover.NodeID]*Peer),
delivery: delivery,
}
streamer.api = NewAPI(streamer, streamer.store)
delivery.getPeer = streamer.getPeer
streamer.RegisterServerFunc(swarmChunkServerStreamName, func(_ *Peer, t []byte) (Server, error) {
return NewSwarmChunkServer(delivery.db), nil
})
streamer.RegisterClientFunc(swarmChunkServerStreamName, func(p *Peer, t []byte) (Client, error) {
return NewSwarmSyncerClient(p, delivery.db, nil)
})
return streamer
}
// RegisterClient registers an incoming streamer constructor
func (r *Registry) RegisterClientFunc(stream string, f func(*Peer, []byte) (Client, error)) {
r.clientMu.Lock()
defer r.clientMu.Unlock()
r.clientFuncs[stream] = f
}
// RegisterServer registers an outgoing streamer constructor
func (r *Registry) RegisterServerFunc(stream string, f func(*Peer, []byte) (Server, error)) {
r.serverMu.Lock()
defer r.serverMu.Unlock()
r.serverFuncs[stream] = f
}
// GetClient accessor for incoming streamer constructors
func (r *Registry) GetClientFunc(stream string) (func(*Peer, []byte) (Client, error), error) {
r.clientMu.RLock()
defer r.clientMu.RUnlock()
f := r.clientFuncs[stream]
if f == nil {
return nil, fmt.Errorf("stream %v not registered", stream)
}
return f, nil
}
// GetServer accessor for incoming streamer constructors
func (r *Registry) GetServerFunc(stream string) (func(*Peer, []byte) (Server, error), error) {
r.serverMu.RLock()
defer r.serverMu.RUnlock()
f := r.serverFuncs[stream]
if f == nil {
return nil, fmt.Errorf("stream %v not registered", stream)
}
return f, nil
}
// Subscribe initiates the streamer
func (r *Registry) Subscribe(peerId discover.NodeID, s string, t []byte, from, to uint64, priority uint8, live bool) error {
f, err := r.GetClientFunc(s)
if err != nil {
return err
}
peer := r.getPeer(peerId)
if peer == nil {
return fmt.Errorf("peer not found %v", peerId)
}
is, err := f(peer, t)
if err != nil {
return err
}
err = peer.setClient(s, t, is, priority, live)
if err != nil {
return err
}
msg := &SubscribeMsg{
Stream: s,
Key: t,
// Live: live,
From: from,
To: to,
Priority: priority,
}
log.Debug("Subscribe ", "peer", peerId, "stream", s, "key", t, "from", from, "to", to)
return peer.SendPriority(msg, priority)
}
func (r *Registry) Unsubscribe(peerId discover.NodeID, s string, t []byte) error {
peer := r.getPeer(peerId)
if peer == nil {
return fmt.Errorf("peer not found %v", peerId)
}
msg := &UnsubscribeMsg{
Stream: s,
Key: t,
}
log.Debug("Unsubscribe ", "peer", peerId, "stream", s, "key", t)
if err := peer.Send(msg); err != nil {
return err
}
return peer.removeClient(s, t)
}
func (r *Registry) Retrieve(chunk *storage.Chunk) error {
return r.delivery.RequestFromPeers(chunk.Key[:], r.skipCheck)
}
func (r *Registry) NodeInfo() interface{} {
return nil
}
func (r *Registry) PeerInfo(id discover.NodeID) interface{} {
return nil
}
func (r *Registry) getPeer(peerId discover.NodeID) *Peer {
r.peersMu.RLock()
defer r.peersMu.RUnlock()
return r.peers[peerId]
}
func (r *Registry) setPeer(peer *Peer) {
r.peersMu.Lock()
r.peers[peer.ID()] = peer
r.peersMu.Unlock()
}
func (r *Registry) deletePeer(peer *Peer) {
r.peersMu.Lock()
delete(r.peers, peer.ID())
r.peersMu.Unlock()
}
func (r *Registry) peersCount() (c int) {
r.peersMu.Lock()
c = len(r.peers)
r.peersMu.Unlock()
return
}
// Run protocol run function
func (r *Registry) run(p *protocols.Peer) error {
sp := NewPeer(p, r)
r.setPeer(sp)
defer r.deletePeer(sp)
defer close(sp.quit)
defer sp.close()
return sp.Run(sp.HandleMsg)
}
func (r *Registry) runProtocol(p *p2p.Peer, rw p2p.MsgReadWriter) error {
peer := protocols.NewPeer(p, rw, Spec)
bzzPeer := network.NewBzzTestPeer(peer, r.addr)
r.delivery.overlay.On(bzzPeer)
defer r.delivery.overlay.Off(bzzPeer)
return r.run(peer)
}
// HandleMsg is the message handler that delegates incoming messages
func (p *Peer) HandleMsg(msg interface{}) error {
switch msg := msg.(type) {
case *SubscribeMsg:
return p.handleSubscribeMsg(msg)
case *SubscribeErrorMsg:
return p.handleSubscribeErrorMsg(msg)
case *UnsubscribeMsg:
return p.handleUnsubscribeMsg(msg)
case *OfferedHashesMsg:
return p.handleOfferedHashesMsg(msg)
case *TakeoverProofMsg:
return p.handleTakeoverProofMsg(msg)
case *WantedHashesMsg:
return p.handleWantedHashesMsg(msg)
case *ChunkDeliveryMsg:
return p.streamer.delivery.handleChunkDeliveryMsg(p, msg)
case *RetrieveRequestMsg:
return p.streamer.delivery.handleRetrieveRequestMsg(p, msg)
default:
return fmt.Errorf("unknown message type: %T", msg)
}
}
func keyToString(key []byte) string {
l := len(key)
if l == 0 {
return ""
}
return fmt.Sprintf("%s-%d", string(key[:l-1]), key[l-1])
}
type server struct {
Server
priority uint8
currentBatch []byte
stream string
key []byte
}
// Server interface for outgoing peer Streamer
type Server interface {
SetNextBatch(uint64, uint64) (hashes []byte, from uint64, to uint64, proof *HandoverProof, err error)
GetData([]byte) ([]byte, error)
Close()
}
type client struct {
Client
priority uint8
sessionAt uint64
live bool
stream string
key []byte
next chan error
}
// Client interface for incoming peer Streamer
type Client interface {
NeedData([]byte) func()
BatchDone(string, uint64, []byte, []byte) func() (*TakeoverProof, error)
Close()
}
// nextBatch adjusts the indexes by inspecting the intervals
func (c *client) nextBatch(from uint64) (nextFrom uint64, nextTo uint64) {
var intervals []uint64
if c.live {
if len(intervals) == 0 {
intervals = []uint64{c.sessionAt, from}
} else {
intervals[1] = from
}
nextFrom = from
} else if from >= c.sessionAt { // history sync complete
intervals = nil
nextFrom = from
nextTo = math.MaxUint64
} else if len(intervals) > 2 && from >= intervals[2] { // filled a gap in the intervals
intervals = append(intervals[:1], intervals[3:]...)
nextFrom = intervals[1]
if len(intervals) > 2 {
nextTo = intervals[2]
} else {
nextTo = c.sessionAt
}
} else {
nextFrom = from
intervals[1] = from
nextTo = c.sessionAt
}
// b.intervals.set(intervals)
return nextFrom, nextTo
}
func (c *client) batchDone(p *Peer, req *OfferedHashesMsg, hashes []byte) error {
if tf := c.BatchDone(req.Stream, req.From, hashes, req.Root); tf != nil {
tp, err := tf()
if err != nil {
return err
}
return p.SendPriority(tp, c.priority)
}
return nil
}
func (c *client) close() {
close(c.next)
c.Close()
}
// Spec is the spec of the streamer protocol
var Spec = &protocols.Spec{
Name: "stream",
Version: 1,
MaxMsgSize: 10 * 1024 * 1024,
Messages: []interface{}{
UnsubscribeMsg{},
OfferedHashesMsg{},
WantedHashesMsg{},
TakeoverProofMsg{},
SubscribeMsg{},
RetrieveRequestMsg{},
ChunkDeliveryMsg{},
SubscribeErrorMsg{},
},
}
func (r *Registry) Protocols() []p2p.Protocol {
return []p2p.Protocol{
{
Name: Spec.Name,
Version: Spec.Version,
Length: Spec.Length(),
Run: r.runProtocol,
// NodeInfo: ,
// PeerInfo: ,
},
}
}
func (r *Registry) APIs() []rpc.API {
return []rpc.API{
{
Namespace: "stream",
Version: "0.1",
Service: r.api,
Public: true,
},
}
}
func (r *Registry) Start(server *p2p.Server) error {
r.api.dpa.Start()
return nil
}
func (r *Registry) Stop() error {
r.api.dpa.Stop()
return nil
}
type API struct {
streamer *Registry
dpa *storage.DPA
}
func NewAPI(r *Registry, store storage.ChunkStore) *API {
dpa := storage.NewDPA(store, storage.NewChunkerParams())
return &API{
streamer: r,
dpa: dpa,
}
}
func readAll(dpa *storage.DPA, hash []byte) (int64, error) {
r := dpa.Retrieve(hash)
buf := make([]byte, 1024)
var n int
var total int64
var err error
for (total == 0 || n > 0) && err == nil {
n, err = r.ReadAt(buf, total)
total += int64(n)
}
if err != nil && err != io.EOF {
return total, err
}
return total, nil
}
func (api *API) ReadAll(hash common.Hash) (int64, error) {
return readAll(api.dpa, hash[:])
}
func (api *API) SubscribeStream(peerId discover.NodeID, s string, t []byte, from, to uint64, priority uint8, live bool) error {
return api.streamer.Subscribe(peerId, s, t, from, to, priority, live)
}
func (api *API) UnsubscribeStream(peerId discover.NodeID, s string, t []byte) error {
return api.streamer.Unsubscribe(peerId, s, t)
}

371
swarm/network/stream/streamer_test.go Normal file
View file

@ -0,0 +1,371 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package stream
import (
"bytes"
"testing"
"time"
"github.com/ethereum/go-ethereum/crypto/sha3"
p2ptest "github.com/ethereum/go-ethereum/p2p/testing"
)
func TestStreamerSubscribe(t *testing.T) {
tester, streamer, _, teardown, err := newStreamerTester(t)
defer teardown()
if err != nil {
t.Fatal(err)
}
err = streamer.Subscribe(tester.IDs[0], "foo", nil, 0, 0, Top, true)
if err == nil || err.Error() != "stream foo not registered" {
t.Fatalf("Expected error %v, got %v", "stream foo not registered", err)
}
}
var (
hash0 = sha3.Sum256([]byte{0})
hash1 = sha3.Sum256([]byte{1})
hash2 = sha3.Sum256([]byte{2})
hashesTmp = append(hash0[:], hash1[:]...)
hashes = append(hashesTmp, hash2[:]...)
receivedHashes map[string][]byte = make(map[string][]byte)
wait0 = make(chan bool)
wait2 = make(chan bool)
batchDone = make(chan bool)
)
type testClient struct {
t []byte
}
type testServer struct {
t []byte
}
func (self *testClient) NeedData(hash []byte) func() {
receivedHashes[string(hash)] = hash
if bytes.Equal(hash, hash0[:]) {
return func() {
<-wait0
}
} else if bytes.Equal(hash, hash2[:]) {
return func() {
<-wait2
}
}
return nil
}
func (self *testClient) BatchDone(string, uint64, []byte, []byte) func() (*TakeoverProof, error) {
close(batchDone)
return nil
}
func (self *testClient) Close() {}
func (self *testServer) SetNextBatch(from uint64, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) {
return make([]byte, HashSize), from + 1, to + 1, nil, nil
}
func (self *testServer) GetData([]byte) ([]byte, error) {
return nil, nil
}
func (self *testServer) Close() {
}
func TestStreamerDownstreamSubscribeUnsubscribeMsgExchange(t *testing.T) {
tester, streamer, _, teardown, err := newStreamerTester(t)
defer teardown()
if err != nil {
t.Fatal(err)
}
streamer.RegisterClientFunc("foo", func(p *Peer, t []byte) (Client, error) {
return &testClient{
t: t,
}, nil
})
peerID := tester.IDs[0]
err = streamer.Subscribe(peerID, "foo", []byte{}, 5, 8, Top, true)
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
err = tester.TestExchanges(p2ptest.Exchange{
Label: "Subscribe message",
Expects: []p2ptest.Expect{
{
Code: 4,
Msg: &SubscribeMsg{
Stream: "foo",
Key: []byte{},
From: 5,
To: 8,
Priority: Top,
},
Peer: peerID,
},
},
})
if err != nil {
t.Fatal(err)
}
err = streamer.Unsubscribe(peerID, "foo", []byte{})
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
err = tester.TestExchanges(p2ptest.Exchange{
Label: "Unsubscribe message",
Expects: []p2ptest.Expect{
{
Code: 0,
Msg: &UnsubscribeMsg{
Stream: "foo",
Key: []byte{},
},
Peer: peerID,
},
},
})
if err != nil {
t.Fatal(err)
}
}
func TestStreamerUpstreamSubscribeUnsubscribeMsgExchange(t *testing.T) {
tester, streamer, _, teardown, err := newStreamerTester(t)
defer teardown()
if err != nil {
t.Fatal(err)
}
streamer.RegisterServerFunc("foo", func(p *Peer, t []byte) (Server, error) {
return &testServer{
t: t,
}, nil
})
peerID := tester.IDs[0]
err = tester.TestExchanges(p2ptest.Exchange{
Label: "Subscribe message",
Triggers: []p2ptest.Trigger{
{
Code: 4,
Msg: &SubscribeMsg{
Stream: "foo",
Key: []byte{},
From: 5,
To: 8,
Priority: Top,
},
Peer: peerID,
},
},
Expects: []p2ptest.Expect{
{
Code: 1,
Msg: &OfferedHashesMsg{
Stream: "foo",
Key: []byte{},
HandoverProof: &HandoverProof{
Handover: &Handover{},
},
Hashes: make([]byte, HashSize),
From: 6,
To: 9,
},
Peer: peerID,
},
},
})
if err != nil {
t.Fatal(err)
}
err = tester.TestExchanges(p2ptest.Exchange{
Label: "unsubscribe message",
Triggers: []p2ptest.Trigger{
{
Code: 0,
Msg: &UnsubscribeMsg{
Stream: "foo",
Key: []byte{},
},
Peer: peerID,
},
},
})
if err != nil {
t.Fatal(err)
}
}
func TestStreamerUpstreamSubscribeErrorMsgExchange(t *testing.T) {
tester, streamer, _, teardown, err := newStreamerTester(t)
defer teardown()
if err != nil {
t.Fatal(err)
}
streamer.RegisterServerFunc("foo", func(p *Peer, t []byte) (Server, error) {
return &testServer{
t: t,
}, nil
})
peerID := tester.IDs[0]
err = tester.TestExchanges(p2ptest.Exchange{
Label: "Subscribe message",
Triggers: []p2ptest.Trigger{
{
Code: 4,
Msg: &SubscribeMsg{
Stream: "bar",
Key: []byte{},
From: 5,
To: 8,
Priority: Top,
},
Peer: peerID,
},
},
Expects: []p2ptest.Expect{
{
Code: 7,
Msg: &SubscribeErrorMsg{
Error: "stream bar not registered",
},
Peer: peerID,
},
},
})
if err != nil {
t.Fatal(err)
}
}
func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) {
tester, streamer, _, teardown, err := newStreamerTester(t)
defer teardown()
if err != nil {
t.Fatal(err)
}
streamer.RegisterClientFunc("foo", func(p *Peer, t []byte) (Client, error) {
return &testClient{
t: t,
}, nil
})
peerID := tester.IDs[0]
err = streamer.Subscribe(peerID, "foo", []byte{}, 5, 8, Top, true)
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
err = tester.TestExchanges(p2ptest.Exchange{
Label: "Subscribe message",
Expects: []p2ptest.Expect{
{
Code: 4,
Msg: &SubscribeMsg{
Stream: "foo",
Key: []byte{},
From: 5,
To: 8,
Priority: Top,
},
Peer: peerID,
},
},
},
p2ptest.Exchange{
Label: "WantedHashes message",
Triggers: []p2ptest.Trigger{
{
Code: 1,
Msg: &OfferedHashesMsg{
HandoverProof: &HandoverProof{
Handover: &Handover{},
},
Hashes: hashes,
From: 5,
To: 8,
Stream: "foo",
},
Peer: peerID,
},
},
Expects: []p2ptest.Expect{
{
Code: 2,
Msg: &WantedHashesMsg{
Stream: "foo",
Want: []byte{5},
From: 8,
To: 0,
},
Peer: peerID,
},
},
})
if err != nil {
t.Fatal(err)
}
if len(receivedHashes) != 3 {
t.Fatalf("Expected number of received hashes %v, got %v", 3, len(receivedHashes))
}
close(wait0)
timeout := time.NewTimer(100 * time.Millisecond)
defer timeout.Stop()
select {
case <-batchDone:
t.Fatal("batch done early")
case <-timeout.C:
}
close(wait2)
timeout2 := time.NewTimer(10000 * time.Millisecond)
defer timeout2.Stop()
select {
case <-batchDone:
case <-timeout2.C:
t.Fatal("timeout waiting batchdone call")
}
}

257
swarm/network/stream/syncer.go Normal file
View file

@ -0,0 +1,257 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package stream
import (
"bytes"
"errors"
"fmt"
"io"
"math"
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/storage"
)
const (
// BatchSize = 2
BatchSize = 128
)
// SwarmSyncerServer implements an Server for history syncing on bins
// offered streams:
// * live request delivery with or without checkback
// * (live/non-live historical) chunk syncing per proximity bin
type SwarmSyncerServer struct {
po uint8
db *storage.DBAPI
sessionAt uint64
start uint64
quit chan struct{}
}
// NewSwarmSyncerServer is contructor for SwarmSyncerServer
func NewSwarmSyncerServer(live bool, po uint8, db *storage.DBAPI) (*SwarmSyncerServer, error) {
sessionAt := db.CurrentBucketStorageIndex(po)
var start uint64
if live {
start = sessionAt
}
return &SwarmSyncerServer{
po: po,
db: db,
sessionAt: sessionAt,
start: start,
quit: make(chan struct{}),
}, nil
}
const maxPO = 32
func RegisterSwarmSyncerServer(streamer *Registry, db *storage.DBAPI) {
streamer.RegisterServerFunc("SYNC", func(p *Peer, t []byte) (Server, error) {
po := t[0]
// TODO: make this work for HISTORY too
return NewSwarmSyncerServer(false, po, db)
})
// streamer.RegisterServerFunc(stream, func(p *Peer) (Server, error) {
// return NewOutgoingProvableSwarmSyncer(po, db)
// })
}
// Close needs to be called on a stream server
func (s *SwarmSyncerServer) Close() {
close(s.quit)
}
// GetSection retrieves the actual chunk from localstore
func (s *SwarmSyncerServer) GetData(key []byte) ([]byte, error) {
chunk, err := s.db.Get(storage.Key(key))
if err == storage.ErrFetching {
<-chunk.ReqC
} else if err != nil {
return nil, err
}
return chunk.SData, nil
}
// GetBatch retrieves the next batch of hashes from the dbstore
func (s *SwarmSyncerServer) SetNextBatch(from, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) {
var batch []byte
i := 0
if from == 0 {
from = s.start
}
if to <= from || from >= s.sessionAt {
to = math.MaxUint64
}
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-ticker.C:
case <-s.quit:
return nil, 0, 0, nil, nil
}
err := s.db.Iterator(from, to, s.po, func(key storage.Key, idx uint64) bool {
batch = append(batch, key[:]...)
i++
to = idx
return i < BatchSize
})
if err != nil {
return nil, 0, 0, nil, err
}
if len(batch) > 0 {
break
}
}
log.Debug("Swarm syncer offer batch", "po", s.po, "len", i, "from", from, "to", to, "current store count", s.db.CurrentBucketStorageIndex(s.po))
return batch, from, to, nil, nil
}
// SwarmSyncerClient
type SwarmSyncerClient struct {
sessionAt uint64
nextC chan struct{}
sessionRoot storage.Key
sessionReader storage.LazySectionReader
retrieveC chan *storage.Chunk
storeC chan *storage.Chunk
db *storage.DBAPI
chunker storage.Chunker
currentRoot storage.Key
requestFunc func(chunk *storage.Chunk)
end, start uint64
}
// NewSwarmSyncerClient is a contructor for provable data exchange syncer
func NewSwarmSyncerClient(_ *Peer, db *storage.DBAPI, chunker storage.Chunker) (*SwarmSyncerClient, error) {
return &SwarmSyncerClient{
db: db,
chunker: chunker,
}, nil
}
// // NewIncomingProvableSwarmSyncer is a contructor for provable data exchange syncer
// func NewIncomingProvableSwarmSyncer(po int, priority int, index uint64, sessionAt uint64, intervals []uint64, sessionRoot storage.Key, chunker *storage.PyramidChunker, store storage.ChunkStore, p Peer) *SwarmSyncerClient {
// retrieveC := make(storage.Chunk, chunksCap)
// RunChunkRequestor(p, retrieveC)
// storeC := make(storage.Chunk, chunksCap)
// RunChunkStorer(store, storeC)
// s := &SwarmSyncerClient{
// po: po,
// priority: priority,
// sessionAt: sessionAt,
// start: index,
// end: index,
// nextC: make(chan struct{}, 1),
// intervals: intervals,
// sessionRoot: sessionRoot,
// sessionReader: chunker.Join(sessionRoot, retrieveC),
// retrieveC: retrieveC,
// storeC: storeC,
// }
// return s
// }
// // StartSyncing is called on the Peer to start the syncing process
// // the idea is that it is called only after kademlia is close to healthy
// func StartSyncing(s *Streamer, peerId discover.NodeID, po uint8, nn bool) {
// lastPO := po
// if nn {
// lastPO = maxPO
// }
//
// for i := po; i <= lastPO; i++ {
// s.Subscribe(peerId, "SYNC", newSyncLabel("LIVE", po), 0, 0, High, true)
// s.Subscribe(peerId, "SYNC", newSyncLabel("HISTORY", po), 0, 0, Mid, false)
// }
// }
// RegisterSwarmSyncerClient registers the client constructor function for
// to handle incoming sync streams
func RegisterSwarmSyncerClient(streamer *Registry, db *storage.DBAPI) {
streamer.RegisterClientFunc("SYNC", func(p *Peer, t []byte) (Client, error) {
return NewSwarmSyncerClient(p, db, nil)
})
}
// NeedData
func (s *SwarmSyncerClient) NeedData(key []byte) (wait func()) {
chunk, _ := s.db.GetOrCreateRequest(key)
// TODO: we may want to request from this peer anyway even if the request exists
if chunk.ReqC == nil {
return nil
}
// create request and wait until the chunk data arrives and is stored
return func() {
chunk.WaitToStore()
}
}
// BatchDone
func (s *SwarmSyncerClient) BatchDone(streamName string, from uint64, hashes []byte, root []byte) func() (*TakeoverProof, error) {
if s.chunker != nil {
return func() (*TakeoverProof, error) { return s.TakeoverProof(streamName, from, hashes, root) }
}
return nil
}
func (s *SwarmSyncerClient) TakeoverProof(streamName string, from uint64, hashes []byte, root storage.Key) (*TakeoverProof, error) {
// for provable syncer currentRoot is non-zero length
if s.chunker != nil {
if from > s.sessionAt { // for live syncing currentRoot is always updated
//expRoot, err := s.chunker.Append(s.currentRoot, bytes.NewReader(hashes), s.retrieveC, s.storeC)
expRoot, _, err := s.chunker.Append(s.currentRoot, bytes.NewReader(hashes), s.retrieveC)
if err != nil {
return nil, err
}
if !bytes.Equal(root, expRoot) {
return nil, fmt.Errorf("HandoverProof mismatch")
}
s.currentRoot = root
} else {
expHashes := make([]byte, len(hashes))
_, err := s.sessionReader.ReadAt(expHashes, int64(s.end*HashSize))
if err != nil && err != io.EOF {
return nil, err
}
if !bytes.Equal(expHashes, hashes) {
return nil, errors.New("invalid proof")
}
}
return nil, nil
}
s.end += uint64(len(hashes)) / HashSize
takeover := &Takeover{
Stream: streamName,
// Key: s.Key,
Start: s.start,
End: s.end,
Root: root,
}
// serialise and sign
return &TakeoverProof{
Takeover: takeover,
Sig: nil,
}, nil
}
func (s *SwarmSyncerClient) Close() {}

221
swarm/network/stream/syncer_test.go Normal file
View file

@ -0,0 +1,221 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package stream
import (
"context"
crand "crypto/rand"
"fmt"
"io"
"math"
"testing"
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/p2p/discover"
"github.com/ethereum/go-ethereum/p2p/simulations"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/swarm/network"
streamTesting "github.com/ethereum/go-ethereum/swarm/network/stream/testing"
"github.com/ethereum/go-ethereum/swarm/storage"
)
const dataChunkCount = 500
func TestSyncerSimulation(t *testing.T) {
testSyncBetweenNodes(t, 2, 1, dataChunkCount, true, 1)
testSyncBetweenNodes(t, 4, 1, dataChunkCount, true, 1)
testSyncBetweenNodes(t, 8, 1, dataChunkCount, true, 1)
testSyncBetweenNodes(t, 16, 1, dataChunkCount, true, 1)
}
func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck bool, po uint8) {
defaultSkipCheck = skipCheck
toAddr = func(id discover.NodeID) *network.BzzAddr {
addr := network.NewAddrFromNodeID(id)
addr.OAddr[0] = byte(0)
return addr
}
conf := &streamTesting.RunConfig{
Adapter: *adapter,
NodeCount: nodes,
ConnLevel: conns,
ToAddr: toAddr,
Services: services,
EnableMsgEvents: false,
}
// create context for simulation run
timeout := 30 * time.Second
ctx, cancel := context.WithTimeout(context.Background(), timeout)
// defer cancel should come before defer simulation teardown
defer cancel()
// create simulation network with the config
sim, teardown, err := streamTesting.NewSimulation(conf)
defer teardown()
if err != nil {
t.Fatal(err.Error())
}
// HACK: these are global variables in the test so that they are available for
// the service constructor function
// TODO: will this work with exec/docker adapter?
// localstore of nodes made available for action and check calls
stores = make(map[discover.NodeID]storage.ChunkStore)
nodeIndex := make(map[discover.NodeID]int)
for i, id := range sim.IDs {
nodeIndex[id] = i
stores[id] = sim.Stores[i]
}
deliveries = make(map[discover.NodeID]*Delivery)
// peerCount function gives the number of peer connections for a nodeID
// this is needed for the service run function to wait until
// each protocol instance runs and the streamer peers are available
peerCount = func(id discover.NodeID) int {
if sim.IDs[0] == id || sim.IDs[nodes-1] == id {
return 1
}
return 2
}
waitPeerErrC = make(chan error)
// here we distribute chunks of a random file into stores 1...nodes
rrdpa := storage.NewDPA(newRoundRobinStore(sim.Stores[1:]...), storage.NewChunkerParams())
rrdpa.Start()
size := chunkCount * chunkSize
_, wait, err := rrdpa.Store(io.LimitReader(crand.Reader, int64(size)), int64(size))
// need to wait cos we then immediately collect the relevant bin content
wait()
defer rrdpa.Stop()
if err != nil {
t.Fatal(err.Error())
}
// create DBAPI-s for all nodes
dbs := make([]*storage.DBAPI, nodes)
for i := 0; i < nodes; i++ {
dbs[i] = storage.NewDBAPI(sim.Stores[i].(*storage.LocalStore))
}
// collect hashes in po 1 bin for each node
hashes := make([][]storage.Key, nodes)
totalHashes := 0
hashCounts := make([]int, nodes)
for i := nodes - 1; i >= 0; i-- {
if i < nodes-1 {
hashCounts[i] = hashCounts[i+1]
}
dbs[i].Iterator(0, math.MaxUint64, po, func(key storage.Key, index uint64) bool {
hashes[i] = append(hashes[i], key)
totalHashes++
hashCounts[i]++
return true
})
}
// errc is error channel for simulation
errc := make(chan error, 1)
quitC := make(chan struct{})
defer close(quitC)
// action is subscribe
action := func(ctx context.Context) error {
// need to wait till an aynchronous process registers the peers in streamer.peers
// that is used by Subscribe
// the global peerCount function tells how many connections each node has
// TODO: this is to be reimplemented with peerEvent watcher without global var
i := 0
for err := range waitPeerErrC {
if err != nil {
return fmt.Errorf("error waiting for peers: %s", err)
}
i++
if i == nodes {
break
}
}
// each node Subscribes to each other's swarmChunkServerStreamName
for j := 0; j < nodes-1; j++ {
id := sim.IDs[j]
err := sim.CallClient(id, func(client *rpc.Client) error {
// report disconnect events to the error channel cos peers should not disconnect
err := streamTesting.WatchDisconnections(id, client, errc, quitC)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(ctx, 1*time.Second)
defer cancel()
// start syncing, i.e., subscribe to upstream peers po 1 bin
sid := sim.IDs[j+1]
return client.CallContext(ctx, nil, "stream_subscribeStream", sid, "SYNC", []byte{1}, 0, 0, Top, false)
})
if err != nil {
return err
}
}
return nil
}
// this makes sure check is not called before the previous call finishes
check := func(ctx context.Context, id discover.NodeID) (bool, error) {
select {
case err := <-errc:
return false, err
case <-ctx.Done():
return false, ctx.Err()
default:
}
i := nodeIndex[id]
var total, found int
for j := i; j < nodes; j++ {
total += len(hashes[j])
for _, key := range hashes[j] {
chunk, err := dbs[i].Get(key)
if err == storage.ErrFetching {
<-chunk.ReqC
} else if err != nil {
continue
}
// needed for leveldb not to be closed?
// chunk.WaitToStore()
found++
}
}
log.Debug("sync check", "node", id, "index", i, "bin", po, "found", found, "total", total)
return total == found, nil
}
conf.Step = &simulations.Step{
Action: action,
Trigger: streamTesting.Trigger(500*time.Millisecond, quitC, sim.IDs[0:nodes-1]...),
Expect: &simulations.Expectation{
Nodes: sim.IDs[0:1],
Check: check,
},
}
startedAt := time.Now()
result, err := sim.Run(ctx, conf)
finishedAt := time.Now()
if err != nil {
t.Fatalf("Setting up simulation failed: %v", err)
}
if result.Error != nil {
t.Fatalf("Simulation failed: %s", result.Error)
}
streamTesting.CheckResult(t, result, startedAt, finishedAt)
}

267
swarm/network/stream/testing/testing.go Normal file
View file

@ -0,0 +1,267 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package testing
import (
"context"
"errors"
"fmt"
"io/ioutil"
"math/rand"
"os"
"sync"
"testing"
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/discover"
"github.com/ethereum/go-ethereum/p2p/simulations"
"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/swarm/network"
"github.com/ethereum/go-ethereum/swarm/storage"
)
type Simulation struct {
Net *simulations.Network
Stores []storage.ChunkStore
Addrs []network.Addr
IDs []discover.NodeID
}
func SetStores(addrs ...network.Addr) ([]storage.ChunkStore, func(), error) {
var datadirs []string
stores := make([]storage.ChunkStore, len(addrs))
var err error
for i, addr := range addrs {
var datadir string
datadir, err = ioutil.TempDir("", "streamer")
if err != nil {
break
}
var store storage.ChunkStore
store, err = storage.NewTestLocalStoreForAddr(datadir, addr.Over())
if err != nil {
break
}
datadirs = append(datadirs, datadir)
stores[i] = store
}
teardown := func() {
for i, datadir := range datadirs {
stores[i].Close()
os.RemoveAll(datadir)
}
}
return stores, teardown, err
}
func NewAdapter(adapterType string, services adapters.Services) (adapter adapters.NodeAdapter, teardown func(), err error) {
teardown = func() {}
switch adapterType {
case "sim":
adapter = adapters.NewSimAdapter(services)
case "socket":
adapter = adapters.NewSocketAdapter(services)
case "exec":
baseDir, err0 := ioutil.TempDir("", "swarm-test")
if err0 != nil {
return nil, teardown, err0
}
teardown = func() { os.RemoveAll(baseDir) }
adapter = adapters.NewExecAdapter(baseDir)
case "docker":
adapter, err = adapters.NewDockerAdapter()
if err != nil {
return nil, teardown, err
}
default:
return nil, teardown, errors.New("adapter needs to be one of sim, socket, exec, docker")
}
return adapter, teardown, nil
}
func CheckResult(t *testing.T, result *simulations.StepResult, startedAt, finishedAt time.Time) {
t.Logf("Simulation passed in %s", result.FinishedAt.Sub(result.StartedAt))
if len(result.Passes) > 1 {
var min, max time.Duration
var sum int
for _, pass := range result.Passes {
duration := pass.Sub(result.StartedAt)
if sum == 0 || duration < min {
min = duration
}
if duration > max {
max = duration
}
sum += int(duration.Nanoseconds())
}
t.Logf("Min: %s, Max: %s, Average: %s", min, max, time.Duration(sum/len(result.Passes))*time.Nanosecond)
}
t.Logf("Setup: %s, Shutdown: %s", result.StartedAt.Sub(startedAt), finishedAt.Sub(result.FinishedAt))
}
type RunConfig struct {
Adapter string
Step *simulations.Step
NodeCount int
ConnLevel int
ToAddr func(discover.NodeID) *network.BzzAddr
Services adapters.Services
EnableMsgEvents bool
}
func NewSimulation(conf *RunConfig) (*Simulation, func(), error) {
// create network
nodes := conf.NodeCount
adapter, adapterTeardown, err := NewAdapter(conf.Adapter, conf.Services)
if err != nil {
return nil, adapterTeardown, err
}
net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{
ID: "0",
DefaultService: "streamer",
})
teardown := func() {
adapterTeardown()
net.Shutdown()
}
ids := make([]discover.NodeID, nodes)
addrs := make([]network.Addr, nodes)
// start nodes
for i := 0; i < nodes; i++ {
nodeconf := adapters.RandomNodeConfig()
nodeconf.EnableMsgEvents = conf.EnableMsgEvents
node, err := net.NewNodeWithConfig(nodeconf)
if err != nil {
return nil, teardown, fmt.Errorf("error creating node: %s", err)
}
ids[i] = node.ID()
addrs[i] = conf.ToAddr(ids[i])
}
// set nodes number of Stores available
stores, storeTeardown, err := SetStores(addrs...)
teardown = func() {
net.Shutdown()
adapterTeardown()
storeTeardown()
}
if err != nil {
return nil, teardown, err
}
s := &Simulation{
Net: net,
Stores: stores,
IDs: ids,
Addrs: addrs,
}
return s, teardown, nil
}
func (s *Simulation) Run(ctx context.Context, conf *RunConfig) (*simulations.StepResult, error) {
// bring up nodes, launch the servive
nodes := conf.NodeCount
conns := conf.ConnLevel
for i := 0; i < nodes; i++ {
if err := s.Net.Start(s.IDs[i]); err != nil {
return nil, fmt.Errorf("error starting node %s: %s", s.IDs[i].TerminalString(), err)
}
}
// run a simulation which connects the 10 nodes in a chain
wg := sync.WaitGroup{}
for i := range s.IDs {
// collect the overlay addresses, to
for j := 0; j < conns; j++ {
var k int
if j == 0 {
k = i - 1
} else {
k = rand.Intn(len(s.IDs))
}
if i > 0 {
wg.Add(1)
go func(i, k int) {
defer wg.Done()
s.Net.Connect(s.IDs[i], s.IDs[k])
}(i, k)
}
}
}
wg.Wait()
log.Info(fmt.Sprintf("simulation with %v nodes", len(s.Addrs)))
// create an only locally retrieving dpa for the pivot node to test
// if retriee requests have arrived
result := simulations.NewSimulation(s.Net).Run(ctx, conf.Step)
return result, nil
}
func WatchDisconnections(id discover.NodeID, client *rpc.Client, errc chan error, quitC chan struct{}) error {
events := make(chan *p2p.PeerEvent)
sub, err := client.Subscribe(context.Background(), "admin", events, "peerEvents")
if err != nil {
return fmt.Errorf("error getting peer events for node %v: %s", id, err)
}
go func() {
for {
select {
case <-quitC:
return
case e := <-events:
errc <- fmt.Errorf("peerEvent for node %v: %v", id, e)
case err := <-sub.Err():
if err != nil {
errc <- fmt.Errorf("error getting peer events for node %v: %v", id, err)
}
}
}
}()
return nil
}
func Trigger(d time.Duration, quitC chan struct{}, ids ...discover.NodeID) chan discover.NodeID {
trigger := make(chan discover.NodeID)
go func() {
defer close(trigger)
ticker := time.NewTicker(d)
defer ticker.Stop()
// we are only testing the pivot node (net.Nodes[0])
for range ticker.C {
for _, id := range ids {
select {
case trigger <- id:
case <-quitC:
return
}
}
}
}()
return trigger
}
func (sim *Simulation) CallClient(id discover.NodeID, f func(*rpc.Client) error) error {
node := sim.Net.GetNode(id)
if node == nil {
return fmt.Errorf("unknown node: %s", id)
}
client, err := node.Client()
if err != nil {
return fmt.Errorf("error getting node client: %s", err)
}
return f(client)
}

12
swarm/pss/client/client_test.go
View file

@ -180,9 +180,9 @@ func setupNetwork(numnodes int) (clients []*rpc.Client, err error) {
DefaultService: "bzz",
})
for i := 0; i < numnodes; i++ {
nodes[i], err = net.NewNodeWithConfig(&adapters.NodeConfig{
Services: []string{"bzz", "pss"},
})
nodeconf := adapters.RandomNodeConfig()
nodeconf.Services = []string{"bzz", "pss"}
nodes[i], err = net.NewNodeWithConfig(nodeconf)
if err != nil {
return nil, fmt.Errorf("error creating node 1: %v", err)
}
@ -232,11 +232,11 @@ func newServices() adapters.Services {
"pss": func(ctx *adapters.ServiceContext) (node.Service, error) {
cachedir, err := ioutil.TempDir("", "pss-cache")
if err != nil {
return nil, fmt.Errorf("create pss cache tmpdir failed: %v", err)
return nil, fmt.Errorf("create pss cache tmpdir failed: %s", err)
}
dpa, err := storage.NewLocalDPA(cachedir)
dpa, err := storage.NewLocalDPA(cachedir, make([]byte, 32))
if err != nil {
return nil, fmt.Errorf("local dpa creation failed: %v", err)
return nil, fmt.Errorf("local dpa creation failed: %s", err)
}
ctxlocal, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()

6
swarm/pss/pss.go
View file

@ -498,7 +498,7 @@ func (self *Pss) processSym(envelope *whisper.Envelope) (*whisper.ReceivedMessag
func (self *Pss) processAsym(envelope *whisper.Envelope) (*whisper.ReceivedMessage, string, *PssAddress, error) {
recvmsg, err := envelope.OpenAsymmetric(self.privateKey)
if err != nil {
return nil, "", nil, fmt.Errorf("could not decrypt message: %v", err)
return nil, "", nil, fmt.Errorf("could not decrypt message: %s", err)
}
// check signature (if signed), strip padding
if !recvmsg.Validate() {
@ -774,10 +774,8 @@ func (self *Pss) checkFwdCache(addr []byte, digest pssDigest) bool {
// DPA storage handler for message cache
func (self *Pss) storeMsg(msg *PssMsg) (pssDigest, error) {
swg := &sync.WaitGroup{}
wwg := &sync.WaitGroup{}
buf := bytes.NewReader(msg.serialize())
key, err := self.dpa.Store(buf, int64(buf.Len()), swg, wwg)
key, _, err := self.dpa.Store(buf, int64(buf.Len()))
if err != nil {
log.Warn("Could not store in swarm", "err", err)
return pssDigest{}, err

694
swarm/pss/pss_test.go
View file

@ -4,7 +4,9 @@ import (
"bytes"
"context"
"crypto/ecdsa"
"encoding/binary"
"encoding/hex"
"encoding/json"
"flag"
"fmt"
"io/ioutil"
@ -17,6 +19,7 @@ import (
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node"
@ -260,8 +263,8 @@ func TestKeys(t *testing.T) {
if err != nil {
t.Fatalf("create 'our' key fail")
}
ctx, cancel = context.WithTimeout(context.Background(), time.Second)
defer cancel()
ctx, cancel2 := context.WithTimeout(context.Background(), time.Second)
defer cancel2()
theirkeys, err := wapi.NewKeyPair(ctx)
if err != nil {
t.Fatalf("create 'their' key fail")
@ -392,6 +395,679 @@ func TestMismatch(t *testing.T) {
}
// send symmetrically encrypted message between two directly connected peers
func TestSymSend(t *testing.T) {
t.Run("32", testSymSend)
t.Run("8", testSymSend)
t.Run("0", testSymSend)
}
func testSymSend(t *testing.T) {
// address hint size
var addrsize int64
var err error
paramstring := strings.Split(t.Name(), "/")
addrsize, _ = strconv.ParseInt(paramstring[1], 10, 0)
log.Info("sym send test", "addrsize", addrsize)
clients, err := setupNetwork(2)
if err != nil {
t.Fatal(err)
}
var topic string
err = clients[0].Call(&topic, "pss_stringToTopic", "foo:42")
if err != nil {
t.Fatal(err)
}
var loaddrhex string
err = clients[0].Call(&loaddrhex, "pss_baseAddr")
if err != nil {
t.Fatalf("rpc get node 1 baseaddr fail: %v", err)
}
loaddrhex = loaddrhex[:2+(addrsize*2)]
var roaddrhex string
err = clients[1].Call(&roaddrhex, "pss_baseAddr")
if err != nil {
t.Fatalf("rpc get node 2 baseaddr fail: %v", err)
}
roaddrhex = roaddrhex[:2+(addrsize*2)]
// retrieve public key from pss instance
// set this public key reciprocally
var lpubkeyhex string
err = clients[0].Call(&lpubkeyhex, "pss_getPublicKey")
if err != nil {
t.Fatalf("rpc get node 1 pubkey fail: %v", err)
}
var rpubkeyhex string
err = clients[1].Call(&rpubkeyhex, "pss_getPublicKey")
if err != nil {
t.Fatalf("rpc get node 2 pubkey fail: %v", err)
}
time.Sleep(time.Millisecond * 500)
// at this point we've verified that symkeys are saved and match on each peer
// now try sending symmetrically encrypted message, both directions
lmsgC := make(chan APIMsg)
lctx, lcancel := context.WithTimeout(context.Background(), time.Second*10)
defer lcancel()
lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic)
log.Trace("lsub", "id", lsub)
defer lsub.Unsubscribe()
rmsgC := make(chan APIMsg)
rctx, rcancel := context.WithTimeout(context.Background(), time.Second*10)
defer rcancel()
rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic)
log.Trace("rsub", "id", rsub)
defer rsub.Unsubscribe()
lrecvkey := network.RandomAddr().Over()
rrecvkey := network.RandomAddr().Over()
var lkeyids [2]string
var rkeyids [2]string
// manually set reciprocal symkeys
err = clients[0].Call(&lkeyids, "psstest_setSymKeys", rpubkeyhex, lrecvkey, rrecvkey, defaultSymKeySendLimit, topic, roaddrhex)
if err != nil {
t.Fatal(err)
}
err = clients[1].Call(&rkeyids, "psstest_setSymKeys", rpubkeyhex, rrecvkey, lrecvkey, defaultSymKeySendLimit, topic, loaddrhex)
if err != nil {
t.Fatal(err)
}
// send and verify delivery
lmsg := []byte("plugh")
err = clients[1].Call(nil, "pss_sendSym", rkeyids[1], topic, hexutil.Encode(lmsg))
if err != nil {
t.Fatal(err)
}
select {
case recvmsg := <-lmsgC:
if !bytes.Equal(recvmsg.Msg, lmsg) {
t.Fatalf("node 1 received payload mismatch: expected %v, got %v", lmsg, recvmsg)
}
case cerr := <-lctx.Done():
t.Fatalf("test message timed out: %v", cerr)
}
rmsg := []byte("xyzzy")
err = clients[0].Call(nil, "pss_sendSym", lkeyids[1], topic, hexutil.Encode(rmsg))
if err != nil {
t.Fatal(err)
}
select {
case recvmsg := <-rmsgC:
if !bytes.Equal(recvmsg.Msg, rmsg) {
t.Fatalf("node 2 received payload mismatch: expected %v, got %v", rmsg, recvmsg.Msg)
}
case cerr := <-rctx.Done():
t.Fatalf("test message timed out: %v", cerr)
}
}
// send asymmetrically encrypted message between two directly connected peers
func TestAsymSend(t *testing.T) {
t.Run("32", testAsymSend)
t.Run("8", testAsymSend)
t.Run("0", testAsymSend)
}
func testAsymSend(t *testing.T) {
// address hint size
var addrsize int64
var err error
paramstring := strings.Split(t.Name(), "/")
addrsize, _ = strconv.ParseInt(paramstring[1], 10, 0)
log.Info("asym send test", "addrsize", addrsize)
clients, err := setupNetwork(2)
if err != nil {
t.Fatal(err)
}
var topic string
err = clients[0].Call(&topic, "pss_stringToTopic", "foo:42")
if err != nil {
t.Fatal(err)
}
time.Sleep(time.Millisecond * 250)
var loaddrhex string
err = clients[0].Call(&loaddrhex, "pss_baseAddr")
if err != nil {
t.Fatalf("rpc get node 1 baseaddr fail: %v", err)
}
loaddrhex = loaddrhex[:2+(addrsize*2)]
var roaddrhex string
err = clients[1].Call(&roaddrhex, "pss_baseAddr")
if err != nil {
t.Fatalf("rpc get node 2 baseaddr fail: %v", err)
}
roaddrhex = roaddrhex[:2+(addrsize*2)]
// retrieve public key from pss instance
// set this public key reciprocally
var lpubkey string
err = clients[0].Call(&lpubkey, "pss_getPublicKey")
if err != nil {
t.Fatalf("rpc get node 1 pubkey fail: %v", err)
}
var rpubkey string
err = clients[1].Call(&rpubkey, "pss_getPublicKey")
if err != nil {
t.Fatalf("rpc get node 2 pubkey fail: %v", err)
}
time.Sleep(time.Millisecond * 500) // replace with hive healthy code
lmsgC := make(chan APIMsg)
lctx, lcancel := context.WithTimeout(context.Background(), time.Second*10)
defer lcancel()
lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic)
log.Trace("lsub", "id", lsub)
defer lsub.Unsubscribe()
rmsgC := make(chan APIMsg)
rctx, rcancel := context.WithTimeout(context.Background(), time.Second*10)
defer rcancel()
rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic)
log.Trace("rsub", "id", rsub)
defer rsub.Unsubscribe()
// store reciprocal public keys
err = clients[0].Call(nil, "pss_setPeerPublicKey", rpubkey, topic, roaddrhex)
if err != nil {
t.Fatal(err)
}
err = clients[1].Call(nil, "pss_setPeerPublicKey", lpubkey, topic, loaddrhex)
if err != nil {
t.Fatal(err)
}
// send and verify delivery
rmsg := []byte("xyzzy")
err = clients[0].Call(nil, "pss_sendAsym", rpubkey, topic, hexutil.Encode(rmsg))
if err != nil {
t.Fatal(err)
}
select {
case recvmsg := <-rmsgC:
if !bytes.Equal(recvmsg.Msg, rmsg) {
t.Fatalf("node 2 received payload mismatch: expected %v, got %v", rmsg, recvmsg.Msg)
}
case cerr := <-rctx.Done():
t.Fatalf("test message timed out: %v", cerr)
}
lmsg := []byte("plugh")
err = clients[1].Call(nil, "pss_sendAsym", lpubkey, topic, hexutil.Encode(lmsg))
if err != nil {
t.Fatal(err)
}
select {
case recvmsg := <-lmsgC:
if !bytes.Equal(recvmsg.Msg, lmsg) {
t.Fatalf("node 1 received payload mismatch: expected %v, got %v", lmsg, recvmsg.Msg)
}
case cerr := <-lctx.Done():
t.Fatalf("test message timed out: %v", cerr)
}
}
type Job struct {
Msg []byte
SendNode discover.NodeID
RecvNode discover.NodeID
}
func worker(id int, jobs <-chan Job, rpcs map[discover.NodeID]*rpc.Client, pubkeys map[discover.NodeID]string, topic string) {
for j := range jobs {
rpcs[j.SendNode].Call(nil, "pss_sendAsym", pubkeys[j.RecvNode], topic, hexutil.Encode(j.Msg))
}
}
// params in run name:
// nodes/msgs/addrbytes/adaptertype
// if adaptertype is exec uses execadapter, simadapter otherwise
func XTestNetwork(t *testing.T) {
t.Run("3/2000/4/sock", testNetwork)
t.Run("4/2000/4/sock", testNetwork)
t.Run("8/2000/4/sock", testNetwork)
t.Run("16/2000/4/sock", testNetwork)
t.Run("32/2000/4/sock", testNetwork)
t.Run("64/2000/4/sim", testNetwork)
}
func testNetwork(t *testing.T) {
type msgnotifyC struct {
id discover.NodeID
msgIdx int
}
paramstring := strings.Split(t.Name(), "/")
nodecount, _ := strconv.ParseInt(paramstring[1], 10, 0)
msgcount, _ := strconv.ParseInt(paramstring[2], 10, 0)
addrsize, _ := strconv.ParseInt(paramstring[3], 10, 0)
adapter := paramstring[4]
log.Info("network test", "nodecount", nodecount, "msgcount", msgcount, "addrhintsize", addrsize)
nodes := make([]discover.NodeID, nodecount)
bzzaddrs := make(map[discover.NodeID]string, nodecount)
rpcs := make(map[discover.NodeID]*rpc.Client, nodecount)
pubkeys := make(map[discover.NodeID]string, nodecount)
sentmsgs := make([][]byte, msgcount)
recvmsgs := make([]bool, msgcount)
nodemsgcount := make(map[discover.NodeID]int, nodecount)
trigger := make(chan discover.NodeID)
var a adapters.NodeAdapter
if adapter == "exec" {
dirname, err := ioutil.TempDir(".", "")
if err != nil {
t.Fatal(err)
}
a = adapters.NewExecAdapter(dirname)
} else if adapter == "sock" {
a = adapters.NewSocketAdapter(services)
} else if adapter == "tcp" {
a = adapters.NewTCPAdapter(services)
} else if adapter == "sim" {
a = adapters.NewSimAdapter(services)
}
net := simulations.NewNetwork(a, &simulations.NetworkConfig{
ID: "0",
})
defer net.Shutdown()
f, err := os.Open(fmt.Sprintf("testdata/snapshot_%d.json", nodecount))
if err != nil {
t.Fatal(err)
}
jsonbyte, err := ioutil.ReadAll(f)
if err != nil {
t.Fatal(err)
}
var snap simulations.Snapshot
err = json.Unmarshal(jsonbyte, &snap)
if err != nil {
t.Fatal(err)
}
err = net.Load(&snap)
if err != nil {
t.Fatal(err)
}
triggerChecks := func(trigger chan discover.NodeID, id discover.NodeID, rpcclient *rpc.Client, topic string) error {
msgC := make(chan APIMsg)
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
sub, err := rpcclient.Subscribe(ctx, "pss", msgC, "receive", topic)
if err != nil {
t.Fatal(err)
}
go func() {
defer sub.Unsubscribe()
for {
select {
case recvmsg := <-msgC:
idx, _ := binary.Uvarint(recvmsg.Msg)
if !recvmsgs[idx] {
log.Debug("msg recv", "idx", idx, "id", id)
recvmsgs[idx] = true
trigger <- id
}
case <-sub.Err():
return
}
}
}()
return nil
}
var topic string
for i, nod := range net.GetNodes() {
nodes[i] = nod.ID()
rpcs[nodes[i]], err = nod.Client()
if err != nil {
t.Fatal(err)
}
if topic == "" {
err = rpcs[nodes[i]].Call(&topic, "pss_stringToTopic", "foo:42")
if err != nil {
t.Fatal(err)
}
}
var pubkey string
err = rpcs[nodes[i]].Call(&pubkey, "pss_getPublicKey")
if err != nil {
t.Fatal(err)
}
pubkeys[nod.ID()] = pubkey
var addrhex string
err = rpcs[nodes[i]].Call(&addrhex, "pss_baseAddr")
if err != nil {
t.Fatal(err)
}
bzzaddrs[nodes[i]] = addrhex
err = triggerChecks(trigger, nodes[i], rpcs[nodes[i]], topic)
if err != nil {
t.Fatal(err)
}
}
// setup workers
jobs := make(chan Job, 10)
for w := 1; w <= 10; w++ {
go worker(w, jobs, rpcs, pubkeys, topic)
}
for i := 0; i < int(msgcount); i++ {
sendnodeidx := rand.Intn(int(nodecount))
recvnodeidx := rand.Intn(int(nodecount - 1))
if recvnodeidx >= sendnodeidx {
recvnodeidx++
}
nodemsgcount[nodes[recvnodeidx]]++
sentmsgs[i] = make([]byte, 8)
c := binary.PutUvarint(sentmsgs[i], uint64(i))
if c == 0 {
t.Fatal("0 byte message")
}
if err != nil {
t.Fatal(err)
}
err = rpcs[nodes[sendnodeidx]].Call(nil, "pss_setPeerPublicKey", pubkeys[nodes[recvnodeidx]], topic, bzzaddrs[nodes[recvnodeidx]])
if err != nil {
t.Fatal(err)
}
jobs <- Job{
Msg: sentmsgs[i],
SendNode: nodes[sendnodeidx],
RecvNode: nodes[recvnodeidx],
}
}
finalmsgcount := 0
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
outer:
for i := 0; i < int(msgcount); i++ {
select {
case id := <-trigger:
nodemsgcount[id]--
finalmsgcount++
case <-ctx.Done():
log.Warn("timeout")
break outer
}
}
for i, msg := range recvmsgs {
if !msg {
log.Debug("missing message", "idx", i)
}
}
t.Logf("%d of %d messages received", finalmsgcount, msgcount)
if finalmsgcount != int(msgcount) {
t.Fatalf("%d messages were not received", int(msgcount)-finalmsgcount)
}
}
// symmetric send performance with varying message sizes
func BenchmarkSymkeySend(b *testing.B) {
b.Run(fmt.Sprintf("%d", 256), benchmarkSymKeySend)
b.Run(fmt.Sprintf("%d", 1024), benchmarkSymKeySend)
b.Run(fmt.Sprintf("%d", 1024*1024), benchmarkSymKeySend)
b.Run(fmt.Sprintf("%d", 1024*1024*10), benchmarkSymKeySend)
b.Run(fmt.Sprintf("%d", 1024*1024*100), benchmarkSymKeySend)
}
func benchmarkSymKeySend(b *testing.B) {
msgsizestring := strings.Split(b.Name(), "/")
if len(msgsizestring) != 2 {
b.Fatalf("benchmark called without msgsize param")
}
msgsize, err := strconv.ParseInt(msgsizestring[1], 10, 0)
if err != nil {
b.Fatalf("benchmark called with invalid msgsize param '%s': %v", msgsizestring[1], err)
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
keys, err := wapi.NewKeyPair(ctx)
privkey, err := w.GetPrivateKey(keys)
ps := newTestPss(privkey, nil, nil)
msg := make([]byte, msgsize)
rand.Read(msg)
topic := BytesToTopic([]byte("foo"))
to := make(PssAddress, 32)
copy(to[:], network.RandomAddr().Over())
symkeyid, err := ps.generateSymmetricKey(topic, &to, true)
if err != nil {
b.Fatalf("could not generate symkey: %v", err)
}
symkey, err := ps.w.GetSymKey(symkeyid)
if err != nil {
b.Fatalf("could not retrieve symkey: %v", err)
}
ps.SetSymmetricKey(symkey, topic, &to, false)
b.ResetTimer()
for i := 0; i < b.N; i++ {
ps.SendSym(symkeyid, topic, msg)
}
}
// asymmetric send performance with varying message sizes
func BenchmarkAsymkeySend(b *testing.B) {
b.Run(fmt.Sprintf("%d", 256), benchmarkAsymKeySend)
b.Run(fmt.Sprintf("%d", 1024), benchmarkAsymKeySend)
b.Run(fmt.Sprintf("%d", 1024*1024), benchmarkAsymKeySend)
b.Run(fmt.Sprintf("%d", 1024*1024*10), benchmarkAsymKeySend)
b.Run(fmt.Sprintf("%d", 1024*1024*100), benchmarkAsymKeySend)
}
func benchmarkAsymKeySend(b *testing.B) {
msgsizestring := strings.Split(b.Name(), "/")
if len(msgsizestring) != 2 {
b.Fatalf("benchmark called without msgsize param")
}
msgsize, err := strconv.ParseInt(msgsizestring[1], 10, 0)
if err != nil {
b.Fatalf("benchmark called with invalid msgsize param '%s': %v", msgsizestring[1], err)
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
keys, err := wapi.NewKeyPair(ctx)
privkey, err := w.GetPrivateKey(keys)
ps := newTestPss(privkey, nil, nil)
msg := make([]byte, msgsize)
rand.Read(msg)
topic := BytesToTopic([]byte("foo"))
to := make(PssAddress, 32)
copy(to[:], network.RandomAddr().Over())
ps.SetPeerPublicKey(&privkey.PublicKey, topic, &to)
b.ResetTimer()
for i := 0; i < b.N; i++ {
ps.SendAsym(common.ToHex(crypto.FromECDSAPub(&privkey.PublicKey)), topic, msg)
}
}
func BenchmarkSymkeyBruteforceChangeaddr(b *testing.B) {
for i := 100; i < 100000; i = i * 10 {
for j := 32; j < 10000; j = j * 8 {
b.Run(fmt.Sprintf("%d/%d", i, j), benchmarkSymkeyBruteforceChangeaddr)
}
//b.Run(fmt.Sprintf("%d", i), benchmarkSymkeyBruteforceChangeaddr)
}
}
// decrypt performance using symkey cache, worst case
// (decrypt key always last in cache)
func benchmarkSymkeyBruteforceChangeaddr(b *testing.B) {
keycountstring := strings.Split(b.Name(), "/")
cachesize := int64(0)
var ps *Pss
if len(keycountstring) < 2 {
b.Fatalf("benchmark called without count param")
}
keycount, err := strconv.ParseInt(keycountstring[1], 10, 0)
if err != nil {
b.Fatalf("benchmark called with invalid count param '%s': %v", keycountstring[1], err)
}
if len(keycountstring) == 3 {
cachesize, err = strconv.ParseInt(keycountstring[2], 10, 0)
if err != nil {
b.Fatalf("benchmark called with invalid cachesize '%s': %v", keycountstring[2], err)
}
}
pssmsgs := make([]*PssMsg, 0, keycount)
var keyid string
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
keys, err := wapi.NewKeyPair(ctx)
privkey, err := w.GetPrivateKey(keys)
if cachesize > 0 {
ps = newTestPss(privkey, nil, &PssParams{SymKeyCacheCapacity: int(cachesize)})
} else {
ps = newTestPss(privkey, nil, nil)
}
topic := BytesToTopic([]byte("foo"))
for i := 0; i < int(keycount); i++ {
to := make(PssAddress, 32)
copy(to[:], network.RandomAddr().Over())
keyid, err = ps.generateSymmetricKey(topic, &to, true)
if err != nil {
b.Fatalf("cant generate symkey #%d: %v", i, err)
}
symkey, err := ps.w.GetSymKey(keyid)
if err != nil {
b.Fatalf("could not retrieve symkey %s: %v", keyid, err)
}
wparams := &whisper.MessageParams{
TTL: defaultWhisperTTL,
KeySym: symkey,
Topic: whisper.TopicType(topic),
WorkTime: defaultWhisperWorkTime,
PoW: defaultWhisperPoW,
Payload: []byte("xyzzy"),
Padding: []byte("1234567890abcdef"),
}
woutmsg, err := whisper.NewSentMessage(wparams)
if err != nil {
b.Fatalf("could not create whisper message: %v", err)
}
env, err := woutmsg.Wrap(wparams)
if err != nil {
b.Fatalf("could not generate whisper envelope: %v", err)
}
ps.Register(&topic, func(msg []byte, p *p2p.Peer, asymmetric bool, keyid string) error {
return nil
})
pssmsgs = append(pssmsgs, &PssMsg{
To: to,
Payload: env,
})
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
if !ps.process(pssmsgs[len(pssmsgs)-(i%len(pssmsgs))-1]) {
b.Fatalf("pss processing failed: %v", err)
}
}
}
func BenchmarkSymkeyBruteforceSameaddr(b *testing.B) {
for i := 100; i < 100000; i = i * 10 {
for j := 32; j < 10000; j = j * 8 {
b.Run(fmt.Sprintf("%d/%d", i, j), benchmarkSymkeyBruteforceSameaddr)
}
}
}
// decrypt performance using symkey cache, best case
// (decrypt key always first in cache)
func benchmarkSymkeyBruteforceSameaddr(b *testing.B) {
var keyid string
var ps *Pss
cachesize := int64(0)
keycountstring := strings.Split(b.Name(), "/")
if len(keycountstring) < 2 {
b.Fatalf("benchmark called without count param")
}
keycount, err := strconv.ParseInt(keycountstring[1], 10, 0)
if err != nil {
b.Fatalf("benchmark called with invalid count param '%s': %v", keycountstring[1], err)
}
if len(keycountstring) == 3 {
cachesize, err = strconv.ParseInt(keycountstring[2], 10, 0)
if err != nil {
b.Fatalf("benchmark called with invalid cachesize '%s': %v", keycountstring[2], err)
}
}
addr := make([]PssAddress, keycount)
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
keys, err := wapi.NewKeyPair(ctx)
privkey, err := w.GetPrivateKey(keys)
if cachesize > 0 {
ps = newTestPss(privkey, nil, &PssParams{SymKeyCacheCapacity: int(cachesize)})
} else {
ps = newTestPss(privkey, nil, nil)
}
topic := BytesToTopic([]byte("foo"))
for i := 0; i < int(keycount); i++ {
copy(addr[i], network.RandomAddr().Over())
keyid, err = ps.generateSymmetricKey(topic, &addr[i], true)
if err != nil {
b.Fatalf("cant generate symkey #%d: %v", i, err)
}
}
symkey, err := ps.w.GetSymKey(keyid)
if err != nil {
b.Fatalf("could not retrieve symkey %s: %v", keyid, err)
}
wparams := &whisper.MessageParams{
TTL: defaultWhisperTTL,
KeySym: symkey,
Topic: whisper.TopicType(topic),
WorkTime: defaultWhisperWorkTime,
PoW: defaultWhisperPoW,
Payload: []byte("xyzzy"),
Padding: []byte("1234567890abcdef"),
}
woutmsg, err := whisper.NewSentMessage(wparams)
if err != nil {
b.Fatalf("could not create whisper message: %v", err)
}
env, err := woutmsg.Wrap(wparams)
if err != nil {
b.Fatalf("could not generate whisper envelope: %v", err)
}
ps.Register(&topic, func(msg []byte, p *p2p.Peer, asymmetric bool, keyid string) error {
return nil
})
pssmsg := &PssMsg{
To: addr[len(addr)-1][:],
Payload: env,
}
for i := 0; i < b.N; i++ {
if !ps.process(pssmsg) {
b.Fatalf("pss processing failed: %v", err)
}
}
}
// setup simulated network and connect nodes in circle
func setupNetwork(numnodes int) (clients []*rpc.Client, err error) {
nodes := make([]*simulations.Node, numnodes)
@ -405,9 +1081,9 @@ func setupNetwork(numnodes int) (clients []*rpc.Client, err error) {
DefaultService: "bzz",
})
for i := 0; i < numnodes; i++ {
nodes[i], err = net.NewNodeWithConfig(&adapters.NodeConfig{
Services: []string{"bzz", pssProtocolName},
})
nodeconf := adapters.RandomNodeConfig()
nodeconf.Services = []string{"bzz", pssProtocolName}
nodes[i], err = net.NewNodeWithConfig(nodeconf)
if err != nil {
return nil, fmt.Errorf("error creating node 1: %v", err)
}
@ -457,11 +1133,11 @@ func newServices() adapters.Services {
pssProtocolName: func(ctx *adapters.ServiceContext) (node.Service, error) {
cachedir, err := ioutil.TempDir("", "pss-cache")
if err != nil {
return nil, fmt.Errorf("create pss cache tmpdir failed: %v", err)
return nil, fmt.Errorf("create pss cache tmpdir failed: %s", err)
}
dpa, err := storage.NewLocalDPA(cachedir)
dpa, err := storage.NewLocalDPA(cachedir, network.NewAddrFromNodeID(ctx.Config.ID).Over())
if err != nil {
return nil, fmt.Errorf("local dpa creation failed: %v", err)
return nil, fmt.Errorf("local dpa creation failed: %s", err)
}
// execadapter does not exec init()
@ -532,7 +1208,7 @@ func newTestPss(privkey *ecdsa.PrivateKey, overlay network.Overlay, ppextra *Pss
log.Error("create pss cache tmpdir failed", "error", err)
os.Exit(1)
}
dpa, err := storage.NewLocalDPA(cachedir)
dpa, err := storage.NewLocalDPA(cachedir, addr.Over())
if err != nil {
log.Error("local dpa creation failed", "error", err)
os.Exit(1)

118
swarm/storage/chunker.go
View file

@ -13,7 +13,6 @@
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package storage
import (
@ -118,23 +117,19 @@ func (self *TreeChunker) decrementWorkerCount() {
self.workerCount -= 1
}
func (self *TreeChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, swg, wwg *sync.WaitGroup) (Key, error) {
func (self *TreeChunker) Split(data io.Reader, size int64, chunkC chan *Chunk) (k Key, wait func(), err error) {
if self.chunkSize <= 0 {
panic("chunker must be initialised")
}
jobC := make(chan *hashJob, 2*ChunkProcessors)
wg := &sync.WaitGroup{}
storeWg := &sync.WaitGroup{}
errC := make(chan error)
quitC := make(chan bool)
// wwg = workers waitgroup keeps track of hashworkers spawned by this split call
if wwg != nil {
wwg.Add(1)
}
self.incrementWorkerCount()
go self.hashWorker(jobC, chunkC, errC, quitC, swg, wwg)
self.runHashWorker(jobC, chunkC, errC, quitC, storeWg)
depth := 0
treeSize := self.chunkSize
@ -149,16 +144,12 @@ func (self *TreeChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, s
// this waitgroup member is released after the root hash is calculated
wg.Add(1)
//launch actual recursive function passing the waitgroups
go self.split(depth, treeSize/self.branches, key, data, size, jobC, chunkC, errC, quitC, wg, swg, wwg)
go self.split(depth, treeSize/self.branches, key, data, size, jobC, chunkC, errC, quitC, wg, storeWg)
// closes internal error channel if all subprocesses in the workgroup finished
go func() {
// waiting for all threads to finish
wg.Wait()
// if storage waitgroup is non-nil, we wait for storage to finish too
if swg != nil {
swg.Wait()
}
close(errC)
}()
@ -166,16 +157,16 @@ func (self *TreeChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, s
select {
case err := <-errC:
if err != nil {
return nil, err
return nil, nil, err
}
case <-time.NewTimer(splitTimeout).C:
return nil, errOperationTimedOut
return nil, nil, errOperationTimedOut
}
return key, nil
return key, storeWg.Wait, nil
}
func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reader, size int64, jobC chan *hashJob, chunkC chan *Chunk, errC chan error, quitC chan bool, parentWg, swg, wwg *sync.WaitGroup) {
func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reader, size int64, jobC chan *hashJob, chunkC chan *Chunk, errC chan error, quitC chan bool, parentWg, storeWg *sync.WaitGroup) {
//
@ -225,7 +216,7 @@ func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reade
subTreeKey := chunk[8+i*self.hashSize : 8+(i+1)*self.hashSize]
childrenWg.Add(1)
self.split(depth-1, treeSize/self.branches, subTreeKey, data, secSize, jobC, chunkC, errC, quitC, childrenWg, swg, wwg)
self.split(depth-1, treeSize/self.branches, subTreeKey, data, secSize, jobC, chunkC, errC, quitC, childrenWg, storeWg)
i++
pos += treeSize
@ -237,11 +228,8 @@ func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reade
worker := self.getWorkerCount()
if int64(len(jobC)) > worker && worker < ChunkProcessors {
if wwg != nil {
wwg.Add(1)
}
self.incrementWorkerCount()
go self.hashWorker(jobC, chunkC, errC, quitC, swg, wwg)
self.runHashWorker(jobC, chunkC, errC, quitC, storeWg)
}
select {
@ -250,60 +238,58 @@ func (self *TreeChunker) split(depth int, treeSize int64, key Key, data io.Reade
}
}
func (self *TreeChunker) hashWorker(jobC chan *hashJob, chunkC chan *Chunk, errC chan error, quitC chan bool, swg, wwg *sync.WaitGroup) {
defer self.decrementWorkerCount()
func (self *TreeChunker) runHashWorker(jobC chan *hashJob, chunkC chan *Chunk, errC chan error, quitC chan bool, storeWg *sync.WaitGroup) {
storeWg.Add(1)
hasher := self.hashFunc()
if wwg != nil {
defer wwg.Done()
}
for {
select {
go func() {
defer self.decrementWorkerCount()
defer storeWg.Done()
hasher := self.hashFunc()
for {
select {
case job, ok := <-jobC:
if !ok {
case job, ok := <-jobC:
if !ok {
return
}
// now we got the hashes in the chunk, then hash the chunks
self.hashChunk(hasher, job, chunkC, storeWg)
case <-quitC:
return
}
// now we got the hashes in the chunk, then hash the chunks
self.hashChunk(hasher, job, chunkC, swg)
case <-quitC:
return
}
}
}()
}
// The treeChunkers own Hash hashes together
// - the size (of the subtree encoded in the Chunk)
// - the Chunk, ie. the contents read from the input reader
func (self *TreeChunker) hashChunk(hasher SwarmHash, job *hashJob, chunkC chan *Chunk, swg *sync.WaitGroup) {
func (self *TreeChunker) hashChunk(hasher SwarmHash, job *hashJob, chunkC chan *Chunk, storeWg *sync.WaitGroup) {
hasher.ResetWithLength(job.chunk[:8]) // 8 bytes of length
hasher.Write(job.chunk[8:]) // minus 8 []byte length
h := hasher.Sum(nil)
newChunk := &Chunk{
Key: h,
SData: job.chunk,
Size: job.size,
wg: swg,
}
newChunk := NewChunk(h, nil)
newChunk.SData = job.chunk
newChunk.Size = job.size
// report hash of this chunk one level up (keys corresponds to the proper subslice of the parent chunk)
copy(job.key, h)
// send off new chunk to storage
if chunkC != nil {
if swg != nil {
swg.Add(1)
}
}
job.parentWg.Done()
if chunkC != nil {
chunkC <- newChunk
storeWg.Add(1)
go func() {
defer storeWg.Done()
<-newChunk.dbStored
}()
}
}
func (self *TreeChunker) Append(key Key, data io.Reader, chunkC chan *Chunk, swg, wwg *sync.WaitGroup) (Key, error) {
return nil, errAppendOppNotSuported
func (self *TreeChunker) Append(key Key, data io.Reader, chunkC chan *Chunk) (Key, func(), error) {
return nil, nil, errAppendOppNotSuported
}
// LazyChunkReader implements LazySectionReader
@ -315,16 +301,18 @@ type LazyChunkReader struct {
chunkSize int64 // inherit from chunker
branches int64 // inherit from chunker
hashSize int64 // inherit from chunker
depth int
}
// implements the Joiner interface
func (self *TreeChunker) Join(key Key, chunkC chan *Chunk) LazySectionReader {
func (self *TreeChunker) Join(key Key, chunkC chan *Chunk, depth int) LazySectionReader {
return &LazyChunkReader{
key: key,
chunkC: chunkC,
chunkSize: self.chunkSize,
branches: self.branches,
hashSize: self.hashSize,
depth: depth,
}
}
@ -371,8 +359,13 @@ func (self *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) {
depth++
}
wg := sync.WaitGroup{}
length := int64(len(b))
for d := 0; d < self.depth; d++ {
off *= self.chunkSize
length *= self.chunkSize
}
wg.Add(1)
go self.join(b, off, off+int64(len(b)), depth, treeSize/self.branches, self.chunk, &wg, errC, quitC)
go self.join(b, off, off+length, depth, treeSize/self.branches, self.chunk, &wg, errC, quitC)
go func() {
wg.Wait()
close(errC)
@ -385,25 +378,21 @@ func (self *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) {
return 0, err
}
if off+int64(len(b)) >= size {
return len(b), io.EOF
return int(size - off), io.EOF
}
return len(b), nil
}
func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeSize int64, chunk *Chunk, parentWg *sync.WaitGroup, errC chan error, quitC chan bool) {
defer parentWg.Done()
// return NewDPA(&LocalStore{})
// chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8]))
// find appropriate block level
for chunk.Size < treeSize && depth > 0 {
for chunk.Size < treeSize && depth > self.depth {
treeSize /= self.branches
depth--
}
// leaf chunk found
if depth == 0 {
if depth == self.depth {
extra := 8 + eoff - int64(len(chunk.SData))
if extra > 0 {
eoff -= extra
@ -456,10 +445,8 @@ func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, tr
// block until they time out or arrive
// abort if quitC is readable
func retrieve(key Key, chunkC chan *Chunk, quitC chan bool) *Chunk {
chunk := &Chunk{
Key: key,
C: make(chan bool), // close channel to signal data delivery
}
chunk := NewChunk(key, nil)
chunk.C = make(chan bool)
// submit chunk for retrieval
select {
case chunkC <- chunk: // submit retrieval request, someone should be listening on the other side (or we will time out globally)
@ -475,8 +462,7 @@ func retrieve(key Key, chunkC chan *Chunk, quitC chan bool) *Chunk {
case <-chunk.C: // bells are ringing, data have been delivered
}
if len(chunk.SData) == 0 {
return nil // chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8]))
return nil
}
return chunk
}

135
swarm/storage/chunker_test.go
View file

@ -23,7 +23,6 @@ import (
"errors"
"fmt"
"io"
"sync"
"testing"
"time"
@ -45,7 +44,7 @@ type chunkerTester struct {
t test
}
func (self *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, chunkC chan *Chunk, swg *sync.WaitGroup, expectedError error) (key Key, err error) {
func (self *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, chunkC chan *Chunk, expectedError error) (key Key, wait func(), err error) {
// reset
self.chunks = make(map[string]*Chunk)
@ -65,31 +64,31 @@ func (self *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, c
return nil
case chunk := <-chunkC:
// self.chunks = append(self.chunks, chunk)
self.chunks[chunk.Key.String()] = chunk
if chunk.wg != nil {
chunk.wg.Done()
}
self.chunks[chunk.Key.Hex()] = chunk
close(chunk.dbStored)
}
}
}()
}
key, err = chunker.Split(data, size, chunkC, swg, nil)
var w func()
key, w, err = chunker.Split(data, size, chunkC)
if err != nil && expectedError == nil {
err = fmt.Errorf("Split error: %v", err)
}
if chunkC != nil {
if swg != nil {
swg.Wait()
wait = func() {
w()
close(quitC)
}
close(quitC)
} else {
wait = func() {}
}
return key, err
return key, wait, err
}
func (self *chunkerTester) Append(chunker Splitter, rootKey Key, data io.Reader, chunkC chan *Chunk, swg *sync.WaitGroup, expectedError error) (key Key, err error) {
func (self *chunkerTester) Append(chunker Splitter, rootKey Key, data io.Reader, chunkC chan *Chunk, expectedError error) (key Key, wait func(), err error) {
quitC := make(chan bool)
timeout := time.After(60 * time.Second)
if chunkC != nil {
@ -102,17 +101,16 @@ func (self *chunkerTester) Append(chunker Splitter, rootKey Key, data io.Reader,
return nil
case chunk := <-chunkC:
if chunk != nil {
stored, success := self.chunks[chunk.Key.String()]
stored, success := self.chunks[chunk.Key.Hex()]
if !success {
// Requesting data
self.chunks[chunk.Key.String()] = chunk
if chunk.wg != nil {
chunk.wg.Done()
}
self.chunks[chunk.Key.Hex()] = chunk
close(chunk.dbStored)
} else {
// getting data
chunk.SData = stored.SData
chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8]))
close(chunk.dbStored)
if chunk.C != nil {
close(chunk.C)
}
@ -122,26 +120,26 @@ func (self *chunkerTester) Append(chunker Splitter, rootKey Key, data io.Reader,
}
}()
}
key, err = chunker.Append(rootKey, data, chunkC, swg, nil)
var w func()
key, w, err = chunker.Append(rootKey, data, chunkC)
if err != nil && expectedError == nil {
err = fmt.Errorf("Append error: %v", err)
}
if chunkC != nil {
if swg != nil {
swg.Wait()
wait = func() {
w()
close(quitC)
}
close(quitC)
} else {
wait = func() {}
}
return key, err
return key, wait, err
}
func (self *chunkerTester) Join(chunker Chunker, key Key, c int, chunkC chan *Chunk, quitC chan bool) LazySectionReader {
// reset but not the chunks
reader := chunker.Join(key, chunkC)
timeout := time.After(600 * time.Second)
i := 0
go func() error {
@ -155,7 +153,7 @@ func (self *chunkerTester) Join(chunker Chunker, key Key, c int, chunkC chan *Ch
return nil
}
// this just mocks the behaviour of a chunk store retrieval
stored, success := self.chunks[chunk.Key.String()]
stored, success := self.chunks[chunk.Key.Hex()]
if !success {
return errors.New("Not found")
}
@ -166,6 +164,8 @@ func (self *chunkerTester) Join(chunker Chunker, key Key, c int, chunkC chan *Ch
}
}
}()
reader := chunker.Join(key, chunkC, 0)
return reader
}
@ -183,10 +183,9 @@ func testRandomBrokenData(splitter Splitter, n int, tester *chunkerTester) {
brokendata = brokenLimitReader(data, n, n/2)
chunkC := make(chan *Chunk, 1000)
swg := &sync.WaitGroup{}
expectedError := fmt.Errorf("Broken reader")
key, err := tester.Split(splitter, brokendata, int64(n), chunkC, swg, expectedError)
key, _, err := tester.Split(splitter, brokendata, int64(n), chunkC, expectedError)
if err == nil || err.Error() != expectedError.Error() {
tester.t.Fatalf("Not receiving the correct error! Expected %v, received %v", expectedError, err)
}
@ -207,14 +206,13 @@ func testRandomData(splitter Splitter, n int, tester *chunkerTester) Key {
}
chunkC := make(chan *Chunk, 1000)
swg := &sync.WaitGroup{}
key, err := tester.Split(splitter, data, int64(n), chunkC, swg, nil)
key, wait, err := tester.Split(splitter, data, int64(n), chunkC, nil)
if err != nil {
tester.t.Fatalf(err.Error())
}
tester.t.Logf(" Key = %v\n", key)
wait()
chunkC = make(chan *Chunk, 1000)
quitC := make(chan bool)
@ -250,12 +248,12 @@ func testRandomDataAppend(splitter Splitter, n, m int, tester *chunkerTester) {
}
chunkC := make(chan *Chunk, 1000)
swg := &sync.WaitGroup{}
key, err := tester.Split(splitter, data, int64(n), chunkC, swg, nil)
key, wait, err := tester.Split(splitter, data, int64(n), chunkC, nil)
if err != nil {
tester.t.Fatalf(err.Error())
}
wait()
tester.t.Logf(" Key = %v\n", key)
//create a append data stream
@ -269,12 +267,12 @@ func testRandomDataAppend(splitter Splitter, n, m int, tester *chunkerTester) {
}
chunkC = make(chan *Chunk, 1000)
swg = &sync.WaitGroup{}
newKey, err := tester.Append(splitter, key, appendData, chunkC, swg, nil)
newKey, wait, err := tester.Append(splitter, key, appendData, chunkC, nil)
if err != nil {
tester.t.Fatalf(err.Error())
}
wait()
tester.t.Logf(" NewKey = %v\n", newKey)
chunkC = make(chan *Chunk, 1000)
@ -325,17 +323,19 @@ func TestSha3ForCorrectness(t *testing.T) {
}
func TestDataAppend(t *testing.T) {
sizes := []int{1, 1, 1, 4095, 4096, 4097, 1, 1, 1, 123456, 2345678, 2345678}
appendSizes := []int{4095, 4096, 4097, 1, 1, 1, 8191, 8192, 8193, 9000, 3000, 5000}
tester := &chunkerTester{t: t}
chunker := NewPyramidChunker(NewChunkerParams())
for i, s := range sizes {
testRandomDataAppend(chunker, s, appendSizes[i], tester)
}
}
// func TestDataAppend(t *testing.T) {
// // sizes := []int{1, 1, 1, 4095, 4096, 4097, 1, 1, 1, 123456, 2345678, 2345678}
// sizes := []int{1}
// // appendSizes := []int{4095, 4096, 4097, 1, 1, 1, 8191, 8192, 8193, 9000, 3000, 5000}
// appendSizes := []int{4095}
//
// tester := &chunkerTester{t: t}
// chunker := NewPyramidChunker(NewChunkerParams())
// for i, s := range sizes {
// testRandomDataAppend(chunker, s, appendSizes[i], tester)
//
// }
// }
func TestRandomData(t *testing.T) {
sizes := []int{1, 60, 83, 179, 253, 1024, 4095, 4096, 4097, 8191, 8192, 8193, 12287, 12288, 12289, 123456, 2345678}
@ -390,12 +390,12 @@ func benchmarkJoin(n int, t *testing.B) {
data := testDataReader(n)
chunkC := make(chan *Chunk, 1000)
swg := &sync.WaitGroup{}
key, err := tester.Split(chunker, data, int64(n), chunkC, swg, nil)
key, wait, err := tester.Split(chunker, data, int64(n), chunkC, nil)
if err != nil {
tester.t.Fatalf(err.Error())
}
wait()
chunkC = make(chan *Chunk, 1000)
quitC := make(chan bool)
reader := tester.Join(chunker, key, i, chunkC, quitC)
@ -411,7 +411,7 @@ func benchmarkSplitTreeSHA3(n int, t *testing.B) {
chunker := NewTreeChunker(NewChunkerParams())
tester := &chunkerTester{t: t}
data := testDataReader(n)
_, err := tester.Split(chunker, data, int64(n), nil, nil, nil)
_, _, err := tester.Split(chunker, data, int64(n), nil, nil)
if err != nil {
tester.t.Fatalf(err.Error())
}
@ -426,7 +426,7 @@ func benchmarkSplitTreeBMT(n int, t *testing.B) {
chunker := NewTreeChunker(cp)
tester := &chunkerTester{t: t}
data := testDataReader(n)
_, err := tester.Split(chunker, data, int64(n), nil, nil, nil)
_, _, err := tester.Split(chunker, data, int64(n), nil, nil)
if err != nil {
tester.t.Fatalf(err.Error())
}
@ -439,10 +439,11 @@ func benchmarkSplitPyramidSHA3(n int, t *testing.B) {
splitter := NewPyramidChunker(NewChunkerParams())
tester := &chunkerTester{t: t}
data := testDataReader(n)
_, err := tester.Split(splitter, data, int64(n), nil, nil, nil)
_, _, err := tester.Split(splitter, data, int64(n), nil, nil)
if err != nil {
tester.t.Fatalf(err.Error())
}
}
}
@ -454,7 +455,7 @@ func benchmarkSplitPyramidBMT(n int, t *testing.B) {
splitter := NewPyramidChunker(cp)
tester := &chunkerTester{t: t}
data := testDataReader(n)
_, err := tester.Split(splitter, data, int64(n), nil, nil, nil)
_, _, err := tester.Split(splitter, data, int64(n), nil, nil)
if err != nil {
tester.t.Fatalf(err.Error())
}
@ -470,20 +471,18 @@ func benchmarkAppendPyramid(n, m int, t *testing.B) {
data1 := testDataReader(m)
chunkC := make(chan *Chunk, 1000)
swg := &sync.WaitGroup{}
key, err := tester.Split(chunker, data, int64(n), chunkC, swg, nil)
key, wait, err := tester.Split(chunker, data, int64(n), chunkC, nil)
if err != nil {
tester.t.Fatalf(err.Error())
}
wait()
chunkC = make(chan *Chunk, 1000)
swg = &sync.WaitGroup{}
_, err = tester.Append(chunker, key, data1, chunkC, swg, nil)
_, wait, err = tester.Append(chunker, key, data1, chunkC, nil)
if err != nil {
tester.t.Fatalf(err.Error())
}
wait()
close(chunkC)
}
}
@ -494,7 +493,8 @@ func BenchmarkJoin_4(t *testing.B) { benchmarkJoin(10000, t) }
func BenchmarkJoin_5(t *testing.B) { benchmarkJoin(100000, t) }
func BenchmarkJoin_6(t *testing.B) { benchmarkJoin(1000000, t) }
func BenchmarkJoin_7(t *testing.B) { benchmarkJoin(10000000, t) }
func BenchmarkJoin_8(t *testing.B) { benchmarkJoin(100000000, t) }
// func BenchmarkJoin_8(t *testing.B) { benchmarkJoin(100000000, t) }
func BenchmarkSplitTreeSHA3_2(t *testing.B) { benchmarkSplitTreeSHA3(100, t) }
func BenchmarkSplitTreeSHA3_2h(t *testing.B) { benchmarkSplitTreeSHA3(500, t) }
@ -505,7 +505,8 @@ func BenchmarkSplitTreeSHA3_4h(t *testing.B) { benchmarkSplitTreeSHA3(50000, t)
func BenchmarkSplitTreeSHA3_5(t *testing.B) { benchmarkSplitTreeSHA3(100000, t) }
func BenchmarkSplitTreeSHA3_6(t *testing.B) { benchmarkSplitTreeSHA3(1000000, t) }
func BenchmarkSplitTreeSHA3_7(t *testing.B) { benchmarkSplitTreeSHA3(10000000, t) }
func BenchmarkSplitTreeSHA3_8(t *testing.B) { benchmarkSplitTreeSHA3(100000000, t) }
// func BenchmarkSplitTreeSHA3_8(t *testing.B) { benchmarkSplitTreeSHA3(100000000, t) }
func BenchmarkSplitTreeBMT_2(t *testing.B) { benchmarkSplitTreeBMT(100, t) }
func BenchmarkSplitTreeBMT_2h(t *testing.B) { benchmarkSplitTreeBMT(500, t) }
@ -516,7 +517,8 @@ func BenchmarkSplitTreeBMT_4h(t *testing.B) { benchmarkSplitTreeBMT(50000, t) }
func BenchmarkSplitTreeBMT_5(t *testing.B) { benchmarkSplitTreeBMT(100000, t) }
func BenchmarkSplitTreeBMT_6(t *testing.B) { benchmarkSplitTreeBMT(1000000, t) }
func BenchmarkSplitTreeBMT_7(t *testing.B) { benchmarkSplitTreeBMT(10000000, t) }
func BenchmarkSplitTreeBMT_8(t *testing.B) { benchmarkSplitTreeBMT(100000000, t) }
// func BenchmarkSplitTreeBMT_8(t *testing.B) { benchmarkSplitTreeBMT(100000000, t) }
func BenchmarkSplitPyramidSHA3_2(t *testing.B) { benchmarkSplitPyramidSHA3(100, t) }
func BenchmarkSplitPyramidSHA3_2h(t *testing.B) { benchmarkSplitPyramidSHA3(500, t) }
@ -527,7 +529,8 @@ func BenchmarkSplitPyramidSHA3_4h(t *testing.B) { benchmarkSplitPyramidSHA3(5000
func BenchmarkSplitPyramidSHA3_5(t *testing.B) { benchmarkSplitPyramidSHA3(100000, t) }
func BenchmarkSplitPyramidSHA3_6(t *testing.B) { benchmarkSplitPyramidSHA3(1000000, t) }
func BenchmarkSplitPyramidSHA3_7(t *testing.B) { benchmarkSplitPyramidSHA3(10000000, t) }
func BenchmarkSplitPyramidSHA3_8(t *testing.B) { benchmarkSplitPyramidSHA3(100000000, t) }
// func BenchmarkSplitPyramidSHA3_8(t *testing.B) { benchmarkSplitPyramidSHA3(100000000, t) }
func BenchmarkSplitPyramidBMT_2(t *testing.B) { benchmarkSplitPyramidBMT(100, t) }
func BenchmarkSplitPyramidBMT_2h(t *testing.B) { benchmarkSplitPyramidBMT(500, t) }
@ -538,7 +541,8 @@ func BenchmarkSplitPyramidBMT_4h(t *testing.B) { benchmarkSplitPyramidBMT(50000,
func BenchmarkSplitPyramidBMT_5(t *testing.B) { benchmarkSplitPyramidBMT(100000, t) }
func BenchmarkSplitPyramidBMT_6(t *testing.B) { benchmarkSplitPyramidBMT(1000000, t) }
func BenchmarkSplitPyramidBMT_7(t *testing.B) { benchmarkSplitPyramidBMT(10000000, t) }
func BenchmarkSplitPyramidBMT_8(t *testing.B) { benchmarkSplitPyramidBMT(100000000, t) }
// func BenchmarkSplitPyramidBMT_8(t *testing.B) { benchmarkSplitPyramidBMT(100000000, t) }
func BenchmarkAppendPyramid_2(t *testing.B) { benchmarkAppendPyramid(100, 1000, t) }
func BenchmarkAppendPyramid_2h(t *testing.B) { benchmarkAppendPyramid(500, 1000, t) }
@ -548,7 +552,8 @@ func BenchmarkAppendPyramid_4h(t *testing.B) { benchmarkAppendPyramid(50000, 100
func BenchmarkAppendPyramid_5(t *testing.B) { benchmarkAppendPyramid(1000000, 1000, t) }
func BenchmarkAppendPyramid_6(t *testing.B) { benchmarkAppendPyramid(1000000, 1000, t) }
func BenchmarkAppendPyramid_7(t *testing.B) { benchmarkAppendPyramid(10000000, 1000, t) }
func BenchmarkAppendPyramid_8(t *testing.B) { benchmarkAppendPyramid(100000000, 1000, t) }
// func BenchmarkAppendPyramid_8(t *testing.B) { benchmarkAppendPyramid(100000000, 1000, t) }
// go test -timeout 20m -cpu 4 -bench=./swarm/storage -run no
// If you dont add the timeout argument above .. the benchmark will timeout and dump

218
swarm/storage/common_test.go
View file

@ -19,14 +19,29 @@ package storage
import (
"bytes"
"crypto/rand"
"encoding/binary"
"flag"
"fmt"
"hash"
"io"
"os"
"sync"
"testing"
"time"
"github.com/ethereum/go-ethereum/crypto/sha3"
"github.com/ethereum/go-ethereum/log"
)
var (
loglevel = flag.Int("loglevel", 2, "verbosity of logs")
)
func init() {
flag.Parse()
log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(false))))
}
type brokenLimitedReader struct {
lr io.Reader
errAt int
@ -42,16 +57,117 @@ func brokenLimitReader(data io.Reader, size int, errAt int) *brokenLimitedReader
}
}
func mputChunks(store ChunkStore, processors int, n int, chunksize int, hash hash.Hash) (hs []Key) {
f := func(int) *Chunk {
data := make([]byte, chunksize)
rand.Reader.Read(data)
hash.Reset()
hash.Write(data)
h := hash.Sum(nil)
chunk := NewChunk(Key(h), nil)
chunk.SData = data
return chunk
}
return mput(store, processors, n, f)
}
func mputRandomKey(store ChunkStore, processors int, n int, chunksize int) (hs []Key) {
data := make([]byte, chunksize+8)
binary.LittleEndian.PutUint64(data[0:8], uint64(chunksize))
f := func(int) *Chunk {
h := make([]byte, 32)
rand.Reader.Read(h)
chunk := NewChunk(Key(h), nil)
chunk.SData = data
return chunk
}
return mput(store, processors, n, f)
}
func mput(store ChunkStore, processors int, n int, f func(i int) *Chunk) (hs []Key) {
wg := sync.WaitGroup{}
wg.Add(processors)
c := make(chan *Chunk)
for i := 0; i < processors; i++ {
go func() {
defer wg.Done()
for chunk := range c {
wg.Add(1)
chunk := chunk
go func() {
defer wg.Done()
store.Put(chunk)
<-chunk.dbStored
}()
}
}()
}
fa := f
if _, ok := store.(*MemStore); ok {
fa = func(i int) *Chunk {
chunk := f(i)
close(chunk.dbStored)
return chunk
}
}
for i := 0; i < n; i++ {
chunk := fa(i)
hs = append(hs, chunk.Key)
c <- chunk
}
close(c)
wg.Wait()
return hs
}
func mget(store ChunkStore, hs []Key, f func(h Key, chunk *Chunk) error) error {
wg := sync.WaitGroup{}
wg.Add(len(hs))
errc := make(chan error)
for _, k := range hs {
go func(h Key) {
defer wg.Done()
chunk, err := store.Get(h)
if err != nil {
errc <- err
return
}
if f != nil {
err = f(h, chunk)
if err != nil {
errc <- err
return
}
}
}(k)
}
go func() {
wg.Wait()
close(errc)
}()
var err error
select {
case err = <-errc:
case <-time.NewTimer(5 * time.Second).C:
err = fmt.Errorf("timed out after 5 seconds")
}
return err
}
func testDataReader(l int) (r io.Reader) {
return io.LimitReader(rand.Reader, int64(l))
}
func (self *brokenLimitedReader) Read(buf []byte) (int, error) {
if self.off+len(buf) > self.errAt {
func (r *brokenLimitedReader) Read(buf []byte) (int, error) {
if r.off+len(buf) > r.errAt {
return 0, fmt.Errorf("Broken reader")
}
self.off += len(buf)
return self.lr.Read(buf)
r.off += len(buf)
return r.lr.Read(buf)
}
func testDataReaderAndSlice(l int) (r io.Reader, slice []byte) {
@ -63,54 +179,50 @@ func testDataReaderAndSlice(l int) (r io.Reader, slice []byte) {
return
}
func testStore(m ChunkStore, l int64, branches int64, t *testing.T) {
chunkC := make(chan *Chunk)
go func() {
for chunk := range chunkC {
m.Put(chunk)
if chunk.wg != nil {
chunk.wg.Done()
}
}
}()
chunker := NewTreeChunker(&ChunkerParams{
Branches: branches,
Hash: SHA3Hash,
})
swg := &sync.WaitGroup{}
key, _ := chunker.Split(rand.Reader, l, chunkC, swg, nil)
swg.Wait()
close(chunkC)
chunkC = make(chan *Chunk)
quit := make(chan bool)
go func() {
for ch := range chunkC {
go func(chunk *Chunk) {
storedChunk, err := m.Get(chunk.Key)
if err == notFound {
log.Trace(fmt.Sprintf("chunk '%v' not found", chunk.Key.Log()))
} else if err != nil {
log.Trace(fmt.Sprintf("error retrieving chunk %v: %v", chunk.Key.Log(), err))
} else {
chunk.SData = storedChunk.SData
chunk.Size = storedChunk.Size
}
log.Trace(fmt.Sprintf("chunk '%v' not found", chunk.Key.Log()))
close(chunk.C)
}(ch)
}
close(quit)
}()
r := chunker.Join(key, chunkC)
b := make([]byte, l)
n, err := r.ReadAt(b, 0)
if err != io.EOF {
t.Fatalf("read error (%v/%v) %v", n, l, err)
func testStoreRandom(m ChunkStore, processors int, n int, chunksize int, t *testing.T) {
hs := mputRandomKey(m, processors, n, chunksize)
err := mget(m, hs, nil)
if err != nil {
t.Fatalf("testStore failed: %v", err)
}
}
func testStoreCorrect(m ChunkStore, processors int, n int, chunksize int, t *testing.T) {
hs := mputChunks(m, processors, n, chunksize, sha3.NewKeccak256())
f := func(h Key, chunk *Chunk) error {
if !bytes.Equal(h, chunk.Key) {
return fmt.Errorf("key does not match retrieved chunk Key")
}
hasher := sha3.NewKeccak256()
hasher.Write(chunk.SData)
exp := hasher.Sum(nil)
if !bytes.Equal(h, exp) {
return fmt.Errorf("key is not hash of chunk data")
}
return nil
}
err := mget(m, hs, f)
if err != nil {
t.Fatalf("testStore failed: %v", err)
}
}
func benchmarkStorePut(store ChunkStore, processors int, n int, chunksize int, b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
mputRandomKey(store, processors, n, chunksize)
}
}
func benchmarkStoreGet(store ChunkStore, processors int, n int, chunksize int, b *testing.B) {
hs := mputRandomKey(store, processors, n, chunksize)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
err := mget(store, hs, nil)
if err != nil {
b.Fatalf("mget failed: %v", err)
}
}
close(chunkC)
<-quit
}

52
swarm/storage/dbapi.go Normal file
View file

@ -0,0 +1,52 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package storage
// wrapper of db-s to provide mockable custom local chunk store access to syncer
type DBAPI struct {
db *DbStore
loc *LocalStore
}
func NewDBAPI(loc *LocalStore) *DBAPI {
return &DBAPI{loc.DbStore.(*DbStore), loc}
}
// to obtain the chunks from key or request db entry only
func (self *DBAPI) Get(key Key) (*Chunk, error) {
return self.loc.Get(key)
}
// current storage counter of chunk db
func (self *DBAPI) CurrentBucketStorageIndex(po uint8) uint64 {
return self.db.CurrentBucketStorageIndex(po)
}
// iteration storage counter and proximity order
func (self *DBAPI) Iterator(from uint64, to uint64, po uint8, f func(Key, uint64) bool) error {
return self.db.SyncIterator(from, to, po, f)
}
// to obtain the chunks from key or request db entry only
func (self *DBAPI) GetOrCreateRequest(key Key) (*Chunk, bool) {
return self.loc.GetOrCreateRequest(key)
}
// to obtain the chunks from key or request db entry only
func (self *DBAPI) Put(chunk *Chunk) {
self.loc.Put(chunk)
}

442
swarm/storage/dbstore.go
View file

@ -36,7 +36,7 @@ import (
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/swarm/storage/mock"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/iterator"
"github.com/syndtr/goleveldb/leveldb/opt"
)
const (
@ -52,10 +52,13 @@ const (
)
var (
keyAccessCnt = []byte{2}
keyEntryCnt = []byte{3}
keyDataIdx = []byte{4}
keyGCPos = []byte{5}
keyOldData = byte(1)
keyAccessCnt = []byte{2}
keyEntryCnt = []byte{3}
keyDataIdx = []byte{4}
keyGCPos = []byte{5}
keyData = byte(6)
keyDistanceCnt = byte(7)
)
type gcItem struct {
@ -69,13 +72,19 @@ type DbStore struct {
// this should be stored in db, accessed transactionally
entryCnt, accessCnt, dataIdx, capacity uint64
bucketCnt []uint64
gcPos, gcStartPos []byte
gcArray []*gcItem
hashfunc SwarmHasher
po func(Key) uint8
lock sync.Mutex
batchC chan bool
batchesC chan struct{}
batch *leveldb.Batch
lock sync.RWMutex
trusted bool // if hash integity check is to be performed (for testing only)
// Functions encodeDataFunc is used to bypass
// the default functionality of DbStore with
@ -87,43 +96,63 @@ type DbStore struct {
getDataFunc func(key Key) (data []byte, err error)
}
func NewDbStore(path string, hash SwarmHasher, capacity uint64, radius int) (s *DbStore, err error) {
// TODO: Instead of passing the distance function, just pass the address from which distances are calculated
// to avoid the appearance of a pluggable distance metric and opportunities of bugs associated with providing
// a function different from the one that is actually used.
func NewDbStore(path string, hash SwarmHasher, capacity uint64, po func(Key) uint8) (s *DbStore, err error) {
s = new(DbStore)
s.hashfunc = hash
s.batchC = make(chan bool)
s.batchesC = make(chan struct{}, 1)
go s.writeBatches()
s.batch = new(leveldb.Batch)
// associate encodeData with default functionality
s.encodeDataFunc = encodeData
s.db, err = NewLDBDatabase(path)
if err != nil {
return
return nil, err
}
s.po = po
s.setCapacity(capacity)
s.gcStartPos = make([]byte, 1)
s.gcStartPos[0] = kpIndex
s.gcArray = make([]*gcItem, gcArraySize)
s.bucketCnt = make([]uint64, 0x100)
for i := 0; i < 0x100; i++ {
k := make([]byte, 2)
k[0] = keyDistanceCnt
k[1] = uint8(i)
cnt, _ := s.db.Get(k)
s.bucketCnt[i] = BytesToU64(cnt)
s.bucketCnt[i]++
}
data, _ := s.db.Get(keyEntryCnt)
s.entryCnt = BytesToU64(data)
s.entryCnt++
data, _ = s.db.Get(keyAccessCnt)
s.accessCnt = BytesToU64(data)
s.accessCnt++
data, _ = s.db.Get(keyDataIdx)
s.dataIdx = BytesToU64(data)
s.dataIdx++
s.gcPos, _ = s.db.Get(keyGCPos)
if s.gcPos == nil {
s.gcPos = s.gcStartPos
}
return
return s, nil
}
// NewMockDbStore creates a new instance of DbStore with
// mockStore set to a provided value. If mockStore argument is nil,
// this function behaves exactly as NewDbStore.
func NewMockDbStore(path string, hash SwarmHasher, capacity uint64, radius int, mockStore *mock.NodeStore) (s *DbStore, err error) {
s, err = NewDbStore(path, hash, capacity, radius)
func NewMockDbStore(path string, hash SwarmHasher, capacity uint64, po func(Key) uint8, mockStore *mock.NodeStore) (s *DbStore, err error) {
s, err = NewDbStore(path, hash, capacity, po)
if err != nil {
return nil, err
}
@ -144,12 +173,14 @@ func BytesToU64(data []byte) uint64 {
if len(data) < 8 {
return 0
}
return binary.LittleEndian.Uint64(data)
//return binary.LittleEndian.Uint64(data)
return binary.BigEndian.Uint64(data)
}
func U64ToBytes(val uint64) []byte {
data := make([]byte, 8)
binary.LittleEndian.PutUint64(data, val)
//binary.LittleEndian.PutUint64(data, val)
binary.BigEndian.PutUint64(data, val)
return data
}
@ -162,38 +193,53 @@ func (s *DbStore) updateIndexAccess(index *dpaDBIndex) {
}
func getIndexKey(hash Key) []byte {
HashSize := len(hash)
key := make([]byte, HashSize+1)
hashSize := len(hash)
key := make([]byte, hashSize+1)
key[0] = 0
copy(key[1:], hash[:])
return key
}
func getDataKey(idx uint64) []byte {
func getOldDataKey(idx uint64) []byte {
key := make([]byte, 9)
key[0] = 1
key[0] = keyOldData
binary.BigEndian.PutUint64(key[1:9], idx)
return key
}
func getDataKey(idx uint64, po uint8) []byte {
key := make([]byte, 10)
key[0] = keyData
key[1] = po
binary.BigEndian.PutUint64(key[2:], idx)
return key
}
func encodeIndex(index *dpaDBIndex) []byte {
data, _ := rlp.EncodeToBytes(index)
return data
}
func encodeData(chunk *Chunk) []byte {
return chunk.SData
return append(chunk.Key[:], chunk.SData...)
}
func decodeIndex(data []byte, index *dpaDBIndex) {
func decodeIndex(data []byte, index *dpaDBIndex) error {
dec := rlp.NewStream(bytes.NewReader(data), 0)
dec.Decode(index)
return dec.Decode(index)
}
func decodeData(data []byte, chunk *Chunk) {
chunk.SData = data[32:]
chunk.Size = int64(binary.BigEndian.Uint64(data[32:40]))
}
func decodeOldData(data []byte, chunk *Chunk) {
chunk.SData = data
chunk.Size = int64(binary.LittleEndian.Uint64(data[0:8]))
chunk.Size = int64(binary.BigEndian.Uint64(data[0:8]))
}
func gcListPartition(list []*gcItem, left int, right int, pivotIndex int) int {
@ -279,17 +325,13 @@ func (s *DbStore) collectGarbage(ratio float32) {
cutidx := gcListSelect(s.gcArray, 0, gcnt-1, int(float32(gcnt)*ratio))
cutval := s.gcArray[cutidx].value
// fmt.Print(gcnt, " ", s.entryCnt, " ")
// actual gc
for i := 0; i < gcnt; i++ {
if s.gcArray[i].value <= cutval {
s.delete(s.gcArray[i].idx, s.gcArray[i].idxKey)
s.delete(s.gcArray[i].idx, s.gcArray[i].idxKey, s.po(Key(s.gcPos[1:])))
}
}
// fmt.Println(s.entryCnt)
s.db.Put(keyGCPos, s.gcPos)
}
@ -311,14 +353,16 @@ func (s *DbStore) Export(out io.Writer) (int64, error) {
var index dpaDBIndex
decodeIndex(it.Value(), &index)
data, err := s.db.Get(getDataKey(index.Idx))
hash := key[1:]
data, err := s.db.Get(getDataKey(index.Idx, s.po(hash)))
if err != nil {
log.Warn(fmt.Sprintf("Chunk %x found but could not be accessed: %v", key[:], err))
continue
}
hdr := &tar.Header{
Name: hex.EncodeToString(key[1:]),
Name: hex.EncodeToString(hash),
Mode: 0644,
Size: int64(len(data)),
}
@ -334,12 +378,12 @@ func (s *DbStore) Export(out io.Writer) (int64, error) {
return count, nil
}
// Import reads chunks into the store from a tar archive, returning the number
// of chunks read.
func (s *DbStore) Import(in io.Reader) (int64, error) {
tr := tar.NewReader(in)
var count int64
var wg sync.WaitGroup
for {
hdr, err := tr.Next()
if err == io.EOF {
@ -363,11 +407,17 @@ func (s *DbStore) Import(in io.Reader) (int64, error) {
if err != nil {
return count, err
}
s.Put(&Chunk{Key: key, SData: data})
chunk := NewChunk(key, nil)
chunk.SData = data
s.Put(chunk)
wg.Add(1)
go func() {
defer wg.Done()
<-chunk.dbStored
}()
count++
}
wg.Wait()
return count, nil
}
@ -385,21 +435,23 @@ func (s *DbStore) Cleanup() {
}
total++
var index dpaDBIndex
decodeIndex(it.Value(), &index)
data, err := s.db.Get(getDataKey(index.Idx))
err := decodeIndex(it.Value(), &index)
if err != nil {
it.Next()
continue
}
data, err := s.db.Get(getDataKey(index.Idx, s.po(Key(key[1:]))))
if err != nil {
log.Warn(fmt.Sprintf("Chunk %x found but could not be accessed: %v", key[:], err))
s.delete(index.Idx, getIndexKey(key[1:]))
s.delete(index.Idx, getIndexKey(key[1:]), s.po(Key(key[1:])))
errorsFound++
} else {
hasher := s.hashfunc()
hasher.Write(data)
hasher.Write(data[32:])
hash := hasher.Sum(nil)
if !bytes.Equal(hash, key[1:]) {
log.Warn(fmt.Sprintf("Found invalid chunk. Hash mismatch. hash=%x, key=%x", hash, key[:]))
s.delete(index.Idx, getIndexKey(key[1:]))
errorsFound++
s.delete(index.Idx, getIndexKey(key[1:]), s.po(Key(key[1:])))
}
}
it.Next()
@ -408,65 +460,165 @@ func (s *DbStore) Cleanup() {
log.Warn(fmt.Sprintf("Found %v errors out of %v entries", errorsFound, total))
}
func (s *DbStore) delete(idx uint64, idxKey []byte) {
func (s *DbStore) ReIndex() {
//Iterates over the database and checks that there are no faulty chunks
it := s.db.NewIterator()
startPosition := []byte{keyOldData}
it.Seek(startPosition)
var key []byte
var errorsFound, total int
for it.Valid() {
key = it.Key()
if (key == nil) || (key[0] != keyOldData) {
break
}
data := it.Value()
hasher := s.hashfunc()
hasher.Write(data)
hash := hasher.Sum(nil)
newKey := make([]byte, 10)
oldCntKey := make([]byte, 2)
newCntKey := make([]byte, 2)
oldCntKey[0] = keyDistanceCnt
newCntKey[0] = keyDistanceCnt
key[0] = keyData
key[1] = s.po(Key(key[1:]))
oldCntKey[1] = key[1]
newCntKey[1] = s.po(Key(newKey[1:]))
copy(newKey[2:], key[1:])
newValue := append(hash, data...)
batch := new(leveldb.Batch)
batch.Delete(key)
s.bucketCnt[oldCntKey[1]]--
batch.Put(oldCntKey, U64ToBytes(s.bucketCnt[oldCntKey[1]]))
batch.Put(newKey, newValue)
s.bucketCnt[newCntKey[1]]++
batch.Put(newCntKey, U64ToBytes(s.bucketCnt[newCntKey[1]]))
s.db.Write(batch)
it.Next()
}
it.Release()
log.Warn(fmt.Sprintf("Found %v errors out of %v entries", errorsFound, total))
}
func (s *DbStore) delete(idx uint64, idxKey []byte, po uint8) {
batch := new(leveldb.Batch)
batch.Delete(idxKey)
batch.Delete(getDataKey(idx))
batch.Delete(getDataKey(idx, po))
s.entryCnt--
s.bucketCnt[po]--
cntKey := make([]byte, 2)
cntKey[0] = keyDistanceCnt
cntKey[1] = po
batch.Put(keyEntryCnt, U64ToBytes(s.entryCnt))
batch.Put(cntKey, U64ToBytes(s.bucketCnt[po]))
s.db.Write(batch)
}
func (s *DbStore) Counter() uint64 {
func (s *DbStore) CurrentBucketStorageIndex(po uint8) uint64 {
s.lock.RLock()
defer s.lock.RUnlock()
return s.bucketCnt[po]
}
func (s *DbStore) Size() uint64 {
s.lock.Lock()
defer s.lock.Unlock()
return s.entryCnt
}
func (s *DbStore) CurrentStorageIndex() uint64 {
s.lock.RLock()
defer s.lock.RUnlock()
return s.dataIdx
}
func (s *DbStore) Put(chunk *Chunk) {
s.lock.Lock()
defer s.lock.Unlock()
ikey := getIndexKey(chunk.Key)
var index dpaDBIndex
if s.tryAccessIdx(ikey, &index) {
if chunk.dbStored != nil {
po := s.po(chunk.Key)
s.lock.Lock()
defer s.lock.Unlock()
idata, err := s.db.Get(ikey)
if err != nil {
s.doPut(chunk, ikey, &index, po)
batchC := s.batchC
go func() {
<-batchC
close(chunk.dbStored)
}
log.Trace(fmt.Sprintf("Storing to DB: chunk already exists, only update access"))
return // already exists, only update access
}
data := s.encodeDataFunc(chunk)
//data := ethutil.Encode([]interface{}{entry})
if s.entryCnt >= s.capacity {
s.collectGarbage(gcArrayFreeRatio)
}
batch := new(leveldb.Batch)
batch.Put(getDataKey(s.dataIdx), data)
index.Idx = s.dataIdx
s.updateIndexAccess(&index)
idata := encodeIndex(&index)
batch.Put(ikey, idata)
batch.Put(keyEntryCnt, U64ToBytes(s.entryCnt))
s.entryCnt++
batch.Put(keyDataIdx, U64ToBytes(s.dataIdx))
s.dataIdx++
batch.Put(keyAccessCnt, U64ToBytes(s.accessCnt))
s.accessCnt++
s.db.Write(batch)
if chunk.dbStored != nil {
}()
} else {
log.Trace(fmt.Sprintf("DbStore: chunk already exists, only update access"))
decodeIndex(idata, &index)
close(chunk.dbStored)
}
log.Trace(fmt.Sprintf("DbStore.Put: %v. db storage counter: %v ", chunk.Key.Log(), s.dataIdx))
index.Access = s.accessCnt
s.accessCnt++
idata = encodeIndex(&index)
s.batch.Put(ikey, idata)
select {
case s.batchesC <- struct{}{}:
default:
}
}
// force putting into db, does not check access index
func (s *DbStore) doPut(chunk *Chunk, ikey []byte, index *dpaDBIndex, po uint8) {
data := s.encodeDataFunc(chunk)
s.batch.Put(getDataKey(s.dataIdx, po), data)
index.Idx = s.dataIdx
s.bucketCnt[po] = s.dataIdx
s.entryCnt++
s.dataIdx++
cntKey := make([]byte, 2)
cntKey[0] = keyDistanceCnt
cntKey[1] = po
s.batch.Put(cntKey, U64ToBytes(s.bucketCnt[po]))
}
func (s *DbStore) writeBatches() {
for range s.batchesC {
s.lock.Lock()
b := s.batch
e := s.entryCnt
d := s.dataIdx
a := s.accessCnt
c := s.batchC
s.batchC = make(chan bool)
s.batch = new(leveldb.Batch)
s.lock.Unlock()
err := s.writeBatch(b, e, d, a)
// TODO: set this error on the batch, then tell the chunk
if err != nil {
log.Error(fmt.Sprintf("DbStore: spawn batch write (%d chunks): %v", b.Len(), err))
}
close(c)
if e >= s.capacity {
log.Trace(fmt.Sprintf("DbStore: collecting garbage...(%d chunks)", e))
s.collectGarbage(gcArrayFreeRatio)
}
}
log.Trace(fmt.Sprintf("DbStore: quit batch write loop"))
}
// must be called non concurrently
func (s *DbStore) writeBatch(b *leveldb.Batch, entryCnt, dataIdx, accessCnt uint64) error {
b.Put(keyEntryCnt, U64ToBytes(entryCnt))
b.Put(keyDataIdx, U64ToBytes(dataIdx))
b.Put(keyAccessCnt, U64ToBytes(accessCnt))
l := b.Len()
if err := s.db.Write(b); err != nil {
return fmt.Errorf("unable to write batch: %v", err)
}
log.Trace(fmt.Sprintf("DbStore: batch write (%d chunks) complete", l))
return nil
}
// newMockEncodeDataFunc returns a function that stores the chunk data
@ -475,10 +627,10 @@ func (s *DbStore) Put(chunk *Chunk) {
// not need to store the data, but still need to create the index.
func newMockEncodeDataFunc(mockStore *mock.NodeStore) func(chunk *Chunk) []byte {
return func(chunk *Chunk) []byte {
if err := mockStore.Put(chunk.Key, chunk.SData); err != nil {
if err := mockStore.Put(chunk.Key, encodeData(chunk)); err != nil {
log.Error(fmt.Sprintf("%T: Chunk %v put: %v", mockStore, chunk.Key.Log(), err))
}
return nil
return chunk.Key[:]
}
}
@ -489,27 +641,24 @@ func (s *DbStore) tryAccessIdx(ikey []byte, index *dpaDBIndex) bool {
return false
}
decodeIndex(idata, index)
batch := new(leveldb.Batch)
batch.Put(keyAccessCnt, U64ToBytes(s.accessCnt))
s.batch.Put(keyAccessCnt, U64ToBytes(s.accessCnt))
s.accessCnt++
s.updateIndexAccess(index)
index.Access = s.accessCnt
idata = encodeIndex(index)
batch.Put(ikey, idata)
s.db.Write(batch)
s.batch.Put(ikey, idata)
return true
}
func (s *DbStore) Get(key Key) (chunk *Chunk, err error) {
s.lock.Lock()
defer s.lock.Unlock()
return s.get(key)
}
var index dpaDBIndex
func (s *DbStore) get(key Key) (chunk *Chunk, err error) {
var indx dpaDBIndex
if s.tryAccessIdx(getIndexKey(key), &index) {
if s.tryAccessIdx(getIndexKey(key), &indx) {
var data []byte
if s.getDataFunc != nil {
// if getDataFunc is defined, use it to retrieve the chunk data
@ -519,31 +668,34 @@ func (s *DbStore) Get(key Key) (chunk *Chunk, err error) {
}
} else {
// default DbStore functionality to retrieve chunk data
data, err = s.db.Get(getDataKey(index.Idx))
proximity := s.po(key)
datakey := getDataKey(indx.Idx, proximity)
data, err = s.db.Get(datakey)
log.Trace(fmt.Sprintf("DBStore: Chunk %v indexkey %v datakey %x proximity %d", key.Log(), indx.Idx, datakey, proximity))
if err != nil {
log.Trace(fmt.Sprintf("DBStore: Chunk %v found but could not be accessed: %v", key.Log(), err))
s.delete(index.Idx, getIndexKey(key))
s.delete(indx.Idx, getIndexKey(key), s.po(key))
return
}
}
if s.hashfunc != nil {
if !s.trusted {
data_mod := data[32:]
hasher := s.hashfunc()
hasher.Write(data)
hasher.Write(data_mod)
hash := hasher.Sum(nil)
if !bytes.Equal(hash, key) {
s.delete(index.Idx, getIndexKey(key))
log.Warn("Invalid Chunk in Database. Please repair with command: 'swarm cleandb'")
log.Error(fmt.Sprintf("Apparent key/hash mismatch. Hash %x, key %v", hash, key[:]))
s.delete(indx.Idx, getIndexKey(key), s.po(key))
log.Error("Invalid Chunk in Database. Please repair with command: 'swarm cleandb'")
}
}
chunk = &Chunk{
Key: key,
}
chunk = NewChunk(key, nil)
decodeData(data, chunk)
} else {
err = notFound
err = ErrNotFound
}
return
@ -556,8 +708,8 @@ func newMockGetDataFunc(mockStore *mock.NodeStore) func(key Key) (data []byte, e
return func(key Key) (data []byte, err error) {
data, err = mockStore.Get(key)
if err == mock.ErrNotFound {
// preserve notFound error
err = notFound
// preserve ErrNotFound error
err = ErrNotFound
}
return data, err
}
@ -598,62 +750,36 @@ func (s *DbStore) Close() {
s.db.Close()
}
// describes a section of the DbStore representing the unsynced
// domain relevant to a peer
// Start - Stop designate a continuous area Keys in an address space
// typically the addresses closer to us than to the peer but not closer
// another closer peer in between
// From - To designates a time interval typically from the last disconnect
// till the latest connection (real time traffic is relayed)
type DbSyncState struct {
Start, Stop Key
First, Last uint64
}
// SyncIterator(start, stop, po, f) calls f on each hash of a bin po from start to stop
func (s *DbStore) SyncIterator(since uint64, until uint64, po uint8, f func(Key, uint64) bool) error {
sincekey := getDataKey(since, po)
untilkey := getDataKey(until, po)
it := s.db.NewIterator()
defer it.Release()
// implements the syncer iterator interface
// iterates by storage index (~ time of storage = first entry to db)
type dbSyncIterator struct {
it iterator.Iterator
DbSyncState
}
// initialises a sync iterator from a syncToken (passed in with the handshake)
func (self *DbStore) NewSyncIterator(state DbSyncState) (si *dbSyncIterator, err error) {
if state.First > state.Last {
return nil, fmt.Errorf("no entries found")
}
si = &dbSyncIterator{
it: self.db.NewIterator(),
DbSyncState: state,
}
si.it.Seek(getIndexKey(state.Start))
return si, nil
}
// walk the area from Start to Stop and returns items within time interval
// First to Last
func (self *dbSyncIterator) Next() (key Key) {
for self.it.Valid() {
dbkey := self.it.Key()
if dbkey[0] != 0 {
for ok := it.Seek(sincekey); ok; ok = it.Next() {
dbkey := it.Key()
if dbkey[0] != keyData || dbkey[1] != po || bytes.Compare(untilkey, dbkey) < 0 {
break
}
key = Key(make([]byte, len(dbkey)-1))
copy(key[:], dbkey[1:])
if bytes.Compare(key[:], self.Start) <= 0 {
self.it.Next()
continue
}
if bytes.Compare(key[:], self.Stop) > 0 {
key := make([]byte, 32)
val := it.Value()
copy(key, val[:32])
if !f(Key(key), binary.BigEndian.Uint64(dbkey[2:])) {
break
}
var index dpaDBIndex
decodeIndex(self.it.Value(), &index)
self.it.Next()
if (index.Idx >= self.First) && (index.Idx < self.Last) {
return
}
}
self.it.Release()
return nil
return it.Error()
}
func databaseExists(path string) bool {
o := &opt.Options{
ErrorIfMissing: true,
}
tdb, err := leveldb.OpenFile(path, o)
if err != nil {
return false
}
defer tdb.Close()
return true
}

440
swarm/storage/dbstore_test.go
View file

@ -18,250 +18,256 @@ package storage
import (
"bytes"
"fmt"
"io/ioutil"
"strings"
"os"
"sync"
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/swarm/storage/mock"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
)
func initDbStore(t *testing.T) *DbStore {
type testDbStore struct {
*DbStore
dir string
}
func newTestDbStore(mock bool) (*testDbStore, error) {
dir, err := ioutil.TempDir("", "bzz-storage-test")
if err != nil {
t.Fatal(err)
return nil, err
}
m, err := NewDbStore(dir, MakeHashFunc(SHA3Hash), defaultDbCapacity, defaultRadius)
var db *DbStore
if mock {
globalStore := mem.NewGlobalStore()
addr := common.HexToAddress("0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed")
mockStore := globalStore.NewNodeStore(addr)
db, err = NewMockDbStore(dir, MakeHashFunc(SHA3Hash), defaultDbCapacity, testPoFunc, mockStore)
} else {
db, err = NewDbStore(dir, MakeHashFunc(SHA3Hash), defaultDbCapacity, testPoFunc)
}
return &testDbStore{db, dir}, err
}
func testPoFunc(k Key) (ret uint8) {
basekey := make([]byte, 32)
return uint8(Proximity(basekey[:], k[:]))
}
func (db *testDbStore) close() {
db.Close()
err := os.RemoveAll(db.dir)
if err != nil {
t.Fatal("can't create store:", err)
panic(err)
}
return m
}
func testDbStore(l int64, branches int64, t *testing.T) {
m := initDbStore(t)
defer m.Close()
testStore(m, l, branches, t)
func testDbStoreRandom(n int, processors int, chunksize int, mock bool, t *testing.T) {
db, err := newTestDbStore(mock)
if err != nil {
t.Fatalf("init dbStore failed: %v", err)
}
defer db.close()
db.trusted = true
testStoreRandom(db, processors, n, chunksize, t)
}
func TestDbStore128_0x1000000(t *testing.T) {
testDbStore(0x1000000, 128, t)
func testDbStoreCorrect(n int, processors int, chunksize int, mock bool, t *testing.T) {
db, err := newTestDbStore(mock)
if err != nil {
t.Fatalf("init dbStore failed: %v", err)
}
defer db.close()
testStoreCorrect(db, processors, n, chunksize, t)
}
func TestDbStore128_10000_(t *testing.T) {
testDbStore(10000, 128, t)
func TestDbStoreRandom_1(t *testing.T) {
testDbStoreRandom(1, 1, 0, false, t)
}
func TestDbStore128_1000_(t *testing.T) {
testDbStore(1000, 128, t)
func TestDbStoreCorrect_1(t *testing.T) {
testDbStoreCorrect(1, 1, 4096, false, t)
}
func TestDbStore128_100_(t *testing.T) {
testDbStore(100, 128, t)
func TestDbStoreRandom_1_5k(t *testing.T) {
testDbStoreRandom(8, 5000, 0, false, t)
}
func TestDbStore2_100_(t *testing.T) {
testDbStore(100, 2, t)
func TestDbStoreRandom_8_5k(t *testing.T) {
testDbStoreRandom(8, 5000, 0, false, t)
}
func TestDbStoreCorrect_1_5k(t *testing.T) {
testDbStoreCorrect(1, 5000, 4096, false, t)
}
func TestDbStoreCorrect_8_5k(t *testing.T) {
testDbStoreCorrect(8, 5000, 4096, false, t)
}
func TestMockDbStoreRandom_1(t *testing.T) {
testDbStoreRandom(1, 1, 0, true, t)
}
func TestMockDbStoreCorrect_1(t *testing.T) {
testDbStoreCorrect(1, 1, 4096, true, t)
}
func TestMockDbStoreRandom_1_5k(t *testing.T) {
testDbStoreRandom(8, 5000, 0, true, t)
}
func TestMockDbStoreRandom_8_5k(t *testing.T) {
testDbStoreRandom(8, 5000, 0, true, t)
}
func TestMockDbStoreCorrect_1_5k(t *testing.T) {
testDbStoreCorrect(1, 5000, 4096, true, t)
}
func TestMockDbStoreCorrect_8_5k(t *testing.T) {
testDbStoreCorrect(8, 5000, 4096, true, t)
}
func testDbStoreNotFound(t *testing.T, mock bool) {
db, err := newTestDbStore(mock)
if err != nil {
t.Fatalf("init dbStore failed: %v", err)
}
defer db.close()
_, err = db.Get(ZeroKey)
if err != ErrNotFound {
t.Errorf("Expected ErrNotFound, got %v", err)
}
}
func TestDbStoreNotFound(t *testing.T) {
m := initDbStore(t)
defer m.Close()
_, err := m.Get(ZeroKey)
if err != notFound {
t.Errorf("Expected notFound, got %v", err)
}
testDbStoreNotFound(t, false)
}
func TestDbStoreSyncIterator(t *testing.T) {
m := initDbStore(t)
defer m.Close()
keys := []Key{
Key(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000")),
Key(common.Hex2Bytes("4000000000000000000000000000000000000000000000000000000000000000")),
Key(common.Hex2Bytes("5000000000000000000000000000000000000000000000000000000000000000")),
Key(common.Hex2Bytes("3000000000000000000000000000000000000000000000000000000000000000")),
Key(common.Hex2Bytes("2000000000000000000000000000000000000000000000000000000000000000")),
Key(common.Hex2Bytes("1000000000000000000000000000000000000000000000000000000000000000")),
}
for _, key := range keys {
m.Put(NewChunk(key, nil))
}
it, err := m.NewSyncIterator(DbSyncState{
Start: Key(common.Hex2Bytes("1000000000000000000000000000000000000000000000000000000000000000")),
Stop: Key(common.Hex2Bytes("4000000000000000000000000000000000000000000000000000000000000000")),
First: 2,
Last: 4,
})
if err != nil {
t.Fatalf("unexpected error creating NewSyncIterator")
}
var chunk Key
var res []Key
for {
chunk = it.Next()
if chunk == nil {
break
}
res = append(res, chunk)
}
if len(res) != 1 {
t.Fatalf("Expected 1 chunk, got %v: %v", len(res), res)
}
if !bytes.Equal(res[0][:], keys[3]) {
t.Fatalf("Expected %v chunk, got %v", keys[3], res[0])
}
if err != nil {
t.Fatalf("unexpected error creating NewSyncIterator")
}
it, err = m.NewSyncIterator(DbSyncState{
Start: Key(common.Hex2Bytes("1000000000000000000000000000000000000000000000000000000000000000")),
Stop: Key(common.Hex2Bytes("5000000000000000000000000000000000000000000000000000000000000000")),
First: 2,
Last: 4,
})
res = nil
for {
chunk = it.Next()
if chunk == nil {
break
}
res = append(res, chunk)
}
if len(res) != 2 {
t.Fatalf("Expected 2 chunk, got %v: %v", len(res), res)
}
if !bytes.Equal(res[0][:], keys[3]) {
t.Fatalf("Expected %v chunk, got %v", keys[3], res[0])
}
if !bytes.Equal(res[1][:], keys[2]) {
t.Fatalf("Expected %v chunk, got %v", keys[2], res[1])
}
if err != nil {
t.Fatalf("unexpected error creating NewSyncIterator")
}
it, _ = m.NewSyncIterator(DbSyncState{
Start: Key(common.Hex2Bytes("1000000000000000000000000000000000000000000000000000000000000000")),
Stop: Key(common.Hex2Bytes("4000000000000000000000000000000000000000000000000000000000000000")),
First: 2,
Last: 5,
})
res = nil
for {
chunk = it.Next()
if chunk == nil {
break
}
res = append(res, chunk)
}
if len(res) != 2 {
t.Fatalf("Expected 2 chunk, got %v", len(res))
}
if !bytes.Equal(res[0][:], keys[4]) {
t.Fatalf("Expected %v chunk, got %v", keys[4], res[0])
}
if !bytes.Equal(res[1][:], keys[3]) {
t.Fatalf("Expected %v chunk, got %v", keys[3], res[1])
}
it, _ = m.NewSyncIterator(DbSyncState{
Start: Key(common.Hex2Bytes("2000000000000000000000000000000000000000000000000000000000000000")),
Stop: Key(common.Hex2Bytes("4000000000000000000000000000000000000000000000000000000000000000")),
First: 2,
Last: 5,
})
res = nil
for {
chunk = it.Next()
if chunk == nil {
break
}
res = append(res, chunk)
}
if len(res) != 1 {
t.Fatalf("Expected 1 chunk, got %v", len(res))
}
if !bytes.Equal(res[0][:], keys[3]) {
t.Fatalf("Expected %v chunk, got %v", keys[3], res[0])
}
}
func initMockDbStore(t *testing.T, mockStore *mock.NodeStore) *DbStore {
dir, err := ioutil.TempDir("", "bzz-storage-test-mock")
if err != nil {
t.Fatal(err)
}
m, err := NewMockDbStore(dir, MakeHashFunc(SHA3Hash), defaultDbCapacity, defaultRadius, mockStore)
if err != nil {
t.Fatal("can't create store:", err)
}
return m
}
// testMockDbStore runs the same tests as testDbStore but with mock store configured.
// It also verifies if mock global store is storing the chunk data.
func testMockDbStore(l int64, branches int64, t *testing.T) {
globalStore := mem.NewGlobalStore()
addr := common.HexToAddress("0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed")
mockStore := globalStore.NewNodeStore(addr)
m := initMockDbStore(t, mockStore)
defer m.Close()
key := Key(common.Hex2Bytes("fed1911825fc6a02ebfd19ab218a20455d8d7d275f8bf4d8244eb04364fae6f7"))
data := common.Hex2BytesFixed(strings.Repeat("1234567890abcdf", 10), 4096)
m.Put(&Chunk{
Key: key,
SData: data,
})
_, err := globalStore.Get(addr, key)
if err != nil {
t.Errorf("unexpected error getting the data from global mock store: %v", err)
}
if !globalStore.HasKey(addr, key) {
t.Error("key not found in global store")
}
testStore(m, l, branches, t)
}
func TestMockDbStore128_0x1000000(t *testing.T) {
testMockDbStore(0x1000000, 128, t)
}
func TestMockDbStore128_10000_(t *testing.T) {
testMockDbStore(10000, 128, t)
}
func TestMockDbStore128_1000_(t *testing.T) {
testMockDbStore(1000, 128, t)
}
func TestMockDbStore128_100_(t *testing.T) {
testMockDbStore(100, 128, t)
}
func TestMockDbStore2_100_(t *testing.T) {
testMockDbStore(100, 2, t)
}
func TestMockDbStoreNotFound(t *testing.T) {
globalStore := mem.NewGlobalStore()
mockStore := globalStore.NewNodeStore(common.HexToAddress("0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed"))
m := initMockDbStore(t, mockStore)
defer m.Close()
_, err := m.Get(ZeroKey)
if err != notFound {
t.Errorf("Expected notFound, got %v", err)
}
testDbStoreNotFound(t, true)
}
func testIterator(t *testing.T, mock bool) {
var chunkcount int = 32
var i int
var poc uint
chunkkeys := NewKeyCollection(chunkcount)
chunkkeys_results := NewKeyCollection(chunkcount)
var chunks []*Chunk
for i := 0; i < chunkcount; i++ {
chunks = append(chunks, NewChunk(nil, nil))
}
db, err := newTestDbStore(mock)
if err != nil {
t.Fatalf("init dbStore failed: %v", err)
}
defer db.close()
FakeChunk(getDefaultChunkSize(), chunkcount, chunks)
wg := &sync.WaitGroup{}
wg.Add(len(chunks))
for i = 0; i < len(chunks); i++ {
db.Put(chunks[i])
chunkkeys[i] = chunks[i].Key
j := i
go func() {
defer wg.Done()
<-chunks[j].dbStored
}()
}
//testSplit(m, l, 128, chunkkeys, t)
for i = 0; i < len(chunkkeys); i++ {
log.Trace(fmt.Sprintf("Chunk array pos %d/%d: '%v'", i, chunkcount, chunkkeys[i]))
}
wg.Wait()
i = 0
for poc = 0; poc <= 255; poc++ {
err := db.SyncIterator(0, uint64(chunkkeys.Len()), uint8(poc), func(k Key, n uint64) bool {
log.Trace(fmt.Sprintf("Got key %v number %d poc %d", k, n, uint8(poc)))
chunkkeys_results[n-1] = k
i++
return true
})
if err != nil {
t.Fatalf("Iterator call failed: %v", err)
}
}
for i = 0; i < chunkcount; i++ {
if !bytes.Equal(chunkkeys[i], chunkkeys_results[i]) {
t.Fatalf("Chunk put #%d key '%v' does not match iterator's key '%v'", i, chunkkeys[i], chunkkeys_results[i])
}
}
}
func TestIterator(t *testing.T) {
testIterator(t, false)
}
func TestMockIterator(t *testing.T) {
testIterator(t, true)
}
func benchmarkDbStorePut(n int, processors int, chunksize int, mock bool, b *testing.B) {
db, err := newTestDbStore(mock)
if err != nil {
b.Fatalf("init dbStore failed: %v", err)
}
defer db.close()
db.trusted = true
benchmarkStorePut(db, processors, n, chunksize, b)
}
func benchmarkDbStoreGet(n int, processors int, chunksize int, mock bool, b *testing.B) {
db, err := newTestDbStore(mock)
if err != nil {
b.Fatalf("init dbStore failed: %v", err)
}
defer db.close()
db.trusted = true
benchmarkStoreGet(db, processors, n, chunksize, b)
}
func BenchmarkDbStorePut_1_5k(b *testing.B) {
benchmarkDbStorePut(5000, 1, 4096, false, b)
}
func BenchmarkDbStorePut_8_5k(b *testing.B) {
benchmarkDbStorePut(5000, 8, 4096, false, b)
}
func BenchmarkDbStoreGet_1_5k(b *testing.B) {
benchmarkDbStoreGet(5000, 1, 4096, false, b)
}
func BenchmarkDbStoreGet_8_5k(b *testing.B) {
benchmarkDbStoreGet(5000, 8, 4096, false, b)
}
func BenchmarkMockDbStorePut_1_5k(b *testing.B) {
benchmarkDbStorePut(5000, 1, 4096, true, b)
}
func BenchmarkMockDbStorePut_8_5k(b *testing.B) {
benchmarkDbStorePut(5000, 8, 4096, true, b)
}
func BenchmarkMockDbStoreGet_1_5k(b *testing.B) {
benchmarkDbStoreGet(5000, 1, 4096, true, b)
}
func BenchmarkMockDbStoreGet_8_5k(b *testing.B) {
benchmarkDbStoreGet(5000, 8, 4096, true, b)
}

88
swarm/storage/dpa.go
View file

@ -48,7 +48,10 @@ const (
)
var (
notFound = errors.New("not found")
ErrNotFound = errors.New("not found")
ErrFetching = errors.New("chunk still fetching")
// timeout interval before retrieval is timed out
searchTimeout = 3 * time.Second
)
type DPA struct {
@ -59,15 +62,16 @@ type DPA struct {
lock sync.Mutex
running bool
wg *sync.WaitGroup
quitC chan bool
}
// for testing locally
func NewLocalDPA(datadir string) (*DPA, error) {
func NewLocalDPA(datadir string, basekey []byte) (*DPA, error) {
hash := MakeHashFunc("SHA3")
dbStore, err := NewDbStore(datadir, hash, singletonSwarmDbCapacity, 0)
dbStore, err := NewDbStore(datadir, hash, singletonSwarmDbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) })
if err != nil {
return nil, err
}
@ -79,7 +83,7 @@ func NewLocalDPA(datadir string) (*DPA, error) {
}
func NewDPA(store ChunkStore, params *ChunkerParams) *DPA {
chunker := NewTreeChunker(params)
chunker := NewPyramidChunker(params)
return &DPA{
Chunker: chunker,
ChunkStore: store,
@ -91,13 +95,13 @@ func NewDPA(store ChunkStore, params *ChunkerParams) *DPA {
// Chunk retrieval blocks on netStore requests with a timeout so reader will
// report error if retrieval of chunks within requested range time out.
func (self *DPA) Retrieve(key Key) LazySectionReader {
return self.Chunker.Join(key, self.retrieveC)
return self.Chunker.Join(key, self.retrieveC, 0)
}
// Public API. Main entry point for document storage directly. Used by the
// FS-aware API and httpaccess
func (self *DPA) Store(data io.Reader, size int64, swg *sync.WaitGroup, wwg *sync.WaitGroup) (key Key, err error) {
return self.Chunker.Split(data, size, self.storeC, swg, wwg)
func (self *DPA) Store(data io.Reader, size int64) (key Key, wait func(), err error) {
return self.Chunker.Split(data, size, self.storeC)
}
func (self *DPA) Start() {
@ -135,11 +139,8 @@ func (self *DPA) retrieveLoop() {
func (self *DPA) retrieveWorker() {
for chunk := range self.retrieveC {
log.Trace(fmt.Sprintf("dpa: retrieve loop : chunk %v", chunk.Key.Log()))
storedChunk, err := self.Get(chunk.Key)
if err == notFound {
log.Trace(fmt.Sprintf("chunk %v not found", chunk.Key.Log()))
} else if err != nil {
if err != nil {
log.Trace(fmt.Sprintf("error retrieving chunk %v: %v", chunk.Key.Log(), err))
} else {
chunk.SData = storedChunk.SData
@ -165,14 +166,8 @@ func (self *DPA) storeLoop() {
}
func (self *DPA) storeWorker() {
for chunk := range self.storeC {
self.Put(chunk)
if chunk.wg != nil {
log.Trace(fmt.Sprintf("dpa: store processor %v", chunk.Key.Log()))
chunk.wg.Done()
}
select {
case <-self.quitC:
return
@ -180,62 +175,3 @@ func (self *DPA) storeWorker() {
}
}
}
// DpaChunkStore implements the ChunkStore interface,
// this chunk access layer assumed 2 chunk stores
// local storage eg. LocalStore and network storage eg., NetStore
// access by calling network is blocking with a timeout
type dpaChunkStore struct {
n int
localStore ChunkStore
netStore ChunkStore
}
func NewDpaChunkStore(localStore, netStore ChunkStore) *dpaChunkStore {
return &dpaChunkStore{0, localStore, netStore}
}
// Get is the entrypoint for local retrieve requests
// waits for response or times out
func (self *dpaChunkStore) Get(key Key) (chunk *Chunk, err error) {
chunk, err = self.netStore.Get(key)
// timeout := time.Now().Add(searchTimeout)
if chunk.SData != nil {
log.Trace(fmt.Sprintf("DPA.Get: %v found locally, %d bytes", key.Log(), len(chunk.SData)))
return
}
// TODO: use self.timer time.Timer and reset with defer disableTimer
timer := time.After(searchTimeout)
select {
case <-timer:
log.Trace(fmt.Sprintf("DPA.Get: %v request time out ", key.Log()))
err = notFound
case <-chunk.Req.C:
log.Trace(fmt.Sprintf("DPA.Get: %v retrieved, %d bytes (%p)", key.Log(), len(chunk.SData), chunk))
}
return
}
// Put is the entrypoint for local store requests coming from storeLoop
func (self *dpaChunkStore) Put(entry *Chunk) {
chunk, err := self.localStore.Get(entry.Key)
if err != nil {
log.Trace(fmt.Sprintf("DPA.Put: %v new chunk. call netStore.Put", entry.Key.Log()))
chunk = entry
} else if chunk.SData == nil {
log.Trace(fmt.Sprintf("DPA.Put: %v request entry found", entry.Key.Log()))
chunk.SData = entry.SData
chunk.Size = entry.Size
} else {
log.Trace(fmt.Sprintf("DPA.Put: %v chunk already known", entry.Key.Log()))
return
}
// from this point on the storage logic is the same with network storage requests
log.Trace(fmt.Sprintf("DPA.Put %v: %v", self.n, chunk.Key.Log()))
self.n++
self.netStore.Put(chunk)
}
// Close chunk store
func (self *dpaChunkStore) Close() {}

38
swarm/storage/dpa_test.go
View file

@ -21,19 +21,23 @@ import (
"io"
"io/ioutil"
"os"
"sync"
"testing"
)
const testDataSize = 0x1000000
func TestDPArandom(t *testing.T) {
dbStore := initDbStore(t)
dbStore.setCapacity(50000)
memStore := NewMemStore(dbStore, defaultCacheCapacity)
tdb, err := newTestDbStore(false)
if err != nil {
t.Fatalf("init dbStore failed: %v", err)
}
defer tdb.close()
db := tdb.DbStore
db.setCapacity(50000)
memStore := NewMemStore(db, defaultCacheCapacity)
localStore := &LocalStore{
memStore,
dbStore,
db,
}
chunker := NewTreeChunker(NewChunkerParams())
dpa := &DPA{
@ -45,12 +49,11 @@ func TestDPArandom(t *testing.T) {
defer os.RemoveAll("/tmp/bzz")
reader, slice := testDataReaderAndSlice(testDataSize)
wg := &sync.WaitGroup{}
key, err := dpa.Store(reader, testDataSize, wg, nil)
key, wait, err := dpa.Store(reader, testDataSize)
if err != nil {
t.Errorf("Store error: %v", err)
}
wg.Wait()
wait()
resultReader := dpa.Retrieve(key)
resultSlice := make([]byte, len(slice))
n, err := resultReader.ReadAt(resultSlice, 0)
@ -65,7 +68,7 @@ func TestDPArandom(t *testing.T) {
}
ioutil.WriteFile("/tmp/slice.bzz.16M", slice, 0666)
ioutil.WriteFile("/tmp/result.bzz.16M", resultSlice, 0666)
localStore.memStore = NewMemStore(dbStore, defaultCacheCapacity)
localStore.memStore = NewMemStore(db, defaultCacheCapacity)
resultReader = dpa.Retrieve(key)
for i := range resultSlice {
resultSlice[i] = 0
@ -83,13 +86,17 @@ func TestDPArandom(t *testing.T) {
}
func TestDPA_capacity(t *testing.T) {
dbStore := initDbStore(t)
memStore := NewMemStore(dbStore, defaultCacheCapacity)
tdb, err := newTestDbStore(false)
if err != nil {
t.Fatalf("init dbStore failed: %v", err)
}
defer tdb.close()
db := tdb.DbStore
memStore := NewMemStore(db, 0)
localStore := &LocalStore{
memStore,
dbStore,
db,
}
memStore.setCapacity(0)
chunker := NewTreeChunker(NewChunkerParams())
dpa := &DPA{
Chunker: chunker,
@ -97,12 +104,11 @@ func TestDPA_capacity(t *testing.T) {
}
dpa.Start()
reader, slice := testDataReaderAndSlice(testDataSize)
wg := &sync.WaitGroup{}
key, err := dpa.Store(reader, testDataSize, wg, nil)
key, wait, err := dpa.Store(reader, testDataSize)
if err != nil {
t.Errorf("Store error: %v", err)
}
wg.Wait()
wait()
resultReader := dpa.Retrieve(key)
resultSlice := make([]byte, len(slice))
n, err := resultReader.ReadAt(resultSlice, 0)

16
swarm/storage/forwarder.go
View file

@ -1,16 +0,0 @@
package storage
// implements CloudStore
// noop placeholder for netstore functionality
type Forwarder struct {
}
func (self *Forwarder) Store(chunk *Chunk) {
}
func (self *Forwarder) Retrieve(chunk *Chunk) {
}
func (self *Forwarder) Deliver(chunk *Chunk) {
}

95
swarm/storage/localstore.go
View file

@ -18,10 +18,27 @@ package storage
import (
"encoding/binary"
"fmt"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/storage/mock"
)
type StoreParams struct {
ChunkDbPath string
DbCapacity uint64
CacheCapacity uint
Radius int
}
//create params with default values
func NewDefaultStoreParams() (self *StoreParams) {
return &StoreParams{
DbCapacity: defaultDbCapacity,
CacheCapacity: defaultCacheCapacity,
}
}
// LocalStore is a combination of inmemory db over a disk persisted db
// implements a Get/Put with fallback (caching) logic using any 2 ChunkStores
type LocalStore struct {
@ -29,10 +46,9 @@ type LocalStore struct {
DbStore ChunkStore
}
// This constructor uses MemStore and DbStore as components.
// If mockStore is not nil, it will be used by DbStore to store chunk data.
func NewLocalStore(hash SwarmHasher, params *StoreParams, mockStore *mock.NodeStore) (*LocalStore, error) {
dbStore, err := NewMockDbStore(params.ChunkDbPath, hash, params.DbCapacity, params.Radius, mockStore)
// This constructor uses MemStore and DbStore as components
func NewLocalStore(hash SwarmHasher, params *StoreParams, basekey []byte, mockStore *mock.NodeStore) (*LocalStore, error) {
dbStore, err := NewMockDbStore(params.ChunkDbPath, hash, params.DbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) }, mockStore)
if err != nil {
return nil, err
}
@ -42,20 +58,45 @@ func NewLocalStore(hash SwarmHasher, params *StoreParams, mockStore *mock.NodeSt
}, nil
}
func NewTestLocalStore(path string) (*LocalStore, error) {
basekey := make([]byte, 32)
hasher := MakeHashFunc("SHA3")
dbStore, err := NewDbStore(path, hasher, singletonSwarmDbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) })
if err != nil {
return nil, err
}
localStore := &LocalStore{
memStore: NewMemStore(dbStore, singletonSwarmDbCapacity),
DbStore: dbStore,
}
return localStore, nil
}
func NewTestLocalStoreForAddr(path string, basekey []byte) (*LocalStore, error) {
hasher := MakeHashFunc("SHA3")
dbStore, err := NewDbStore(path, hasher, singletonSwarmDbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) })
if err != nil {
return nil, err
}
localStore := &LocalStore{
memStore: NewMemStore(dbStore, singletonSwarmDbCapacity),
DbStore: dbStore,
}
return localStore, nil
}
// LocalStore is itself a chunk store
// unsafe, in that the data is not integrity checked
func (self *LocalStore) Put(chunk *Chunk) {
chunk.dbStored = make(chan bool)
self.memStore.Put(chunk)
if chunk.wg != nil {
chunk.wg.Add(1)
chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8]))
c := &Chunk{
Key: Key(append([]byte{}, chunk.Key...)),
SData: append([]byte{}, chunk.SData...),
Size: chunk.Size,
dbStored: chunk.dbStored,
}
go func() {
self.DbStore.Put(chunk)
if chunk.wg != nil {
chunk.wg.Done()
}
}()
self.memStore.Put(c)
self.DbStore.Put(c)
}
// Get(chunk *Chunk) looks up a chunk in the local stores
@ -65,6 +106,13 @@ func (self *LocalStore) Put(chunk *Chunk) {
func (self *LocalStore) Get(key Key) (chunk *Chunk, err error) {
chunk, err = self.memStore.Get(key)
if err == nil {
if chunk.ReqC != nil {
select {
case <-chunk.ReqC:
default:
return chunk, ErrFetching
}
}
return
}
chunk, err = self.DbStore.Get(key)
@ -76,6 +124,25 @@ func (self *LocalStore) Get(key Key) (chunk *Chunk, err error) {
return
}
// retrieve logic common for local and network chunk retrieval requests
func (self *LocalStore) GetOrCreateRequest(key Key) (chunk *Chunk, created bool) {
var err error
chunk, err = self.Get(key)
if err == nil {
log.Trace(fmt.Sprintf("LocalStore.GetOrRetrieve: %v found locally", key))
return chunk, false
}
if err == ErrFetching {
log.Trace(fmt.Sprintf("LocalStore.GetOrRetrieve: %v hit on an existing request %v", key, chunk.ReqC))
return chunk, false
}
// no data and no request status
log.Trace(fmt.Sprintf("LocalStore.GetOrRetrieve: %v not found locally. open new request", key))
chunk = NewChunk(key, make(chan bool))
self.memStore.Put(chunk)
return chunk, true
}
// Close the local store
func (self *LocalStore) Close() {
self.DbStore.Close()

58
swarm/storage/memstore.go
View file

@ -168,8 +168,8 @@ func (s *MemStore) Put(entry *Chunk) {
entry.Size = node.entry.Size
entry.SData = node.entry.SData
}
if entry.Req == nil {
entry.Req = node.entry.Req
if entry.ReqC == nil {
entry.ReqC = node.entry.ReqC
}
entry.C = node.entry.C
node.entry = entry
@ -214,7 +214,7 @@ func (s *MemStore) Get(hash Key) (chunk *Chunk, err error) {
l := hash.bits(bitpos, node.bits)
st := node.subtree[l]
if st == nil {
return nil, notFound
return nil, ErrNotFound
}
bitpos += node.bits
node = st
@ -232,7 +232,7 @@ func (s *MemStore) Get(hash Key) (chunk *Chunk, err error) {
}
}
} else {
err = notFound
err = ErrNotFound
}
return
@ -240,7 +240,7 @@ func (s *MemStore) Get(hash Key) (chunk *Chunk, err error) {
func (s *MemStore) removeOldest() {
node := s.memtree
log.Warn("purge memstore")
for node.entry == nil {
aidx := uint(0)
@ -280,17 +280,15 @@ func (s *MemStore) removeOldest() {
}
if node.entry.dbStored != nil {
log.Trace(fmt.Sprintf("Memstore Clean: Waiting for chunk %v to be saved", node.entry.Key.Log()))
<-node.entry.dbStored
log.Trace(fmt.Sprintf("Memstore Clean: Chunk %v saved to DBStore. Ready to clear from mem.", node.entry.Key.Log()))
} else {
log.Trace(fmt.Sprintf("Memstore Clean: Chunk %v already in DB. Ready to delete.", node.entry.Key.Log()))
}
log.Trace(fmt.Sprintf("Memstore Clean: Waiting for chunk %v to be saved", node.entry.Key.Log()))
<-node.entry.dbStored
log.Trace(fmt.Sprintf("Memstore Clean: Chunk %v saved to DBStore. Ready to clear from mem.", node.entry.Key.Log()))
if node.entry.SData != nil {
if node.entry.ReqC == nil {
node.entry = nil
s.entryCnt--
} else {
return
}
node.access[0] = 0
@ -316,5 +314,39 @@ func (s *MemStore) removeOldest() {
}
}
// type MemStore struct {
// m map[string]*Chunk
// mu sync.RWMutex
// }
// func NewMemStore(d *DbStore, capacity uint) (m *MemStore) {
// return &MemStore{
// m: make(map[string]*Chunk),
// }
// }
// func (m *MemStore) Get(key Key) (*Chunk, error) {
// m.mu.RLock()
// defer m.mu.RUnlock()
// c, ok := m.m[string(key[:])]
// if !ok {
// return nil, ErrNotFound
// }
// if !bytes.Equal(c.Key, key) {
// panic(fmt.Errorf("MemStore.Get: chunk key %s != req key %s", c.Key.Hex(), key.Hex()))
// }
// return c, nil
// }
// func (m *MemStore) Put(c *Chunk) {
// m.mu.Lock()
// defer m.mu.Unlock()
// m.m[string(c.Key[:])] = c
// }
// func (m *MemStore) setCapacity(n int) {
// }
// Close memstore
func (s *MemStore) Close() {}

81
swarm/storage/memstore_test.go
View file

@ -16,35 +16,82 @@
package storage
import (
"testing"
)
import "testing"
func testMemStore(l int64, branches int64, t *testing.T) {
m := NewMemStore(nil, defaultCacheCapacity)
testStore(m, l, branches, t)
func newTestMemStore() *MemStore {
return NewMemStore(nil, defaultCacheCapacity)
}
func TestMemStore128_10000(t *testing.T) {
testMemStore(10000, 128, t)
func testMemStoreRandom(n int, processors int, chunksize int, t *testing.T) {
m := newTestMemStore()
defer m.Close()
testStoreRandom(m, processors, n, chunksize, t)
}
func TestMemStore128_1000(t *testing.T) {
testMemStore(1000, 128, t)
func testMemStoreCorrect(n int, processors int, chunksize int, t *testing.T) {
m := newTestMemStore()
defer m.Close()
testStoreCorrect(m, processors, n, chunksize, t)
}
func TestMemStore128_100(t *testing.T) {
testMemStore(100, 128, t)
func TestMemStoreRandom_1(t *testing.T) {
testMemStoreRandom(1, 1, 0, t)
}
func TestMemStore2_100(t *testing.T) {
testMemStore(100, 2, t)
func TestMemStoreCorrect_1(t *testing.T) {
testMemStoreCorrect(1, 1, 4104, t)
}
func TestMemStoreRandom_1_10k(t *testing.T) {
testMemStoreRandom(1, 5000, 0, t)
}
func TestMemStoreCorrect_1_10k(t *testing.T) {
testMemStoreCorrect(1, 5000, 4096, t)
}
func TestMemStoreRandom_8_10k(t *testing.T) {
testMemStoreRandom(8, 5000, 0, t)
}
func TestMemStoreCorrect_8_10k(t *testing.T) {
testMemStoreCorrect(8, 5000, 4096, t)
}
func TestMemStoreNotFound(t *testing.T) {
m := NewMemStore(nil, defaultCacheCapacity)
m := newTestMemStore()
defer m.Close()
_, err := m.Get(ZeroKey)
if err != notFound {
t.Errorf("Expected notFound, got %v", err)
if err != ErrNotFound {
t.Errorf("Expected ErrNotFound, got %v", err)
}
}
func benchmarkMemStorePut(n int, processors int, chunksize int, b *testing.B) {
m := newTestMemStore()
defer m.Close()
benchmarkStorePut(m, processors, n, chunksize, b)
}
func benchmarkMemStoreGet(n int, processors int, chunksize int, b *testing.B) {
m := newTestMemStore()
defer m.Close()
benchmarkStoreGet(m, processors, n, chunksize, b)
}
func BenchmarkMemStorePut_1_5k(b *testing.B) {
benchmarkMemStorePut(5000, 1, 4096, b)
}
func BenchmarkMemStorePut_8_5k(b *testing.B) {
benchmarkMemStorePut(5000, 8, 4096, b)
}
func BenchmarkMemStoreGet_1_5k(b *testing.B) {
benchmarkMemStoreGet(5000, 1, 4096, b)
}
func BenchmarkMemStoreGet_8_5k(b *testing.B) {
benchmarkMemStoreGet(5000, 8, 4096, b)
}

142
swarm/storage/netstore.go
View file

@ -17,53 +17,57 @@
package storage
import (
"fmt"
"path/filepath"
"time"
"github.com/ethereum/go-ethereum/log"
)
/*
NetStore is a cloud storage access abstaction layer for swarm
it contains the shared logic of network served chunk store/retrieval requests
both local (coming from DPA api) and remote (coming from peers via bzz protocol)
it implements the ChunkStore interface and embeds LocalStore
It is called by the bzz protocol instances via Depo (the store/retrieve request handler)
a protocol instance is running on each peer, so this is heavily parallelised.
NetStore falls back to a backend (CloudStorage interface)
implemented by bzz/network/forwarder. forwarder or IPFS or IPΞS
*/
// NetStore implements the ChunkStore interface,
// this chunk access layer assumed 2 chunk stores
// local storage eg. LocalStore and network storage eg., NetStore
// access by calling network is blocking with a timeout
type NetStore struct {
hashfunc SwarmHasher
localStore *LocalStore
cloud CloudStore
retrieve func(chunk *Chunk) error
}
// backend engine for cloud store
// It can be aggregate dispatching to several parallel implementations:
// bzz/network/forwarder. forwarder or IPFS or IPΞS
type CloudStore interface {
Store(*Chunk)
Deliver(*Chunk)
Retrieve(*Chunk)
func NewNetStore(localStore *LocalStore, retrieve func(chunk *Chunk) error) *NetStore {
return &NetStore{localStore, retrieve}
}
type StoreParams struct {
ChunkDbPath string
DbCapacity uint64
CacheCapacity uint
Radius int
}
// Get is the entrypoint for local retrieve requests
// waits for response or times out
func (self *NetStore) Get(key Key) (chunk *Chunk, err error) {
if self.retrieve == nil {
chunk, err = self.localStore.Get(key)
if err == nil {
return chunk, nil
}
if err != ErrFetching {
return nil, err
}
} else {
var created bool
chunk, created = self.localStore.GetOrCreateRequest(key)
if chunk.ReqC == nil {
return chunk, nil
}
//create params with default values
func NewDefaultStoreParams() (self *StoreParams) {
return &StoreParams{
DbCapacity: defaultDbCapacity,
CacheCapacity: defaultCacheCapacity,
Radius: defaultRadius,
if created {
if err := self.retrieve(chunk); err != nil {
return nil, err
}
}
}
t := time.NewTicker(searchTimeout)
defer t.Stop()
select {
case <-t.C:
return nil, ErrNotFound
case <-chunk.ReqC:
}
return chunk, nil
}
//this can only finally be set after all config options (file, cmd line, env vars)
@ -74,70 +78,10 @@ func (self *StoreParams) Init(path string) {
}
}
// netstore contructor, takes path argument that is used to initialise dbStore,
// the persistent (disk) storage component of LocalStore
// the second argument is the hive, the connection/logistics manager for the node
func NewNetStore(hash SwarmHasher, lstore *LocalStore, cloud CloudStore, params *StoreParams) *NetStore {
return &NetStore{
hashfunc: hash,
localStore: lstore,
cloud: cloud,
}
// Put is the entrypoint for local store requests coming from storeLoop
func (self *NetStore) Put(chunk *Chunk) {
self.localStore.Put(chunk)
}
const (
// maximum number of peers that a retrieved message is delivered to
requesterCount = 3
)
var (
// timeout interval before retrieval is timed out
searchTimeout = 3 * time.Second
)
// store logic common to local and network chunk store requests
// ~ unsafe put in localdb no check if exists no extra copy no hash validation
// the chunk is forced to propagate (Cloud.Store) even if locally found!
// caller needs to make sure if that is wanted
func (self *NetStore) Put(entry *Chunk) {
self.localStore.Put(entry)
// handle deliveries
if entry.Req != nil {
log.Trace(fmt.Sprintf("NetStore.Put: localStore.Put %v hit existing request...delivering", entry.Key.Log()))
// closing C signals to other routines (local requests)
// that the chunk is has been retrieved
close(entry.Req.C)
// deliver the chunk to requesters upstream
go self.cloud.Deliver(entry)
} else {
log.Trace(fmt.Sprintf("NetStore.Put: localStore.Put %v stored locally", entry.Key.Log()))
// handle propagating store requests
// go self.cloud.Store(entry)
go self.cloud.Store(entry)
}
}
// retrieve logic common for local and network chunk retrieval requests
func (self *NetStore) Get(key Key) (*Chunk, error) {
var err error
chunk, err := self.localStore.Get(key)
if err == nil {
if chunk.Req == nil {
log.Trace(fmt.Sprintf("NetStore.Get: %v found locally", key))
} else {
log.Trace(fmt.Sprintf("NetStore.Get: %v hit on an existing request", key))
// no need to launch again
}
return chunk, err
}
// no data and no request status
log.Trace(fmt.Sprintf("NetStore.Get: %v not found locally. open new request", key))
chunk = NewChunk(key, newRequestStatus(key))
self.localStore.memStore.Put(chunk)
go self.cloud.Retrieve(chunk)
return chunk, nil
}
// Close netstore
// Close chunk store
func (self *NetStore) Close() {}

98
swarm/storage/pyramid.go
View file

@ -136,10 +136,11 @@ func NewPyramidChunker(params *ChunkerParams) (self *PyramidChunker) {
return
}
func (self *PyramidChunker) Join(key Key, chunkC chan *Chunk) LazySectionReader {
func (self *PyramidChunker) Join(key Key, chunkC chan *Chunk, depth int) LazySectionReader {
return &LazyChunkReader{
key: key,
chunkC: chunkC,
depth: depth,
chunkSize: self.chunkSize,
branches: self.branches,
hashSize: self.hashSize,
@ -164,16 +165,18 @@ func (self *PyramidChunker) decrementWorkerCount() {
self.workerCount -= 1
}
func (self *PyramidChunker) Split(data io.Reader, size int64, chunkC chan *Chunk, storageWG, processorWG *sync.WaitGroup) (Key, error) {
func (self *PyramidChunker) Split(data io.Reader, size int64, chunkC chan *Chunk) (k Key, wait func(), err error) {
jobC := make(chan *chunkJob, 2*ChunkProcessors)
wg := &sync.WaitGroup{}
storageWG := &sync.WaitGroup{}
storageWG.Add(1)
errC := make(chan error)
quitC := make(chan bool)
rootKey := make([]byte, self.hashSize)
chunkLevel := make([][]*TreeEntry, self.branches)
wg.Add(1)
go self.prepareChunks(false, chunkLevel, data, rootKey, quitC, wg, jobC, processorWG, chunkC, errC, storageWG)
self.prepareChunks(false, chunkLevel, data, rootKey, quitC, wg, jobC, chunkC, errC, storageWG)
// closes internal error channel if all subprocesses in the workgroup finished
go func() {
@ -181,10 +184,6 @@ func (self *PyramidChunker) Split(data io.Reader, size int64, chunkC chan *Chunk
// waiting for all chunks to finish
wg.Wait()
// if storage waitgroup is non-nil, we wait for storage to finish too
if storageWG != nil {
storageWG.Wait()
}
//We close errC here because this is passed down to 8 parallel routines underneath.
// if a error happens in one of them.. that particular routine raises error...
// once they all complete successfully, the control comes back and we can safely close this here.
@ -196,15 +195,15 @@ func (self *PyramidChunker) Split(data io.Reader, size int64, chunkC chan *Chunk
select {
case err := <-errC:
if err != nil {
return nil, err
return nil, nil, err
}
case <-time.NewTimer(splitTimeout).C:
}
return rootKey, nil
return rootKey, storageWG.Wait, nil
}
func (self *PyramidChunker) Append(key Key, data io.Reader, chunkC chan *Chunk, storageWG, processorWG *sync.WaitGroup) (Key, error) {
func (self *PyramidChunker) Append(key Key, data io.Reader, chunkC chan *Chunk) (k Key, wait func(), err error) {
quitC := make(chan bool)
rootKey := make([]byte, self.hashSize)
chunkLevel := make([][]*TreeEntry, self.branches)
@ -216,8 +215,11 @@ func (self *PyramidChunker) Append(key Key, data io.Reader, chunkC chan *Chunk,
wg := &sync.WaitGroup{}
errC := make(chan error)
storageWG := &sync.WaitGroup{}
storageWG.Add(1)
wg.Add(1)
go self.prepareChunks(true, chunkLevel, data, rootKey, quitC, wg, jobC, processorWG, chunkC, errC, storageWG)
self.prepareChunks(true, chunkLevel, data, rootKey, quitC, wg, jobC, chunkC, errC, storageWG)
// closes internal error channel if all subprocesses in the workgroup finished
go func() {
@ -225,10 +227,6 @@ func (self *PyramidChunker) Append(key Key, data io.Reader, chunkC chan *Chunk,
// waiting for all chunks to finish
wg.Wait()
// if storage waitgroup is non-nil, we wait for storage to finish too
if storageWG != nil {
storageWG.Wait()
}
close(errC)
}()
@ -237,21 +235,18 @@ func (self *PyramidChunker) Append(key Key, data io.Reader, chunkC chan *Chunk,
select {
case err := <-errC:
if err != nil {
return nil, err
return nil, nil, err
}
case <-time.NewTimer(splitTimeout).C:
}
return rootKey, nil
return rootKey, storageWG.Wait, nil
}
func (self *PyramidChunker) processor(id int64, jobC chan *chunkJob, chunkC chan *Chunk, errC chan error, quitC chan bool, swg, wwg *sync.WaitGroup) {
func (self *PyramidChunker) processor(id int64, jobC chan *chunkJob, chunkC chan *Chunk, errC chan error, quitC chan bool, storageWG *sync.WaitGroup) {
defer self.decrementWorkerCount()
defer storageWG.Done()
hasher := self.hashFunc()
if wwg != nil {
defer wwg.Done()
}
for {
select {
@ -259,38 +254,35 @@ func (self *PyramidChunker) processor(id int64, jobC chan *chunkJob, chunkC chan
if !ok {
return
}
self.processChunk(id, hasher, job, chunkC, swg)
self.processChunk(id, hasher, job, chunkC, storageWG)
case <-quitC:
return
}
}
}
func (self *PyramidChunker) processChunk(id int64, hasher SwarmHash, job *chunkJob, chunkC chan *Chunk, swg *sync.WaitGroup) {
func (self *PyramidChunker) processChunk(id int64, hasher SwarmHash, job *chunkJob, chunkC chan *Chunk, storageWG *sync.WaitGroup) {
hasher.ResetWithLength(job.chunk[:8]) // 8 bytes of length
hasher.Write(job.chunk[8:]) // minus 8 []byte length
h := hasher.Sum(nil)
newChunk := &Chunk{
Key: h,
SData: job.chunk,
Size: job.size,
wg: swg,
}
newChunk := NewChunk(h, nil)
newChunk.SData = job.chunk
newChunk.Size = job.size
// report hash of this chunk one level up (keys corresponds to the proper subslice of the parent chunk)
copy(job.key, h)
// send off new chunk to storage
if chunkC != nil {
if swg != nil {
swg.Add(1)
}
}
job.parentWg.Done()
if chunkC != nil {
chunkC <- newChunk
storageWG.Add(1)
go func() {
defer storageWG.Done()
<-newChunk.dbStored
}()
}
}
@ -374,19 +366,15 @@ func (self *PyramidChunker) loadTree(chunkLevel [][]*TreeEntry, key Key, chunkC
return nil
}
func (self *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEntry, data io.Reader, rootKey []byte, quitC chan bool, wg *sync.WaitGroup, jobC chan *chunkJob, processorWG *sync.WaitGroup, chunkC chan *Chunk, errC chan error, storageWG *sync.WaitGroup) {
func (self *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEntry, data io.Reader, rootKey []byte, quitC chan bool, wg *sync.WaitGroup, jobC chan *chunkJob, chunkC chan *Chunk, errC chan error, storageWG *sync.WaitGroup) {
defer wg.Done()
chunkWG := &sync.WaitGroup{}
totalDataSize := 0
// processorWG keeps track of workers spawned for hashing chunks
if processorWG != nil {
processorWG.Add(1)
}
self.incrementWorkerCount()
go self.processor(self.workerCount, jobC, chunkC, errC, quitC, storageWG, processorWG)
go self.processor(self.workerCount, jobC, chunkC, errC, quitC, storageWG)
parent := NewTreeEntry(self)
var unFinishedChunk *Chunk
@ -426,14 +414,24 @@ func (self *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEnt
var n int
var err error
chunkData := make([]byte, self.chunkSize+8)
maxBuf := len(chunkData)
readBytes := 8
if unFinishedChunk != nil {
copy(chunkData, unFinishedChunk.SData)
n, err = data.Read(chunkData[8+unFinishedChunk.Size:])
n += int(unFinishedChunk.Size)
unFinishedChunk = nil
} else {
n, err = data.Read(chunkData[8:])
readBytes += int(unFinishedChunk.Size)
}
for readBytes < maxBuf {
n0, err0 := data.Read(chunkData[readBytes:])
readBytes += n0
n += n0
if err0 != nil {
if err0 != io.EOF || (n0 == 0 && maxBuf == readBytes) || n == 0 || n0 != 0 {
err = err0
}
break
}
}
unFinishedChunk = nil
totalDataSize += n
if err != nil {
@ -486,11 +484,9 @@ func (self *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEnt
workers := self.getWorkerCount()
if int64(len(jobC)) > workers && workers < ChunkProcessors {
if processorWG != nil {
processorWG.Add(1)
}
self.incrementWorkerCount()
go self.processor(self.workerCount, jobC, chunkC, errC, quitC, storageWG, processorWG)
storageWG.Add(1)
go self.processor(self.workerCount, jobC, chunkC, errC, quitC, storageWG)
}
}

52
swarm/storage/resource.go
View file

@ -140,24 +140,9 @@ type ResourceHandler struct {
}
// Create or open resource update chunk store
//
// If validator is nil, signature and access validation will be deactivated
func NewResourceHandler(datadir string, cloudStore CloudStore, ethClient ethApi, validator ResourceValidator) (*ResourceHandler, error) {
hashfunc := MakeHashFunc(SHA3Hash)
path := filepath.Join(datadir, DbDirName)
dbStore, err := NewDbStore(datadir, hashfunc, singletonSwarmDbCapacity, 0)
if err != nil {
return nil, err
}
localStore := &LocalStore{
memStore: NewMemStore(dbStore, singletonSwarmDbCapacity),
DbStore: dbStore,
}
func NewResourceHandler(hasher SwarmHasher, chunkStore ChunkStore, ethClient ethApi, validator ResourceValidator) (*ResourceHandler, error) {
rh := &ResourceHandler{
ChunkStore: newResourceChunkStore(path, hashfunc, localStore, cloudStore),
ChunkStore: chunkStore,
ethClient: ethClient,
resources: make(map[string]*resource),
validator: validator,
@ -734,10 +719,10 @@ type resourceChunkStore struct {
chunkSize int64
}
func newResourceChunkStore(path string, hasher SwarmHasher, localStore *LocalStore, cloudStore CloudStore) *resourceChunkStore {
func NewResourceChunkStore(localStore *LocalStore, request func(*Chunk) error) ChunkStore {
return &resourceChunkStore{
localStore: localStore,
netStore: NewNetStore(hasher, localStore, cloudStore, NewDefaultStoreParams()),
netStore: NewNetStore(localStore, request),
}
}
@ -749,23 +734,23 @@ func (r *resourceChunkStore) Get(key Key) (*Chunk, error) {
// if the chunk has to be remotely retrieved, we define a timeout of how long to wait for it before failing.
// sadly due to the nature of swarm, the error will never be conclusive as to whether it was a network issue
// that caused the failure or that the chunk doesn't exist.
if chunk.Req == nil {
if chunk.ReqC == nil {
return chunk, nil
}
t := time.NewTimer(time.Second * 1)
select {
case <-t.C:
return nil, errors.New("timeout")
case <-chunk.Req.C:
log.Trace("Received resource update chunk", "peer", chunk.Req.Source)
log.Trace("Timeout on resource chunk store")
return nil, fmt.Errorf("timeout")
case <-chunk.C:
log.Trace("Received resource update chunk")
}
return chunk, nil
}
func (r *resourceChunkStore) Put(chunk *Chunk) {
chunk.wg = &sync.WaitGroup{}
r.netStore.Put(chunk)
chunk.wg.Wait()
chunk.WaitToStore()
}
func (r *resourceChunkStore) Close() {
@ -804,3 +789,20 @@ func (self *ResourceHandler) keyDataHash(key Key, data []byte) common.Hash {
hasher.Write(data)
return common.BytesToHash(hasher.Sum(nil))
}
// TODO: this should not be exposed, but swarm/testutil/http.go needs it
func NewTestResourceHandler(datadir string, ethClient ethApi, validator ResourceValidator) (*ResourceHandler, error) {
path := filepath.Join(datadir, DbDirName)
basekey := make([]byte, 32)
hasher := MakeHashFunc(SHA3Hash)
dbStore, err := NewDbStore(path, hasher, singletonSwarmDbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) })
if err != nil {
return nil, err
}
localStore := &LocalStore{
memStore: NewMemStore(dbStore, singletonSwarmDbCapacity),
DbStore: dbStore,
}
resourceChunkStore := NewResourceChunkStore(localStore, nil)
return NewResourceHandler(hasher, resourceChunkStore, ethClient, validator)
}

16
swarm/storage/resource_test.go
View file

@ -220,7 +220,7 @@ func TestResourceHandler(t *testing.T) {
// it will match on second iteration startblocknumber + (resourceFrequency * 3)
fwdBlocks(int(resourceFrequency*2)-1, backend)
rh2, err := NewResourceHandler(datadir, &testCloudStore{}, rh.ethClient, nil)
rh2, err := NewTestResourceHandler(datadir, rh.ethClient, nil)
_, err = rh2.LookupLatestByName(ctx, safeName, true)
if err != nil {
t.Fatal(err)
@ -351,7 +351,7 @@ func setupTest(backend ethApi, validator ResourceValidator) (rh *ResourceHandler
os.RemoveAll(datadir)
}
rh, err = NewResourceHandler(datadir, &testCloudStore{}, backend, validator)
rh, err = NewTestResourceHandler(datadir, backend, validator)
return rh, datadir, signer, cleanF, nil
}
@ -417,18 +417,6 @@ func newTestSigner() (*testSigner, error) {
}, nil
}
type testCloudStore struct {
}
func (c *testCloudStore) Store(*Chunk) {
}
func (c *testCloudStore) Deliver(*Chunk) {
}
func (c *testCloudStore) Retrieve(*Chunk) {
}
// Default fallthrough validation of mutable resource ownership
type testValidator struct {
*baseValidator

115
swarm/storage/types.go
View file

@ -19,16 +19,19 @@ package storage
import (
"bytes"
"crypto"
"crypto/rand"
"encoding/binary"
"fmt"
"hash"
"io"
"sync"
"github.com/ethereum/go-ethereum/bmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto/sha3"
)
const MaxPO = 7
type Hasher func() hash.Hash
type SwarmHasher func() SwarmHash
@ -73,6 +76,26 @@ func (h Key) bits(i, j uint) uint {
return res
}
func Proximity(one, other []byte) (ret int) {
b := (MaxPO-1)/8 + 1
if b > len(one) {
b = len(one)
}
m := 8
for i := 0; i < b; i++ {
oxo := one[i] ^ other[i]
if i == b-1 {
m = MaxPO % 8
}
for j := 0; j < m; j++ {
if (oxo>>uint8(7-j))&0x01 != 0 {
return i*8 + j
}
}
}
return MaxPO
}
func IsZeroKey(key Key) bool {
return len(key) == 0 || bytes.Equal(key, ZeroKey)
}
@ -100,10 +123,10 @@ func (key Key) Hex() string {
}
func (key Key) Log() string {
if len(key[:]) < 4 {
if len(key[:]) < 8 {
return fmt.Sprintf("%x", []byte(key[:]))
}
return fmt.Sprintf("%08x", []byte(key[:4]))
return fmt.Sprintf("%016x", []byte(key[:8]))
}
func (key Key) String() string {
@ -122,25 +145,22 @@ func (key *Key) UnmarshalJSON(value []byte) error {
return nil
}
// each chunk when first requested opens a record associated with the request
// next time a request for the same chunk arrives, this record is updated
// this request status keeps track of the request ID-s as well as the requesting
// peers and has a channel that is closed when the chunk is retrieved. Multiple
// local callers can wait on this channel (or combined with a timeout, block with a
// select).
type RequestStatus struct {
Key Key
Source Peer
C chan bool
Requesters map[uint64][]interface{}
type KeyCollection []Key
func NewKeyCollection(l int) KeyCollection {
return make(KeyCollection, l)
}
func newRequestStatus(key Key) *RequestStatus {
return &RequestStatus{
Key: key,
Requesters: make(map[uint64][]interface{}),
C: make(chan bool),
}
func (c KeyCollection) Len() int {
return len(c)
}
func (c KeyCollection) Less(i, j int) bool {
return bytes.Compare(c[i], c[j]) == -1
}
func (c KeyCollection) Swap(i, j int) {
c[i], c[j] = c[j], c[i]
}
// Chunk also serves as a request object passed to ChunkStores
@ -149,18 +169,47 @@ func newRequestStatus(key Key) *RequestStatus {
// but the size of the subtree encoded in the chunk
// 0 if request, to be supplied by the dpa
type Chunk struct {
Key Key // always
SData []byte // nil if request, to be supplied by dpa
Size int64 // size of the data covered by the subtree encoded in this chunk
Source Peer // peer
C chan bool // to signal data delivery by the dpa
Req *RequestStatus // request Status needed by netStore
wg *sync.WaitGroup // wg to synchronize
dbStored chan bool // never remove a chunk from memStore before it is written to dbStore
Key Key // always
SData []byte // nil if request, to be supplied by dpa
Size int64 // size of the data covered by the subtree encoded in this chunk
//Source Peer // peer
C chan bool // to signal data delivery by the dpa
ReqC chan bool // to signal the request done
dbStored chan bool // never remove a chunk from memStore before it is written to dbStore
}
func NewChunk(key Key, rs *RequestStatus) *Chunk {
return &Chunk{Key: key, Req: rs}
func NewChunk(key Key, reqC chan bool) *Chunk {
return &Chunk{Key: key, ReqC: reqC, dbStored: make(chan bool)}
}
func (c *Chunk) WaitToStore() {
<-c.dbStored
}
func FakeChunk(size int64, count int, chunks []*Chunk) int {
var i int
hasher := MakeHashFunc(SHA3Hash)()
chunksize := getDefaultChunkSize()
if size > chunksize {
size = chunksize
}
for i = 0; i < count; i++ {
hasher.Reset()
chunks[i].SData = make([]byte, size)
rand.Read(chunks[i].SData)
binary.LittleEndian.PutUint64(chunks[i].SData[:8], uint64(size))
hasher.Write(chunks[i].SData)
chunks[i].Key = make([]byte, 32)
copy(chunks[i].Key, hasher.Sum(nil))
}
return i
}
func getDefaultChunkSize() int64 {
return DefaultBranches * int64(MakeHashFunc(SHA3Hash)().Size())
}
/*
@ -198,14 +247,14 @@ type Splitter interface {
The caller gets returned an error channel, if an error is encountered during splitting, it is fed to errC error channel.
A closed error signals process completion at which point the key can be considered final if there were no errors.
*/
Split(io.Reader, int64, chan *Chunk, *sync.WaitGroup, *sync.WaitGroup) (Key, error)
Split(io.Reader, int64, chan *Chunk) (Key, func(), error)
/* This is the first step in making files mutable (not chunks)..
Append allows adding more data chunks to the end of the already existsing file.
The key for the root chunk is supplied to load the respective tree.
Rest of the parameters behave like Split.
*/
Append(Key, io.Reader, chan *Chunk, *sync.WaitGroup, *sync.WaitGroup) (Key, error)
Append(Key, io.Reader, chan *Chunk) (Key, func(), error)
}
type Joiner interface {
@ -221,7 +270,7 @@ type Joiner interface {
The chunks are not meant to be validated by the chunker when joining. This
is because it is left to the DPA to decide which sources are trusted.
*/
Join(key Key, chunkC chan *Chunk) LazySectionReader
Join(key Key, chunkC chan *Chunk, depth int) LazySectionReader
}
type Chunker interface {

74
swarm/swarm.go
View file

@ -22,7 +22,6 @@ import (
"crypto/ecdsa"
"fmt"
"net"
"path/filepath"
"github.com/ethereum/go-ethereum/accounts/abi/bind"
"github.com/ethereum/go-ethereum/common"
@ -40,6 +39,7 @@ import (
httpapi "github.com/ethereum/go-ethereum/swarm/api/http"
"github.com/ethereum/go-ethereum/swarm/fuse"
"github.com/ethereum/go-ethereum/swarm/network"
"github.com/ethereum/go-ethereum/swarm/network/stream"
"github.com/ethereum/go-ethereum/swarm/pss"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/storage/mock"
@ -51,16 +51,19 @@ type Swarm struct {
api *api.Api // high level api layer (fs/manifest)
dns api.Resolver // DNS registrar
//dbAccess *network.DbAccess // access to local chunk db iterator and storage counter
storage storage.ChunkStore // internal access to storage, common interface to cloud storage backends
dpa *storage.DPA // distributed preimage archive, the local API to the storage with document level storage/retrieval support
//storage storage.ChunkStore // internal access to storage, common interface to cloud storage backends
dpa *storage.DPA // distributed preimage archive, the local API to the storage with document level storage/retrieval support
//depo network.StorageHandler // remote request handler, interface between bzz protocol and the storage
cloud storage.CloudStore // procurement, cloud storage backend (can multi-cloud)
bzz *network.Bzz // the logistic manager
backend chequebook.Backend // simple blockchain Backend
privateKey *ecdsa.PrivateKey
lstore *storage.LocalStore // local store, needs to store for releasing resources after node stopped
sfs *fuse.SwarmFS // need this to cleanup all the active mounts on node exit
ps *pss.Pss
streamer *stream.Registry
//cloud storage.CloudStore // procurement, cloud storage backend (can multi-cloud)
bzz *network.Bzz // the logistic manager
backend chequebook.Backend // simple blockchain Backend
privateKey *ecdsa.PrivateKey
corsString string
swapEnabled bool
lstore *storage.LocalStore // local store, needs to store for releasing resources after node stopped
sfs *fuse.SwarmFS // need this to cleanup all the active mounts on node exit
ps *pss.Pss
}
type SwarmAPI struct {
@ -98,7 +101,7 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e
log.Debug(fmt.Sprintf("Setting up Swarm service components"))
hash := storage.MakeHashFunc(config.ChunkerParams.Hash)
self.lstore, err = storage.NewLocalStore(hash, config.StoreParams, mockStore)
self.lstore, err = storage.NewLocalStore(hash, config.StoreParams, common.Hex2Bytes(config.BzzKey), mockStore)
if err != nil {
return
}
@ -115,8 +118,8 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e
config.HiveParams.Discovery = true
// setup cloud storage internal access layer
self.cloud = &storage.Forwarder{}
self.storage = storage.NewNetStore(hash, self.lstore, self.cloud, config.StoreParams)
//self.cloud = &storage.Forwarder{}
//self.storage = storage.NewNetStore(hash, self.lstore, self.cloud, config.StoreParams)
log.Debug(fmt.Sprintf("-> swarm net store shared access layer to Swarm Chunk Store"))
nodeid := discover.PubkeyID(crypto.ToECDSAPub(common.FromHex(config.PublicKey)))
addr := network.NewAddrFromNodeID(nodeid)
@ -125,10 +128,17 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e
UnderlayAddr: addr.UAddr,
HiveParams: config.HiveParams,
}
db := storage.NewDBAPI(self.lstore)
delivery := stream.NewDelivery(to, db)
self.streamer = stream.NewRegistry(addr, delivery, self.lstore, false)
stream.RegisterSwarmSyncerServer(self.streamer, db)
stream.RegisterSwarmSyncerClient(self.streamer, db)
self.bzz = network.NewBzz(bzzconfig, to, nil)
// set up DPA, the cloud storage local access layer
dpaChunkStore := storage.NewDpaChunkStore(self.lstore, self.storage)
dpaChunkStore := storage.NewNetStore(self.lstore, self.streamer.Retrieve)
log.Debug(fmt.Sprintf("-> Local Access to Swarm"))
// Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage
self.dpa = storage.NewDPA(dpaChunkStore, self.config.ChunkerParams)
@ -166,7 +176,9 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e
return nil, err
}
}
resourceHandler, err = storage.NewResourceHandler(filepath.Join(self.config.Path, storage.DbDirName), self.cloud, ensClient, resourceValidator)
hashfunc := storage.MakeHashFunc(storage.SHA3Hash)
chunkStore := storage.NewResourceChunkStore(self.lstore, func(*storage.Chunk) error { return nil })
resourceHandler, err = storage.NewResourceHandler(hashfunc, chunkStore, ensClient, resourceValidator)
if err != nil {
return nil, err
}
@ -267,12 +279,14 @@ func (self *Swarm) Stop() error {
// implements the node.Service interface
func (self *Swarm) Protocols() (protos []p2p.Protocol) {
protos = append(protos, self.bzz.Protocols()...)
if self.ps != nil {
protos = append(protos, self.ps.Protocols()...)
}
if self.streamer != nil {
protos = append(protos, self.streamer.Protocols()...)
}
return
}
@ -285,7 +299,7 @@ func (self *Swarm) RegisterPssProtocol(spec *protocols.Spec, targetprotocol *p2p
}
// implements node.Service
// Apis returns the RPC Api descriptors the Swarm implementation offers
// APIs returns the RPC Api descriptors the Swarm implementation offers
func (self *Swarm) APIs() []rpc.API {
apis := []rpc.API{
@ -355,32 +369,6 @@ func (self *Swarm) SetChequebook(ctx context.Context) error {
return nil
}
// Local swarm without netStore
func NewLocalSwarm(datadir, port string) (self *Swarm, err error) {
prvKey, err := crypto.GenerateKey()
if err != nil {
return
}
config := api.NewConfig()
config.Path = datadir
config.Port = port
config.Init(prvKey)
dpa, err := storage.NewLocalDPA(datadir)
if err != nil {
return
}
self = &Swarm{
api: api.NewApi(dpa, nil, nil),
config: config,
}
return
}
// serialisable info about swarm
type Info struct {
*api.Config

17
swarm/testutil/http.go
View file

@ -51,9 +51,8 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer {
ChunkDbPath: dir,
DbCapacity: 5000000,
CacheCapacity: 5000,
Radius: 0,
}
localStore, err := storage.NewLocalStore(storage.MakeHashFunc(storage.SHA3Hash), storeparams, nil)
localStore, err := storage.NewLocalStore(storage.MakeHashFunc(storage.SHA3Hash), storeparams, make([]byte, 32), nil)
if err != nil {
os.RemoveAll(dir)
t.Fatal(err)
@ -71,7 +70,7 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer {
t.Fatal(err)
}
rh, err := storage.NewResourceHandler(resourceDir, &testCloudStore{}, &fakeBackend{}, nil)
rh, err := storage.NewTestResourceHandler(resourceDir, &fakeBackend{}, nil)
if err != nil {
t.Fatal(err)
}
@ -104,15 +103,3 @@ type TestSwarmServer struct {
func (t *TestSwarmServer) Close() {
t.cleanup()
}
type testCloudStore struct {
}
func (c *testCloudStore) Store(*storage.Chunk) {
}
func (c *testCloudStore) Deliver(*storage.Chunk) {
}
func (c *testCloudStore) Retrieve(*storage.Chunk) {
}

6
vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go generated vendored
View file

@ -48,7 +48,7 @@ var wg sync.WaitGroup // used to wait until the runloop starts
// started and is ready via the wg. It also serves purpose of a dummy source,
// thanks to it the runloop does not return as it also has at least one source
// registered.
var source = C.CFRunLoopSourceCreate(refZero, 0, &C.CFRunLoopSourceContext{
var source = C.CFRunLoopSourceCreate(nil, 0, &C.CFRunLoopSourceContext{
perform: (C.CFRunLoopPerformCallBack)(C.gosource),
})
@ -162,8 +162,8 @@ func (s *stream) Start() error {
return nil
}
wg.Wait()
p := C.CFStringCreateWithCStringNoCopy(refZero, C.CString(s.path), C.kCFStringEncodingUTF8, refZero)
path := C.CFArrayCreate(refZero, (*unsafe.Pointer)(unsafe.Pointer(&p)), 1, nil)
p := C.CFStringCreateWithCStringNoCopy(nil, C.CString(s.path), C.kCFStringEncodingUTF8, nil)
path := C.CFArrayCreate(nil, (*unsafe.Pointer)(unsafe.Pointer(&p)), 1, nil)
ctx := C.FSEventStreamContext{}
ref := C.EventStreamCreate(&ctx, C.uintptr_t(s.info), path, C.FSEventStreamEventId(atomic.LoadUint64(&since)), latency, flags)
if ref == nilstream {

9
vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go generated vendored
View file

@ -1,9 +0,0 @@
// Copyright (c) 2017 The Notify Authors. All rights reserved.
// Use of this source code is governed by the MIT license that can be
// found in the LICENSE file.
// +build darwin,!kqueue,go1.10
package notify
const refZero = 0

14
vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.9.go generated vendored
View file

@ -1,14 +0,0 @@
// Copyright (c) 2017 The Notify Authors. All rights reserved.
// Use of this source code is governed by the MIT license that can be
// found in the LICENSE file.
// +build darwin,!kqueue,cgo,!go1.10
package notify
/*
#include <CoreServices/CoreServices.h>
*/
import "C"
var refZero = (*C.struct___CFAllocator)(nil)

6
vendor/vendor.json vendored
View file

@ -322,10 +322,10 @@
"revisionTime": "2016-11-28T21:05:44Z"
},
{
"checksumSHA1": "1ESHllhZOIBg7MnlGHUdhz047bI=",
"checksumSHA1": "28UVHMmHx0iqO0XiJsjx+fwILyI=",
"path": "github.com/rjeczalik/notify",
"revision": "27b537f07230b3f917421af6dcf044038dbe57e2",
"revisionTime": "2018-01-03T13:19:05Z"
"revision": "c31e5f2cb22b3e4ef3f882f413847669bf2652b9",
"revisionTime": "2018-02-03T14:01:15Z"
},
{
"checksumSHA1": "5uqO4ITTDMklKi3uNaE/D9LQ5nM=",

2
whisper/whisperv6/peer_test.go
View file

@ -93,7 +93,7 @@ var masterBloomFilter []byte
var masterPow = 0.00000001
var round = 1
func TestSimulation(t *testing.T) {
func XTestSimulation(t *testing.T) {
// create a chain of whisper nodes,
// installs the filters with shared (predefined) parameters
initialize(t)