forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-28 07:36:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge pull request #104 from ethersphere/network-testing-framework-psstalk

Browse source 
pss updates + kademlia fixes
This commit is contained in:
Lewis Marshall 2017-06-25 12:24:56 +02:00 committed by GitHub
commit 4cb8412cd0
35 changed files with 1987 additions and 1875 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
cmd/swarm/main.go
View file

@ -118,13 +118,9 @@ var (
Usage: "force mime type",
}
PssEnabledFlag = cli.BoolFlag{
Name: "pss",
Name: "pss",
Usage: "Enable pss (message passing over swarm)",
}
PssPortFlag = cli.IntFlag{
Name: "pssport",
Usage: fmt.Sprintf("Websockets port for pss (default %d)", node.DefaultWSPort),
}
CorsStringFlag = cli.StringFlag{
Name: "corsdomain",
Usage: "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')",
@ -270,8 +266,15 @@ Cleans database of corrupted entries.
SwarmUploadMimeType,
// pss flags
PssEnabledFlag,
PssPortFlag,
}
rpcFlags := []cli.Flag{
utils.WSEnabledFlag,
utils.WSListenAddrFlag,
utils.WSPortFlag,
utils.WSApiFlag,
utils.WSAllowedOriginsFlag,
}
app.Flags = append(app.Flags, rpcFlags...)
app.Flags = append(app.Flags, debug.Flags...)
app.Before = func(ctx *cli.Context) error {
runtime.GOMAXPROCS(runtime.NumCPU())
@ -306,13 +309,8 @@ func version(ctx *cli.Context) error {
func bzzd(ctx *cli.Context) error {
cfg := defaultNodeConfig
if ctx.GlobalIsSet(PssEnabledFlag.Name) {
cfg.WSHost = "127.0.0.1"
cfg.WSModules = []string{"eth","pss"}
cfg.WSOrigins = []string{"*"}
if ctx.GlobalIsSet(PssPortFlag.Name) {
cfg.WSPort = ctx.GlobalInt(PssPortFlag.Name)
}
if ctx.GlobalBool(PssEnabledFlag.Name) && ctx.GlobalBool(utils.WSEnabledFlag.Name) {
cfg.WSModules = append(cfg.WSModules, "pss")
}
utils.SetNodeConfig(ctx, &cfg)
stack, err := node.New(&cfg)
@ -366,7 +364,7 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) {
swapEnabled := ctx.GlobalBool(SwarmSwapEnabledFlag.Name)
syncEnabled := ctx.GlobalBoolT(SwarmSyncEnabledFlag.Name)
pssEnabled := ctx.GlobalBool(PssEnabledFlag.Name)
ethapi := ctx.GlobalString(EthAPIFlag.Name)
cors := ctx.GlobalString(CorsStringFlag.Name)

2
p2p/simulations/network.go
View file

@ -255,7 +255,7 @@ func (self *Network) newConn(oneID, otherID discover.NodeID) (*Conn, error) {
if other == nil {
return nil, fmt.Errorf("other %v does not exist", other)
}
distance, _ := pot.NewBytesVal(one.Addr(), nil).PO(pot.NewBytesVal(other.Addr(), nil), 0)
distance, _ := pot.DefaultPof(256)(one.Addr(), other.Addr(), 0)
return &Conn{
One: oneID,
Other: otherID,

214
pot/address.go
View file

@ -13,6 +13,8 @@
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Package pot see doc.go
package pot
import (
@ -26,35 +28,40 @@ import (
)
var (
zeroAddr = &common.Hash{}
zerosHex = zeroAddr.Hex()[2:]
zerosBin = Address{}.Bin()
)
var (
addrlen = keylen
)
// Address is an alias for common.Hash
type Address common.Hash
// NewAddressFromBytes constructs an Address from a byte slice
func NewAddressFromBytes(b []byte) Address {
h := common.Hash{}
copy(h[:], b)
return Address(h)
}
func (a Address) String() string {
return fmt.Sprintf("%x", a[:])
}
// MarshalJSON Address serialisation
func (a *Address) MarshalJSON() (out []byte, err error) {
return []byte(`"` + a.String() + `"`), nil
}
// UnmarshalJSON Address deserialisation
func (a *Address) UnmarshalJSON(value []byte) error {
*a = Address(common.HexToHash(string(value[1 : len(value)-1])))
return nil
}
// the string form of the binary representation of an address (only first 8 bits)
// Bin returns the string form of the binary representation of an address (only first 8 bits)
func (a Address) Bin() string {
return ToBin(a[:])
}
// ToBin converts a byteslice to the string binary representation
func ToBin(a []byte) string {
var bs []string
for _, b := range a {
@ -63,6 +70,7 @@ func ToBin(a []byte) string {
return strings.Join(bs, "")
}
// Bytes returns the Address as a byte slice
func (a Address) Bytes() []byte {
return a[:]
}
@ -107,23 +115,23 @@ func posProximity(one, other Address, pos int) (ret int, eq bool) {
return len(one) * 8, true
}
// Address.ProxCmp compares the distances a->target and b->target.
// ProxCmp compares the distances a->target and b->target.
// Returns -1 if a is closer to target, 1 if b is closer to target
// and 0 if they are equal.
func (target Address) ProxCmp(a, b Address) int {
for i := range target {
da := a[i] ^ target[i]
db := b[i] ^ target[i]
if da > db {
func (a Address) ProxCmp(x, y Address) int {
for i := range a {
dx := x[i] ^ a[i]
dy := y[i] ^ a[i]
if dx > dy {
return 1
} else if da < db {
} else if dx < dy {
return -1
}
}
return 0
}
// randomAddressAt(address, prox) generates a random address
// RandomAddressAt (address, prox) generates a random address
// at proximity order prox relative to address
// if prox is negative a random address is generated
func RandomAddressAt(self Address, prox int) (addr Address) {
@ -148,71 +156,12 @@ func RandomAddressAt(self Address, prox int) (addr Address) {
return
}
// KeyRange(a0, a1, proxLimit) returns the address inclusive address
// range that contain addresses closer to one than other
// func KeyRange(one, other Address, proxLimit int) (start, stop Address) {
// prox := proximity(one, other)
// if prox >= proxLimit {
// prox = proxLimit
// }
// start = CommonBitsAddrByte(one, other, byte(0x00), prox)
// stop = CommonBitsAddrByte(one, other, byte(0xff), prox)
// return
// }
func CommonBitsAddrF(self, other Address, f func() byte, p int) (addr Address) {
prox, _ := proximity(self, other)
var pos int
if p <= prox {
prox = p
}
pos = prox / 8
addr = self
trans := byte(prox % 8)
var transbytea byte
if p > prox {
transbytea = byte(0x7f)
} else {
transbytea = byte(0xff)
}
transbytea >>= trans
transbyteb := transbytea ^ byte(0xff)
addrpos := addr[pos]
addrpos &= transbyteb
if p > prox {
addrpos ^= byte(0x80 >> trans)
}
addrpos |= transbytea & f()
addr[pos] = addrpos
for i := pos + 1; i < len(addr); i++ {
addr[i] = f()
}
return
}
func CommonBitsAddr(self, other Address, prox int) (addr Address) {
return CommonBitsAddrF(self, other, func() byte { return byte(rand.Intn(255)) }, prox)
}
func CommonBitsAddrByte(self, other Address, b byte, prox int) (addr Address) {
return CommonBitsAddrF(self, other, func() byte { return b }, prox)
}
// randomAddressAt() generates a random address
// RandomAddress generates a random address
func RandomAddress() Address {
return RandomAddressAt(Address{}, -1)
}
// wraps an Address to implement the PotVal interface
type HashAddress struct {
Address
}
func (a *HashAddress) String() string {
return a.Address.Bin()
}
// NewAddressFromString creates a byte slice from a string in binary representation
func NewAddressFromString(s string) []byte {
ha := [32]byte{}
@ -227,85 +176,16 @@ func NewAddressFromString(s string) []byte {
return ha[:]
}
func NewHashAddress(s string) *HashAddress {
ha := NewAddressFromString(s)
h := common.Hash{}
copy(h[:], ha)
return &HashAddress{Address(h)}
}
func NewHashAddressFromBytes(b []byte) *HashAddress {
h := common.Hash{}
copy(h[:], b)
return &HashAddress{Address(h)}
}
// PO(addr, pos) return the proximity order of addr wrt to
// the pinned address of the tree
// assuming it is greater than or equal to pos
func (self *HashAddress) PO(val PotVal, pos int) (po int, eq bool) {
return posProximity(self.Address, val.(*HashAddress).Address, pos)
}
type BoolAddress struct {
addr []bool
}
func NewBoolAddress(s string) *BoolAddress {
return NewBoolAddressXOR(s, zerosBin[:len(s)])
}
func NewBoolAddressXOR(s, t string) *BoolAddress {
if len(s) != len(t) {
panic("lengths do not match")
}
addr := make([]bool, len(s))
for i, _ := range addr {
addr[i] = s[i] != t[i]
}
return &BoolAddress{addr}
}
func (self *BoolAddress) String() string {
a := self.addr
s := []byte(zerosBin)[:len(a)]
for i, one := range a {
if one {
s[i] = byte('1')
}
}
return string(s)
}
func (self *BoolAddress) PO(val PotVal, pos int) (po int, eq bool) {
a := self.addr
b := val.(*BoolAddress).addr
for po = pos; po < len(b); po++ {
if a[po] != b[po] {
return po, false
}
}
return po, true
}
// BytesAddress is an interface for elements addressable by a byte slice
type BytesAddress interface {
Address() []byte
}
type bytesAddress struct {
bytes []byte
toBytes func(v AnyVal) []byte
}
func NewBytesVal(v AnyVal, f func(v AnyVal) []byte) *bytesAddress {
if f == nil {
f = ToBytes
// ToBytes turns the Val into bytes
func ToBytes(v Val) []byte {
if v == nil {
return nil
}
b := f(v)
return &bytesAddress{b, f}
}
func ToBytes(v AnyVal) []byte {
b, ok := v.([]byte)
if !ok {
ba, ok := v.(BytesAddress)
@ -317,15 +197,17 @@ func ToBytes(v AnyVal) []byte {
return b
}
func (a *bytesAddress) String() string {
return fmt.Sprintf("%08b", a.bytes)
}
func (a *bytesAddress) Address() []byte {
return a.bytes
}
func (a *bytesAddress) PO(val PotVal, i int) (int, bool) {
return proximityOrder(a.bytes, a.toBytes(val), i)
// DefaultPof returns a proximity order comparison operator function
// where all
func DefaultPof(max int) func(one, other Val, pos int) (int, bool) {
return func(one, other Val, pos int) (int, bool) {
po, eq := proximityOrder(ToBytes(one), ToBytes(other), pos)
if po >= max {
eq = true
po = max
}
return po, eq
}
}
func proximityOrder(one, other []byte, pos int) (int, bool) {
@ -346,3 +228,17 @@ func proximityOrder(one, other []byte, pos int) (int, bool) {
}
return len(one) * 8, true
}
// Label displays the node's key in binary format
func Label(v Val) string {
if v == nil {
return "<nil>"
}
if s, ok := v.(fmt.Stringer); ok {
return s.String()
}
if b, ok := v.([]byte); ok {
return ToBin(b)
}
panic(fmt.Sprintf("unsupported value type %T", v))
}

141
pot/address_test.go
View file

@ -1,141 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package pot
import (
"math/rand"
"reflect"
"testing"
"github.com/ethereum/go-ethereum/common"
)
func (Address) Generate(rand *rand.Rand, size int) reflect.Value {
var id Address
for i := 0; i < len(id); i++ {
id[i] = byte(uint8(rand.Intn(255)))
}
return reflect.ValueOf(id)
}
func TestCommonBitsAddrF(t *testing.T) {
a := Address(common.HexToHash("0x0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"))
b := Address(common.HexToHash("0x8123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"))
c := Address(common.HexToHash("0x4123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"))
d := Address(common.HexToHash("0x0023456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"))
e := Address(common.HexToHash("0x01A3456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"))
ab := CommonBitsAddrF(a, b, func() byte { return byte(0x00) }, 10)
expab := Address(common.HexToHash("0x8000000000000000000000000000000000000000000000000000000000000000"))
if ab != expab {
t.Fatalf("%v != %v", ab, expab)
}
ac := CommonBitsAddrF(a, c, func() byte { return byte(0x00) }, 10)
expac := Address(common.HexToHash("0x4000000000000000000000000000000000000000000000000000000000000000"))
if ac != expac {
t.Fatalf("%v != %v", ac, expac)
}
ad := CommonBitsAddrF(a, d, func() byte { return byte(0x00) }, 10)
expad := Address(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"))
if ad != expad {
t.Fatalf("%v != %v", ad, expad)
}
ae := CommonBitsAddrF(a, e, func() byte { return byte(0x00) }, 10)
expae := Address(common.HexToHash("0x0180000000000000000000000000000000000000000000000000000000000000"))
if ae != expae {
t.Fatalf("%v != %v", ae, expae)
}
acf := CommonBitsAddrF(a, c, func() byte { return byte(0xff) }, 10)
expacf := Address(common.HexToHash("0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"))
if acf != expacf {
t.Fatalf("%v != %v", acf, expacf)
}
aeo := CommonBitsAddrF(a, e, func() byte { return byte(0x00) }, 2)
expaeo := Address(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"))
if aeo != expaeo {
t.Fatalf("%v != %v", aeo, expaeo)
}
aep := CommonBitsAddrF(a, e, func() byte { return byte(0xff) }, 2)
expaep := Address(common.HexToHash("0x3fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"))
if aep != expaep {
t.Fatalf("%v != %v", aep, expaep)
}
}
func TestRandomAddressAt(t *testing.T) {
var a Address
for i := 0; i < 100; i++ {
a = RandomAddress()
prox := rand.Intn(255)
b := RandomAddressAt(a, prox)
p, _ := proximity(a, b)
if p != prox {
t.Fatalf("incorrect address prox(%v, %v) == %v (expected %v)", a, b, p, prox)
}
}
}
const (
maxTestPOs = 1000
testPOkeylen = 9
)
func TestPOs(t *testing.T) {
for i := 0; i < maxTestPOs; i++ {
length := rand.Intn(256) + 1
v0 := RandomAddress().Bin()[:length]
v1 := RandomAddress().Bin()[:length]
a0 := NewBoolAddress(v0)
a1 := NewBoolAddress(v1)
b0 := NewHashAddress(v0)
b1 := NewHashAddress(v1)
pos := rand.Intn(length) + 1
apo, aeq := a0.PO(a1, pos)
bpo, beq := b0.PO(b1, pos)
if bpo == 256 {
bpo = length
}
a0s := a0.String()
if a0s != v0 {
t.Fatalf("incorrect bool address. expected %v, got %v", v0, a0s)
}
a1s := a1.String()
if a1s != v1 {
t.Fatalf("incorrect bool address. expected %v, got %v", v1, a1s)
}
b0s := b0.String()[:length]
if b0s != v0 {
t.Fatalf("incorrect hash address. expected %v, got %v", v0, b0s)
}
b1s := b1.String()[:length]
if b1s != v1 {
t.Fatalf("incorrect hash address. expected %v, got %v", v1, b1s)
}
if apo != bpo {
t.Fatalf("PO does not match for %v X %v (pos: %v): expected %v, got %v", v0, v1, pos, apo, bpo)
}
if aeq != beq {
t.Fatalf("PO equality does not match for %v X %v (pos: %v): expected %v, got %v", v0, v1, pos, aeq, beq)
}
}
}

58
pot/doc.go
View file

@ -1,22 +1,36 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
/*
POT: proximity order tree implements a container similar to a binary tree.
Value types implement the PoVal interface which provides the PO (proximity order)
comparison operator.
Package pot (proximity order tree) implements a container similar to a binary tree.
The elements are generic Val interface types.
Each fork in the trie is itself a value. Values of the subtree contained under
a node all share the same order when compared to other elements in the tree.
Example of proximity order is the length of the common prefix over bitvectors.
(which is equivalent to the order of magnitude of the XOR distance over integers).
(which is equivalent to the reverse rank of order of magnitude of the MSB first X
OR distance over finite set of integers).
The package provides two implementations of PoVal,
* BoolAddress: an arbitrary length boolean vector
* HashAddress: a bitvector based address derived from 256 bit hash (common.Hash)
Methods take a comparison operator (pof, proximity order function) to compare two
value types. The default pof assumes Val to be or project to a byte slice using
the reverse rank on the MSB first XOR logarithmic disctance.
If the address space if limited, equality is defined as the maximum proximity order.
Arbitrary value types can extend these base types or define their own PO method.
The container offers applicative (funcional) style methods on PO trees:
* adding/removing en element
* swap (value based add/remove)
@ -26,8 +40,10 @@ as well as iterator accessors that respect proximity order
When synchronicity of membership if not 100% requirement (e.g. used as a database
of network connections), applicative structures have the advantage that nodes
are immutable therefore manipulation does not need locking allowing for parallel retrievals.
For the use case where the entire container is supposed to allow changes by parallel routines, instance methods with write locks and access methods with readlock are provided. The latter only locks while reading the root node.
are immutable therefore manipulation does not need locking allowing for
concurrent retrievals.
For the use case where the entire container is supposed to allow changes by
concurrent routines,
Pot
* retrieval, insertion and deletion by key involves log(n) pointer lookups
@ -35,10 +51,11 @@ Pot
* provide syncronous iterators respecting proximity ordering wrt any item
* provide asyncronous iterator (for parallel execution of operations) over n items
* allows cheap iteration over ranges
* asymmetric parallellised merge (union)
* asymmetric concurrent merge (union)
Note:
* as is, union only makes sense for set representations since which of two values with equal keys survives is random
* as is, union only makes sense for set representations since which of two values
with equal keys survives is random
* intersection is not implemented
* simple get accessor is not implemented (but derivable from EachNeighbour)
@ -47,16 +64,17 @@ Pinned value on the node implies no need to copy keys of the item type.
Note that
* the same set of values allows for a large number of alternative
POT representations.
* values on the top are accessed faster than lower ones and the steps needed to retrieve items has a logarithmic distribution.
* values on the top are accessed faster than lower ones and the steps needed to
retrieve items has a logarithmic distribution.
As a consequence on can organise the tree so that items that need faster access are torwards the top.
In particular for any subset where popularity has a power distriution that is independent of
proximity order (content addressed storage of chunks), it is in principle possible to create a pot where the steps needed to access an item is inversely proportional to its popularity.
As a consequence one can organise the tree so that items that need faster access
are torwards the top. In particular for any subset where popularity has a power
distriution that is independent of proximity order (content addressed storage of
chunks), it is in principle possible to create a pot where the steps needed to
access an item is inversely proportional to its popularity.
Such organisation is not implemented as yet.
TODO:
* swap
* get
* overwrite-style merge
* intersection
* access frequency based optimisations

478
pot/pot.go
View file

@ -13,6 +13,8 @@
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Package pot see doc.go
package pot
import (
@ -21,114 +23,93 @@ import (
)
const (
keylen = 256
maxkeylen = 256
)
// Pot is the root node type, allows locked non-applicative manipulation
// Pot is the node type (same for root, branching node and leaf)
type Pot struct {
lock sync.RWMutex
*pot
}
// pot is the node type (same for root, branching node and leaf)
type pot struct {
pin PotVal
bins []*pot
pin Val
bins []*Pot
size int
po int
}
// PotVal is the interface the generic container item should implement
type PotVal interface {
PO(PotVal, int) (po int, eq bool)
String() string
}
// Val is the element type for Pots
type Val interface{}
type AnyVal interface{}
// Pof is the proximity order comparison operator function
type Pof func(Val, Val, int) (int, bool)
// Pot constructor. Requires value of type PotVal to pin
// and po to point to a span in the PotVal key
// NewPot constructor. Requires a value of type Val to pin
// and po to point to a span in the Val key
// The pinned item counts towards the size
func NewPot(v PotVal, po int) *Pot {
func NewPot(v Val, po int) *Pot {
var size int
if v != nil {
size++
}
return &Pot{
pot: &pot{
pin: v,
po: po,
size: size,
},
pin: v,
po: po,
size: size,
}
}
// Pin() returns the pinned element (key) of the Pot
func (t *Pot) Pin() PotVal {
// Pin returns the pinned element (key) of the Pot
func (t *Pot) Pin() Val {
return t.pin
}
// Size() returns the number of values in the Pot
// Size returns the number of values in the Pot
func (t *Pot) Size() int {
t.lock.RLock()
defer t.lock.RUnlock()
if t == nil {
return 0
}
return t.size
}
// Add(v) inserts v into the Pot and
// Add inserts a new value into the Pot and
// returns the proximity order of v and a boolean
// indicating if the item was found
// Add locks the Pot while using applicative add on its pot
func (t *Pot) Add(val PotVal) (po int, found bool) {
t.lock.Lock()
defer t.lock.Unlock()
t.pot, po, found = add(t.pot, val)
return po, found
}
// Add(t, v) returns a new Pot that contains all the elements of t
// Add called on (t, v) returns a new Pot that contains all the elements of t
// plus the value v, using the applicative add
// the second return value is the proximity order of the inserted element
// the third is boolean indicating if the item was found
// it only readlocks the Pot while reading its pot
func Add(t *Pot, val PotVal) (*Pot, int, bool) {
t.lock.RLock()
n := t.pot
t.lock.RUnlock()
r, po, found := add(n, val)
return &Pot{pot: r}, po, found
func Add(t *Pot, val Val, pof Pof) (*Pot, int, bool) {
return add(t, val, pof)
}
func add(t *pot, val PotVal) (*pot, int, bool) {
var r *pot
func (t *Pot) clone() *Pot {
return &Pot{
pin: t.pin,
size: t.size,
po: t.po,
bins: t.bins,
}
}
func add(t *Pot, val Val, pof Pof) (*Pot, int, bool) {
var r *Pot
if t == nil || t.pin == nil {
r = &pot{
pin: val,
size: t.size + 1,
po: t.po,
bins: t.bins,
}
r = t.clone()
r.pin = val
r.size++
return r, 0, false
}
po, found := t.pin.PO(val, t.po)
po, found := pof(t.pin, val, t.po)
if found {
r = &pot{
pin: val,
size: t.size,
po: t.po,
bins: t.bins,
}
r = t.clone()
r.pin = val
return r, po, true
}
var p *pot
var p *Pot
var i, j int
size := t.size
for i < len(t.bins) {
n := t.bins[i]
if n.po == po {
p, _, found = add(n, val)
p, _, found = add(n, val, pof)
if !found {
size++
}
@ -143,17 +124,17 @@ func add(t *pot, val PotVal) (*pot, int, bool) {
}
if p == nil {
size++
p = &pot{
p = &Pot{
pin: val,
size: 1,
po: po,
}
}
bins := append([]*pot{}, t.bins[:i]...)
bins := append([]*Pot{}, t.bins[:i]...)
bins = append(bins, p)
bins = append(bins, t.bins[j:]...)
r = &pot{
r = &Pot{
pin: t.pin,
size: size,
po: t.po,
@ -163,44 +144,31 @@ func add(t *pot, val PotVal) (*pot, int, bool) {
return r, po, found
}
// T.Re move(v) deletes v from the Pot and returns
// Remove called on (v) deletes v from the Pot and returns
// the proximity order of v and a boolean value indicating
// if the value was found
// Remove locks Pot while using applicative remove on its pot
func (t *Pot) Remove(val PotVal) (po int, found bool) {
t.lock.Lock()
defer t.lock.Unlock()
t.pot, po, found = remove(t.pot, val)
return po, found
}
// Remove(t, v) returns a new Pot that contains all the elements of t
// Remove called on (t, v) returns a new Pot that contains all the elements of t
// minus the value v, using the applicative remove
// the second return value is the proximity order of the inserted element
// the third is boolean indicating if the item was found
// it only readlocks the Pot while reading its pot
func Remove(t *Pot, v PotVal) (*Pot, int, bool) {
t.lock.RLock()
n := t.pot
t.lock.RUnlock()
r, po, found := remove(n, v)
return &Pot{pot: r}, po, found
func Remove(t *Pot, v Val, pof Pof) (*Pot, int, bool) {
return remove(t, v, pof)
}
func remove(t *pot, val PotVal) (r *pot, po int, found bool) {
func remove(t *Pot, val Val, pof Pof) (r *Pot, po int, found bool) {
size := t.size
po, found = t.pin.PO(val, t.po)
po, found = pof(t.pin, val, t.po)
if found {
size--
if size == 0 {
r = &pot{
r = &Pot{
po: t.po,
}
return r, po, true
}
i := len(t.bins) - 1
last := t.bins[i]
r = &pot{
r = &Pot{
pin: last.pin,
bins: append(t.bins[:i], last.bins...),
size: size,
@ -209,12 +177,12 @@ func remove(t *pot, val PotVal) (r *pot, po int, found bool) {
return r, t.po, true
}
var p *pot
var p *Pot
var i, j int
for i < len(t.bins) {
n := t.bins[i]
if n.po == po {
p, po, found = remove(n, val)
p, po, found = remove(n, val, pof)
if found {
size--
}
@ -232,7 +200,7 @@ func remove(t *pot, val PotVal) (r *pot, po int, found bool) {
bins = append(bins, p)
}
bins = append(bins, t.bins[j:]...)
r = &pot{
r = &Pot{
pin: val,
size: size,
po: t.po,
@ -241,169 +209,130 @@ func remove(t *pot, val PotVal) (r *pot, po int, found bool) {
return r, po, found
}
// Swap(k, f) looks up the item at k
// and applies the function f to the value v at k or nil if the item is not found
// if f returns nil, the element is removed
// if f returns v' <> v then v' is inserted into the Pot
// if v' == v the pot is not changed
// it panics if v'.PO(k, 0) says v and k are not equal
func (t *Pot) Swap(val AnyVal, f func(v PotVal) PotVal) (po int, found bool, change bool) {
t.lock.Lock()
defer t.lock.Unlock()
ba := NewBytesVal(val, nil)
var t0 *pot
t0, po, found, change = swap(t.pot, ba, f)
if change {
t.pot = t0
}
return po, found, change
}
func swap(t *pot, k PotVal, f func(v PotVal) PotVal) (r *pot, po int, found bool, change bool) {
var val PotVal
if t == nil || t.pin == nil {
// Swap called on (k, f) looks up the item at k
// and applies the function f to the value v at k or to nil if the item is not found
// if f(v) returns nil, the element is removed
// if f(v) returns v' <> v then v' is inserted into the Pot
// if (v) == v the Pot is not changed
// it panics if Pof(f(v), k) show that v' and v are not key-equal
func Swap(t *Pot, k Val, pof Pof, f func(v Val) Val) (r *Pot, po int, found bool, change bool) {
var val Val
if t.pin == nil {
val = f(nil)
if val == nil {
return t, t.po, false, false
return nil, 0, false, false
}
if _, eq := val.PO(k, t.po); !eq {
panic("value key mismatch")
}
r = &pot{
pin: val,
size: t.size + 1,
po: t.po,
bins: t.bins,
}
return r, t.po, false, true
return NewPot(val, t.po), 0, false, true
}
size := t.size
if k == nil {
panic("k is nil")
}
po, found = k.PO(t.pin, t.po)
po, found = pof(k, t.pin, t.po)
if found {
val = f(t.pin)
// remove element
if val == nil {
size--
if size == 0 {
r = &pot{
r = &Pot{
po: t.po,
}
// return empty pot
return r, po, true, true
}
// actually remove pin, by merging last bin
i := len(t.bins) - 1
last := t.bins[i]
r = &pot{
r = &Pot{
pin: last.pin,
bins: append(t.bins[:i], last.bins...),
size: size,
po: t.po,
}
return r, t.po, true, true
// remove element
} else if val == t.pin {
return nil, po, true, false
} else { // add element
r = &pot{
pin: val,
size: t.size,
po: t.po,
bins: t.bins,
}
return r, po, true, true
}
// element found but no change
if val == t.pin {
return t, po, true, false
}
// actually modify the pinned element, but no change in structure
r = t.clone()
r.pin = val
return r, po, true, true
}
var p *pot
var i, j int
for i < len(t.bins) {
n := t.bins[i]
if n.po == po {
p, po, found, change = swap(n, k, f)
if !change {
return nil, po, found, false
}
size += p.size - n.size
j++
break
// recursive step
var p *Pot
n, i := t.getPos(po)
if n != nil {
p, po, found, change = Swap(n, k, pof, f)
// recursive no change
if !change {
return t, po, found, false
}
if n.po > po {
break
// recursive change
bins := append([]*Pot{}, t.bins[:i]...)
if p.size == 0 {
size--
} else {
size += p.size - n.size
bins = append(bins, p)
}
i++
j++
if i < len(t.bins) {
bins = append(bins, t.bins[i:]...)
}
r = t.clone()
r.bins = bins
r.size = size
return r, po, found, true
}
if p == nil {
val := f(nil)
if val == nil {
return nil, po, false, false
}
size++
p = &pot{
pin: val,
size: 1,
po: po,
}
// key does not exist
val = f(nil)
if val == nil {
// and it should not be created
return t, po, false, false
}
// otherwise check val if equal to k
if _, eq := pof(val, k, po); !eq {
panic("invalid value")
}
///
size++
p = &Pot{
pin: val,
size: 1,
po: po,
}
bins := append([]*pot{}, t.bins[:i]...)
if p.pin != nil {
bins = append(bins, p)
bins := append([]*Pot{}, t.bins[:i]...)
bins = append(bins, p)
if i < len(t.bins) {
bins = append(bins, t.bins[i:]...)
}
bins = append(bins, t.bins[j:]...)
r = &pot{
pin: t.pin,
size: size,
po: t.po,
bins: bins,
}
r = t.clone()
r.bins = bins
r.size = size
return r, po, found, true
}
// t0.Merge(t1) changes t0 to contain all the elements of t1
// it locks t0, but only readlocks t1 while taking its pot
// uses applicative union
func (t *Pot) Merge(t1 *Pot) (c int) {
t.lock.Lock()
defer t.lock.Unlock()
t1.lock.RLock()
n1 := t1.pot
t1.lock.RUnlock()
t.pot, c = union(t.pot, n1)
return c
}
// Union returns the union of t0 and t1
// it only readlocks the Pot-s to read their pots and
// Union called on (t0, t1, pof) returns the union of t0 and t1
// calculates the union using the applicative union
// the second return value is the number of common elements
func Union(t0, t1 *Pot) (*Pot, int) {
t0.lock.RLock()
n0 := t0.pot
t0.lock.RUnlock()
t1.lock.RLock()
n1 := t1.pot
t1.lock.RUnlock()
p, c := union(n0, n1)
return &Pot{
pot: p,
}, c
func Union(t0, t1 *Pot, pof Pof) (*Pot, int) {
return union(t0, t1, pof)
}
func union(t0, t1 *pot) (*pot, int) {
func union(t0, t1 *Pot, pof Pof) (*Pot, int) {
if t0 == nil || t0.size == 0 {
return t1, 0
}
if t1 == nil || t1.size == 0 {
return t0, 0
}
var pin PotVal
var bins []*pot
var pin Val
var bins []*Pot
var mis []int
wg := &sync.WaitGroup{}
wg.Add(1)
pin0 := t0.pin
pin1 := t1.pin
bins0 := t0.bins
@ -411,12 +340,12 @@ func union(t0, t1 *pot) (*pot, int) {
var i0, i1 int
var common int
po, eq := pin0.PO(pin1, 0)
po, eq := pof(pin0, pin1, 0)
for {
l0 := len(bins0)
l1 := len(bins1)
var n0, n1 *pot
var n0, n1 *Pot
var p0, p1 int
var a0, a1 bool
@ -455,11 +384,12 @@ func union(t0, t1 *pot) (*pot, int) {
bins = append(bins, nil)
ml := len(mis)
mis = append(mis, 0)
wg.Add(1)
go func(b, m int, m0, m1 *pot) {
defer wg.Done()
bins[b], mis[m] = union(m0, m1)
}(bl, ml, n0, n1)
// wg.Add(1)
// go func(b, m int, m0, m1 *Pot) {
// defer wg.Done()
// bins[b], mis[m] = union(m0, m1, pof)
// }(bl, ml, n0, n1)
bins[bl], mis[ml] = union(n0, n1, pof)
i0++
i1++
n0 = nil
@ -481,14 +411,14 @@ func union(t0, t1 *pot) (*pot, int) {
for _, n := range bins0[i:] {
size0 += n.size
}
np := &pot{
np := &Pot{
pin: pin0,
bins: bins0[i:],
size: size0 + 1,
po: po,
}
bins2 := []*pot{np}
bins2 := []*Pot{np}
if n0 == nil {
pin0 = pin1
po = maxkeylen + 1
@ -499,7 +429,7 @@ func union(t0, t1 *pot) (*pot, int) {
bins2 = append(bins2, n0.bins...)
pin0 = pin1
pin1 = n0.pin
po, eq = pin0.PO(pin1, n0.po)
po, eq = pof(pin0, pin1, n0.po)
}
bins0 = bins1
@ -509,11 +439,12 @@ func union(t0, t1 *pot) (*pot, int) {
}
wg.Done()
wg.Wait()
for _, c := range mis {
common += c
}
n := &pot{
n := &Pot{
pin: pin,
bins: bins,
size: t0.size + t1.size - common,
@ -522,17 +453,14 @@ func union(t0, t1 *pot) (*pot, int) {
return n, common
}
// Each(f) is a synchronous iterator over the bins of a node
// Each called with (f) is a synchronous iterator over the bins of a node
// respecting an ordering
// proximity > pinnedness
func (t *Pot) Each(f func(PotVal, int) bool) bool {
t.lock.RLock()
n := t.pot
t.lock.RUnlock()
return n.each(f)
func (t *Pot) Each(f func(Val, int) bool) bool {
return t.each(f)
}
func (t *pot) each(f func(PotVal, int) bool) bool {
func (t *Pot) each(f func(Val, int) bool) bool {
var next bool
for _, n := range t.bins {
if n == nil {
@ -543,10 +471,13 @@ func (t *pot) each(f func(PotVal, int) bool) bool {
return false
}
}
if t.size == 0 {
return false
}
return f(t.pin, t.po)
}
// EachFrom(f, start) is a synchronous iterator over the elements of a pot
// EachFrom called with (f, start) is a synchronous iterator over the elements of a Pot
// within the inclusive range starting from proximity order start
// the function argument is passed the value and the proximity order wrt the root pin
// it does NOT include the pinned item of the root
@ -554,14 +485,11 @@ func (t *pot) each(f func(PotVal, int) bool) bool {
// proximity > pinnedness
// the iteration ends if the function return false or there are no more elements
// end of a po range can be implemented since po is passed to the function
func (t *Pot) EachFrom(f func(PotVal, int) bool, po int) bool {
t.lock.RLock()
n := t.pot
t.lock.RUnlock()
return n.eachFrom(f, po)
func (t *Pot) EachFrom(f func(Val, int) bool, po int) bool {
return t.eachFrom(f, po)
}
func (t *pot) eachFrom(f func(PotVal, int) bool, po int) bool {
func (t *Pot) eachFrom(f func(Val, int) bool, po int) bool {
var next bool
_, lim := t.getPos(po)
for i := lim; i < len(t.bins); i++ {
@ -578,21 +506,18 @@ func (t *pot) eachFrom(f func(PotVal, int) bool, po int) bool {
// subtree passing the proximity order and the size
// the iteration continues until the function's return value is false
// or there are no more subtries
func (t *Pot) EachBin(val PotVal, po int, f func(int, int, func(func(val PotVal, i int) bool) bool) bool) {
t.lock.RLock()
n := t.pot
t.lock.RUnlock()
n.eachBin(val, po, f)
func (t *Pot) EachBin(val Val, pof Pof, po int, f func(int, int, func(func(val Val, i int) bool) bool) bool) {
t.eachBin(val, pof, po, f)
}
func (t *pot) eachBin(val PotVal, po int, f func(int, int, func(func(val PotVal, i int) bool) bool) bool) {
func (t *Pot) eachBin(val Val, pof Pof, po int, f func(int, int, func(func(val Val, i int) bool) bool) bool) {
if t == nil || t.size == 0 {
return
}
spr, _ := t.pin.PO(val, t.po)
spr, _ := pof(t.pin, val, t.po)
_, lim := t.getPos(spr)
var size int
var n *pot
var n *Pot
for i := 0; i < lim; i++ {
n = t.bins[i]
size += n.size
@ -604,7 +529,7 @@ func (t *pot) eachBin(val PotVal, po int, f func(int, int, func(func(val PotVal,
}
}
if lim == len(t.bins) {
f(spr, 1, func(g func(PotVal, int) bool) bool {
f(spr, 1, func(g func(Val, int) bool) bool {
return g(t.pin, spr)
})
return
@ -616,42 +541,39 @@ func (t *pot) eachBin(val PotVal, po int, f func(int, int, func(func(val PotVal,
spo++
size += n.size
}
if !f(spr, t.size-size, func(g func(PotVal, int) bool) bool {
return t.eachFrom(func(v PotVal, j int) bool {
if !f(spr, t.size-size, func(g func(Val, int) bool) bool {
return t.eachFrom(func(v Val, j int) bool {
return g(v, spr)
}, spo)
}) {
return
}
if spo > spr {
n.eachBin(val, spo, f)
n.eachBin(val, pof, spo, f)
}
}
// syncronous iterator over neighbours of any target val
// EachNeighbour is a syncronous iterator over neighbours of any target val
// the order of elements retrieved reflect proximity order to the target
// TODO: add maximum proxbin to start range of iteration
func (t *Pot) EachNeighbour(val PotVal, f func(PotVal, int) bool) bool {
t.lock.RLock()
n := t.pot
t.lock.RUnlock()
return n.eachNeighbour(val, f)
func (t *Pot) EachNeighbour(val Val, pof Pof, f func(Val, int) bool) bool {
return t.eachNeighbour(val, pof, f)
}
func (t *pot) eachNeighbour(val PotVal, f func(PotVal, int) bool) bool {
func (t *Pot) eachNeighbour(val Val, pof Pof, f func(Val, int) bool) bool {
if t == nil || t.size == 0 {
return false
}
var next bool
l := len(t.bins)
var n *pot
var n *Pot
ir := l
il := l
po, eq := t.pin.PO(val, t.po)
po, eq := pof(t.pin, val, t.po)
if !eq {
n, il = t.getPos(po)
if n != nil {
next = n.eachNeighbour(val, f)
next = n.eachNeighbour(val, pof, f)
if !next {
return false
}
@ -667,7 +589,7 @@ func (t *pot) eachNeighbour(val PotVal, f func(PotVal, int) bool) bool {
}
for i := l - 1; i > ir; i-- {
next = t.bins[i].each(func(v PotVal, _ int) bool {
next = t.bins[i].each(func(v Val, _ int) bool {
return f(v, po)
})
if !next {
@ -677,7 +599,7 @@ func (t *pot) eachNeighbour(val PotVal, f func(PotVal, int) bool) bool {
for i := il - 1; i >= 0; i-- {
n := t.bins[i]
next = n.each(func(v PotVal, _ int) bool {
next = n.each(func(v Val, _ int) bool {
return f(v, n.po)
})
if !next {
@ -687,21 +609,18 @@ func (t *pot) eachNeighbour(val PotVal, f func(PotVal, int) bool) bool {
return true
}
// EachNeighnbourAsync(val, max, maxPos, f, wait) is an asyncronous iterator
// EachNeighbourAsync called on (val, max, maxPos, f, wait) is an asyncronous iterator
// over elements not closer than maxPos wrt val.
// val does not need to be match an element of the pot, but if it does, and
// val does not need to be match an element of the Pot, but if it does, and
// maxPos is keylength than it is included in the iteration
// Calls to f are parallelised, the order of calls is undefined.
// proximity order is respected in that there is no element in the pot that
// proximity order is respected in that there is no element in the Pot that
// is not visited if a closer node is visited.
// The iteration is finished when max number of nearest nodes is visited
// or if the entire there are no nodes not closer than maxPos that is not visited
// if wait is true, the iterator returns only if all calls to f are finished
// TODO: implement minPos for proper prox range iteration
func (t *Pot) EachNeighbourAsync(val PotVal, max int, maxPos int, f func(PotVal, int), wait bool) {
t.lock.RLock()
n := t.pot
t.lock.RUnlock()
func (t *Pot) EachNeighbourAsync(val Val, pof Pof, max int, maxPos int, f func(Val, int), wait bool) {
if max > t.size {
max = t.size
}
@ -709,29 +628,24 @@ func (t *Pot) EachNeighbourAsync(val PotVal, max int, maxPos int, f func(PotVal,
if wait {
wg = &sync.WaitGroup{}
}
_ = n.eachNeighbourAsync(val, max, maxPos, f, wg)
t.eachNeighbourAsync(val, pof, max, maxPos, f, wg)
if wait {
wg.Wait()
}
}
func (t *pot) eachNeighbourAsync(val PotVal, max int, maxPos int, f func(PotVal, int), wg *sync.WaitGroup) (extra int) {
func (t *Pot) eachNeighbourAsync(val Val, pof Pof, max int, maxPos int, f func(Val, int), wg *sync.WaitGroup) (extra int) {
l := len(t.bins)
var n *pot
il := l
ir := l
// ic := l
po, eq := t.pin.PO(val, t.po)
po, eq := pof(t.pin, val, t.po)
// if po is too close, set the pivot branch (pom) to maxPos
pom := po
if pom > maxPos {
pom = maxPos
}
n, il = t.getPos(pom)
ir = il
n, il := t.getPos(pom)
ir := il
// if pivot branch exists and po is not too close, iterate on the pivot branch
if pom == po {
if n != nil {
@ -742,7 +656,7 @@ func (t *pot) eachNeighbourAsync(val PotVal, max int, maxPos int, f func(PotVal,
}
max -= m
extra = n.eachNeighbourAsync(val, m, maxPos, f, wg)
extra = n.eachNeighbourAsync(val, pof, m, maxPos, f, wg)
} else {
if !eq {
@ -798,8 +712,8 @@ func (t *pot) eachNeighbourAsync(val PotVal, max int, maxPos int, f func(PotVal,
if wg != nil {
wg.Add(m)
}
go func(pn *pot, pm int) {
pn.each(func(v PotVal, _ int) bool {
go func(pn *Pot, pm int) {
pn.each(func(v Val, _ int) bool {
if wg != nil {
defer wg.Done()
}
@ -825,8 +739,8 @@ func (t *pot) eachNeighbourAsync(val PotVal, max int, maxPos int, f func(PotVal,
if wg != nil {
wg.Add(m)
}
go func(pn *pot, pm int) {
pn.each(func(v PotVal, _ int) bool {
go func(pn *Pot, pm int) {
pn.each(func(v Val, _ int) bool {
if wg != nil {
defer wg.Done()
}
@ -840,10 +754,10 @@ func (t *pot) eachNeighbourAsync(val PotVal, max int, maxPos int, f func(PotVal,
return max + extra
}
// getPos(n) returns the forking node at PO n and its index if it exists
// getPos called on (n) returns the forking node at PO n and its index if it exists
// otherwise nil
// caller is suppoed to hold the lock
func (t *pot) getPos(po int) (n *pot, i int) {
func (t *Pot) getPos(po int) (n *Pot, i int) {
for i, n = range t.bins {
if po > n.po {
continue
@ -856,7 +770,7 @@ func (t *pot) getPos(po int) (n *pot, i int) {
return nil, len(t.bins)
}
// need(m, max, extra) uses max m out of extra, and then max
// need called on (m, max, extra) uses max m out of extra, and then max
// if needed, returns the adjusted counts
func need(m, max, extra int) (int, int, int) {
if m <= extra {
@ -869,11 +783,11 @@ func need(m, max, extra int) (int, int, int) {
return m, max, 0
}
func (t *pot) String() string {
func (t *Pot) String() string {
return t.sstring("")
}
func (t *pot) sstring(indent string) string {
func (t *Pot) sstring(indent string) string {
if t == nil {
return "<nil>"
}

387
pot/pot_test.go
View file

@ -19,7 +19,6 @@ import (
"errors"
"fmt"
"math/rand"
"os"
"runtime"
"sync"
"testing"
@ -31,61 +30,43 @@ import (
const (
maxEachNeighbourTests = 420
maxEachNeighbour = 420
maxSwap = 420
maxSwapTests = 420
)
func init() {
log.Root().SetHandler(log.LvlFilterHandler(log.LvlError, log.StreamHandler(os.Stderr, log.TerminalFormat(false))))
}
type testBVAddr struct {
*HashAddress
i int
}
func NewTestBVAddr(s string, i int) *testBVAddr {
return &testBVAddr{NewHashAddress(s), i}
}
func (a *testBVAddr) String() string {
return a.HashAddress.String()[:keylen]
}
func (self *testBVAddr) PO(val PotVal, po int) (int, bool) {
return self.HashAddress.PO(val.(*testBVAddr).HashAddress, po)
}
// func init() {
// log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(false))))
// }
type testAddr struct {
*BoolAddress
a []byte
i int
}
func NewTestAddr(s string, i int) *testAddr {
return &testAddr{NewBoolAddress(s), i}
func newTestAddr(s string, i int) *testAddr {
return &testAddr{NewAddressFromString(s), i}
}
func (self *testAddr) PO(val PotVal, po int) (int, bool) {
return self.BoolAddress.PO(val.(*testAddr).BoolAddress, po)
func (a *testAddr) Address() []byte {
return a.a
}
func str(v PotVal) string {
if v == nil {
return ""
}
return v.(*testAddr).String()
func (a *testAddr) String() string {
return Label(a.a)
}
func randomTestAddr(n int, i int) *testAddr {
v := RandomAddress().Bin()[:n]
return NewTestAddr(v, i)
return newTestAddr(v, i)
}
func randomTestBVAddr(n int, i int) *testBVAddr {
func randomtestAddr(n int, i int) *testAddr {
v := RandomAddress().Bin()[:n]
return NewTestBVAddr(v, i)
return newTestAddr(v, i)
}
func indexes(t *Pot) (i []int, po []int) {
t.Each(func(v PotVal, p int) bool {
t.Each(func(v Val, p int) bool {
a := v.(*testAddr)
i = append(i, a.i)
po = append(po, p)
@ -94,18 +75,19 @@ func indexes(t *Pot) (i []int, po []int) {
return i, po
}
func testAdd(t *Pot, n int, values ...string) {
func testAdd(t *Pot, pof Pof, j int, values ...string) (_ *Pot, n int, f bool) {
for i, val := range values {
t.Add(NewTestAddr(val, i+n))
t, n, f = Add(t, newTestAddr(val, i+j), pof)
}
return t, n, f
}
// func RandomBoolAddress()
func TestPotAdd(t *testing.T) {
n := NewPot(NewTestAddr("001111", 0), 0)
pof := DefaultPof(8)
n := NewPot(newTestAddr("00111100", 0), 0)
// Pin set correctly
exp := "001111"
got := str(n.Pin())[:6]
exp := "00111100"
got := Label(n.Pin())[:8]
if got != exp {
t.Fatalf("incorrect pinned value. Expected %v, got %v", exp, got)
}
@ -116,7 +98,7 @@ func TestPotAdd(t *testing.T) {
t.Fatalf("incorrect number of elements in Pot. Expected %v, got %v", expi, goti)
}
testAdd(n, 1, "011111", "001111", "011111", "000111")
n, _, _ = testAdd(n, pof, 1, "01111100", "00111100", "01111100", "00011100")
// check size
goti = n.Size()
expi = 3
@ -136,17 +118,17 @@ func TestPotAdd(t *testing.T) {
}
}
// func RandomBoolAddress()
func TestPotRemove(t *testing.T) {
n := NewPot(NewTestAddr("001111", 0), 0)
n.Remove(NewTestAddr("001111", 0))
exp := ""
got := str(n.Pin())
pof := DefaultPof(8)
n := NewPot(newTestAddr("00111100", 0), 0)
n, _, _ = Remove(n, newTestAddr("00111100", 0), pof)
exp := "<nil>"
got := Label(n.Pin())
if got != exp {
t.Fatalf("incorrect pinned value. Expected %v, got %v", exp, got)
}
testAdd(n, 1, "000000", "011111", "001111", "000111")
n.Remove(NewTestAddr("001111", 0))
n, _, _ = testAdd(n, pof, 1, "00000000", "01111100", "00111100", "00011100")
n, _, _ = Remove(n, newTestAddr("00111100", 0), pof)
goti := n.Size()
expi := 3
if goti != expi {
@ -164,8 +146,8 @@ func TestPotRemove(t *testing.T) {
t.Fatalf("incorrect po-s in iteration over Pot. Expected %v, got %v", exp, got)
}
// remove again
n.Remove(NewTestAddr("001111", 0))
inds, po = indexes(n)
n, _, _ = Remove(n, newTestAddr("00111100", 0), pof)
inds, _ = indexes(n)
got = fmt.Sprintf("%v", inds)
exp = "[2 4]"
if got != exp {
@ -175,66 +157,75 @@ func TestPotRemove(t *testing.T) {
}
func TestPotSwap(t *testing.T) {
max := maxEachNeighbour
n := NewPot(nil, 0)
var m []*testBVAddr
for j := 0; j < 2*max; {
v := randomTestBVAddr(keylen, j)
_, found := n.Add(v)
if !found {
m = append(m, v)
j++
}
}
k := make(map[string]*testBVAddr)
for j := 0; j < max; {
v := randomTestBVAddr(keylen, 1)
_, found := k[v.String()]
if !found {
k[v.String()] = v
j++
}
}
for _, v := range k {
m = append(m, v)
}
f := func(v PotVal) PotVal {
tv := v.(*testBVAddr)
if tv.i < max {
return nil
}
tv.i = 0
return v
}
for _, val := range m {
n.Swap(val, func(v PotVal) PotVal {
if v == nil {
return val
for i := 0; i < maxSwapTests; i++ {
alen := maxkeylen
pof := DefaultPof(alen)
max := rand.Intn(maxSwap)
n := NewPot(nil, 0)
var m []*testAddr
var found bool
for j := 0; j < 2*max; {
v := randomtestAddr(alen, j)
n, _, found = Add(n, v, pof)
if !found {
m = append(m, v)
j++
}
return f(v)
})
}
sum := 0
n.Each(func(v PotVal, i int) bool {
sum++
tv := v.(*testBVAddr)
if tv.i > 1 {
t.Fatalf("item value incorrect, expected 0, got %v", tv.i)
}
return true
})
if sum != 2*max {
t.Fatalf("incorrect number of elements. expected %v, got %v", 2*max, sum)
}
if sum != n.Size() {
t.Fatalf("incorrect size. expected %v, got %v", sum, n.Size())
k := make(map[string]*testAddr)
for j := 0; j < max; {
v := randomtestAddr(alen, 1)
_, found := k[Label(v)]
if !found {
k[Label(v)] = v
j++
}
}
for _, v := range k {
m = append(m, v)
}
f := func(v Val) Val {
tv := v.(*testAddr)
if tv.i < max {
return nil
}
tv.i = 0
return v
}
for _, val := range m {
n, _, _, _ = Swap(n, val, pof, func(v Val) Val {
if v == nil {
return val
}
return f(v)
})
}
sum := 0
n.Each(func(v Val, i int) bool {
if v == nil {
return true
}
sum++
tv := v.(*testAddr)
if tv.i > 1 {
t.Fatalf("item value incorrect, expected 0, got %v", tv.i)
}
return true
})
if sum != 2*max {
t.Fatalf("incorrect number of elements. expected %v, got %v", 2*max, sum)
}
if sum != n.Size() {
t.Fatalf("incorrect size. expected %v, got %v", sum, n.Size())
}
}
}
func checkPo(val PotVal) func(PotVal, int) error {
return func(v PotVal, po int) error {
func checkPo(val Val, pof Pof) func(Val, int) error {
return func(v Val, po int) error {
// check the po
exp, _ := val.PO(v, 0)
exp, _ := pof(val, v, 0)
if po != exp {
return fmt.Errorf("incorrect prox order for item %v in neighbour iteration for %v. Expected %v, got %v", v, val, exp, po)
}
@ -242,9 +233,9 @@ func checkPo(val PotVal) func(PotVal, int) error {
}
}
func checkOrder(val PotVal) func(PotVal, int) error {
var po int = keylen
return func(v PotVal, p int) error {
func checkOrder(val Val) func(Val, int) error {
po := maxkeylen
return func(v Val, p int) error {
if po < p {
return fmt.Errorf("incorrect order for item %v in neighbour iteration for %v. PO %v > %v (previous max)", v, val, p, po)
}
@ -253,26 +244,26 @@ func checkOrder(val PotVal) func(PotVal, int) error {
}
}
func checkValues(m map[string]bool, val PotVal) func(PotVal, int) error {
return func(v PotVal, po int) error {
duplicate, ok := m[v.String()]
func checkValues(m map[string]bool, val Val) func(Val, int) error {
return func(v Val, po int) error {
duplicate, ok := m[Label(v)]
if !ok {
return fmt.Errorf("alien value %v", v)
}
if duplicate {
return fmt.Errorf("duplicate value returned: %v", v)
}
m[v.String()] = true
m[Label(v)] = true
return nil
}
}
var errNoCount = errors.New("not count")
func testPotEachNeighbour(n *Pot, val PotVal, expCount int, fs ...func(PotVal, int) error) error {
func testPotEachNeighbour(n *Pot, pof Pof, val Val, expCount int, fs ...func(Val, int) error) error {
var err error
var count int
n.EachNeighbour(val, func(v PotVal, po int) bool {
n.EachNeighbour(val, pof, func(v Val, po int) bool {
for _, f := range fs {
err = f(v, po)
if err != nil {
@ -296,30 +287,14 @@ const (
mergeTestChoose = 5
)
func TestPotMergeOne(t *testing.T) {
pot1 := NewPot(nil, 0)
pot1.Add(NewTestAddr("10", 0))
pot1.Add(NewTestAddr("00", 0))
pot2 := NewPot(nil, 0)
pot2.Add(NewTestAddr("01", 0))
log.Debug(fmt.Sprintf("\n%v\n%v", pot2, pot1))
pot1.Merge(pot2)
count := 0
pot1.Each(func(val PotVal, i int) bool {
count++
return true
})
if count != 3 {
t.Fatalf("expected count to be 3, got %d\n%v\n%v", count, pot2, pot1)
}
}
func TestPotMergeCommon(t *testing.T) {
vs := make([]*testBVAddr, mergeTestCount)
vs := make([]*testAddr, mergeTestCount)
for i := 0; i < maxEachNeighbourTests; i++ {
alen := maxkeylen
pof := DefaultPof(alen)
for i := 0; i < len(vs); i++ {
vs[i] = randomTestBVAddr(keylen, i)
for j := 0; j < len(vs); j++ {
vs[j] = randomtestAddr(alen, j)
}
max0 := rand.Intn(mergeTestChoose) + 1
max1 := rand.Intn(mergeTestChoose) + 1
@ -327,13 +302,13 @@ func TestPotMergeCommon(t *testing.T) {
n1 := NewPot(nil, 0)
log.Trace(fmt.Sprintf("round %v: %v - %v", i, max0, max1))
m := make(map[string]bool)
var found bool
for j := 0; j < max0; {
r := rand.Intn(max0)
v := vs[r]
// v := randomTestBVAddr(keylen, j)
_, found := n0.Add(v)
n0, _, found = Add(n0, v, pof)
if !found {
m[v.String()] = false
m[Label(v)] = false
j++
}
}
@ -342,14 +317,14 @@ func TestPotMergeCommon(t *testing.T) {
for j := 0; j < max1; {
r := rand.Intn(max1)
v := vs[r]
_, found := n1.Add(v)
n1, _, found = Add(n1, v, pof)
if !found {
j++
}
_, found = m[v.String()]
_, found = m[Label(v)]
if !found {
expAdded++
m[v.String()] = false
m[Label(v)] = false
}
}
if i < 6 {
@ -359,7 +334,7 @@ func TestPotMergeCommon(t *testing.T) {
log.Trace(fmt.Sprintf("%v-0: pin: %v, size: %v", i, n0.Pin(), max0))
log.Trace(fmt.Sprintf("%v-1: pin: %v, size: %v", i, n1.Pin(), max1))
log.Trace(fmt.Sprintf("%v: merged tree size: %v, newly added: %v", i, expSize, expAdded))
n, common := Union(n0, n1)
n, common := Union(n0, n1, pof)
added := n1.Size() - common
size := n.Size()
@ -369,13 +344,13 @@ func TestPotMergeCommon(t *testing.T) {
if expAdded != added {
t.Fatalf("%v: incorrect number of added elements in merged pot, expected %v, got %v", i, expAdded, added)
}
if !checkDuplicates(n.pot) {
if !checkDuplicates(n) {
t.Fatalf("%v: merged pot contains duplicates: \n%v", i, n)
}
for k, _ := range m {
_, found := n.Add(NewTestBVAddr(k, 0))
for k := range m {
_, _, found = Add(n, newTestAddr(k, 0), pof)
if !found {
t.Fatalf("%v: merged pot (size:%v, added: %v) missing element %v\n%v", i, size, added, k, n)
t.Fatalf("%v: merged pot (size:%v, added: %v) missing element %v", i, size, added, k)
}
}
}
@ -383,34 +358,35 @@ func TestPotMergeCommon(t *testing.T) {
func TestPotMergeScale(t *testing.T) {
for i := 0; i < maxEachNeighbourTests; i++ {
alen := maxkeylen
pof := DefaultPof(alen)
max0 := rand.Intn(maxEachNeighbour) + 1
max1 := rand.Intn(maxEachNeighbour) + 1
n0 := NewPot(nil, 0)
n1 := NewPot(nil, 0)
log.Trace(fmt.Sprintf("round %v: %v - %v", i, max0, max1))
m := make(map[string]bool)
var found bool
for j := 0; j < max0; {
v := randomTestBVAddr(keylen, j)
// v := randomTestBVAddr(keylen, j)
_, found := n0.Add(v)
v := randomtestAddr(alen, j)
n0, _, found = Add(n0, v, pof)
if !found {
m[v.String()] = false
m[Label(v)] = false
j++
}
}
expAdded := 0
for j := 0; j < max1; {
v := randomTestBVAddr(keylen, j)
// v := randomTestBVAddr(keylen, j)
_, found := n1.Add(v)
v := randomtestAddr(alen, j)
n1, _, found = Add(n1, v, pof)
if !found {
j++
}
_, found = m[v.String()]
_, found = m[Label(v)]
if !found {
expAdded++
m[v.String()] = false
m[Label(v)] = false
}
}
if i < 6 {
@ -420,30 +396,30 @@ func TestPotMergeScale(t *testing.T) {
log.Trace(fmt.Sprintf("%v-0: pin: %v, size: %v", i, n0.Pin(), max0))
log.Trace(fmt.Sprintf("%v-1: pin: %v, size: %v", i, n1.Pin(), max1))
log.Trace(fmt.Sprintf("%v: merged tree size: %v, newly added: %v", i, expSize, expAdded))
n, common := Union(n0, n1)
n, common := Union(n0, n1, pof)
added := n1.Size() - common
size := n.Size()
if expSize != size {
t.Fatalf("%v: incorrect number of elements in merged pot, expected %v, got %v\n%v", i, expSize, size, n)
t.Fatalf("%v: incorrect number of elements in merged pot, expected %v, got %v", i, expSize, size)
}
if expAdded != added {
t.Fatalf("%v: incorrect number of added elements in merged pot, expected %v, got %v", i, expAdded, added)
}
if !checkDuplicates(n.pot) {
t.Fatalf("%v: merged pot contains duplicates: \n%v", i, n)
if !checkDuplicates(n) {
t.Fatalf("%v: merged pot contains duplicates", i)
}
for k, _ := range m {
_, found := n.Add(NewTestBVAddr(k, 0))
for k := range m {
_, _, found = Add(n, newTestAddr(k, 0), pof)
if !found {
t.Fatalf("%v: merged pot (size:%v, added: %v) missing element %v\n%v", i, size, added, k, n)
t.Fatalf("%v: merged pot (size:%v, added: %v) missing element %v", i, size, added, k)
}
}
}
}
func checkDuplicates(t *pot) bool {
var po int = -1
func checkDuplicates(t *Pot) bool {
po := -1
for _, c := range t.bins {
if c == nil {
return false
@ -458,15 +434,17 @@ func checkDuplicates(t *pot) bool {
func TestPotEachNeighbourSync(t *testing.T) {
for i := 0; i < maxEachNeighbourTests; i++ {
alen := maxkeylen
pof := DefaultPof(maxkeylen)
max := rand.Intn(maxEachNeighbour/2) + maxEachNeighbour/2
pin := randomTestAddr(keylen, 0)
pin := randomTestAddr(alen, 0)
n := NewPot(pin, 0)
m := make(map[string]bool)
m[pin.String()] = false
m[Label(pin)] = false
for j := 1; j <= max; j++ {
v := randomTestAddr(keylen, j)
n.Add(v)
m[v.String()] = false
v := randomTestAddr(alen, j)
n, _, _ = Add(n, v, pof)
m[Label(v)] = false
}
size := n.Size()
@ -474,16 +452,16 @@ func TestPotEachNeighbourSync(t *testing.T) {
continue
}
count := rand.Intn(size/2) + size/2
val := randomTestAddr(keylen, max+1)
val := randomTestAddr(alen, max+1)
log.Trace(fmt.Sprintf("%v: pin: %v, size: %v, val: %v, count: %v", i, n.Pin(), size, val, count))
err := testPotEachNeighbour(n, val, count, checkPo(val), checkOrder(val), checkValues(m, val))
err := testPotEachNeighbour(n, pof, val, count, checkPo(val, pof), checkOrder(val), checkValues(m, val))
if err != nil {
t.Fatal(err)
}
minPoFound := keylen
minPoFound := alen
maxPoNotFound := 0
for k, found := range m {
po, _ := val.PO(NewTestAddr(k, 0), 0)
po, _ := pof(val, newTestAddr(k, 0), 0)
if found {
if po < minPoFound {
minPoFound = po
@ -503,11 +481,14 @@ func TestPotEachNeighbourSync(t *testing.T) {
func TestPotEachNeighbourAsync(t *testing.T) {
for i := 0; i < maxEachNeighbourTests; i++ {
max := rand.Intn(maxEachNeighbour/2) + maxEachNeighbour/2
n := NewPot(randomTestAddr(keylen, 0), 0)
var size int = 1
alen := maxkeylen
pof := DefaultPof(alen)
n := NewPot(randomTestAddr(alen, 0), 0)
size := 1
var found bool
for j := 1; j <= max; j++ {
v := randomTestAddr(keylen, j)
_, found := n.Add(v)
v := randomTestAddr(alen, j)
n, _, found = Add(n, v, pof)
if !found {
size++
}
@ -519,42 +500,34 @@ func TestPotEachNeighbourAsync(t *testing.T) {
continue
}
count := rand.Intn(size/2) + size/2
val := randomTestAddr(keylen, max+1)
val := randomTestAddr(alen, max+1)
mu := sync.Mutex{}
m := make(map[string]bool)
maxPos := rand.Intn(keylen)
maxPos := rand.Intn(alen)
log.Trace(fmt.Sprintf("%v: pin: %v, size: %v, val: %v, count: %v, maxPos: %v", i, n.Pin(), size, val, count, maxPos))
msize := 0
remember := func(v PotVal, po int) error {
// mu.Lock()
// defer mu.Unlock()
remember := func(v Val, po int) error {
if po > maxPos {
// log.Trace(fmt.Sprintf("NOT ADD %v", v))
return errNoCount
}
// log.Trace(fmt.Sprintf("ADD %v, %v", v, msize))
m[v.String()] = true
m[Label(v)] = true
msize++
return nil
}
if i == 0 {
continue
}
err := testPotEachNeighbour(n, val, count, remember)
if err != nil {
log.Error(err.Error())
}
testPotEachNeighbour(n, pof, val, count, remember)
d := 0
forget := func(v PotVal, po int) {
forget := func(v Val, po int) {
mu.Lock()
defer mu.Unlock()
d++
// log.Trace(fmt.Sprintf("DEL %v", v))
delete(m, v.String())
delete(m, Label(v))
}
n.EachNeighbourAsync(val, count, maxPos, forget, true)
n.EachNeighbourAsync(val, pof, count, maxPos, forget, true)
if d != msize {
t.Fatalf("incorrect number of neighbour calls in async iterator. expected %v, got %v", msize, d)
}
@ -566,20 +539,23 @@ func TestPotEachNeighbourAsync(t *testing.T) {
func benchmarkEachNeighbourSync(t *testing.B, max, count int, d time.Duration) {
t.ReportAllocs()
pin := randomTestAddr(keylen, 0)
alen := maxkeylen
pof := DefaultPof(alen)
pin := randomTestAddr(alen, 0)
n := NewPot(pin, 0)
var found bool
for j := 1; j <= max; {
v := randomTestAddr(keylen, j)
_, found := n.Add(v)
v := randomTestAddr(alen, j)
n, _, found = Add(n, v, pof)
if !found {
j++
}
}
t.ResetTimer()
for i := 0; i < t.N; i++ {
val := randomTestAddr(keylen, max+1)
val := randomTestAddr(alen, max+1)
m := 0
n.EachNeighbour(val, func(v PotVal, po int) bool {
n.EachNeighbour(val, pof, func(v Val, po int) bool {
time.Sleep(d)
m++
if m == count {
@ -591,31 +567,32 @@ func benchmarkEachNeighbourSync(t *testing.B, max, count int, d time.Duration) {
t.StopTimer()
stats := new(runtime.MemStats)
runtime.ReadMemStats(stats)
// fmt.Println(stats.Sys)
}
func benchmarkEachNeighbourAsync(t *testing.B, max, count int, d time.Duration) {
t.ReportAllocs()
pin := randomTestAddr(keylen, 0)
alen := maxkeylen
pof := DefaultPof(alen)
pin := randomTestAddr(alen, 0)
n := NewPot(pin, 0)
var found bool
for j := 1; j <= max; {
v := randomTestAddr(keylen, j)
_, found := n.Add(v)
v := randomTestAddr(alen, j)
n, _, found = Add(n, v, pof)
if !found {
j++
}
}
t.ResetTimer()
for i := 0; i < t.N; i++ {
val := randomTestAddr(keylen, max+1)
n.EachNeighbourAsync(val, count, keylen, func(v PotVal, po int) {
val := randomTestAddr(alen, max+1)
n.EachNeighbourAsync(val, pof, count, alen, func(v Val, po int) {
time.Sleep(d)
}, true)
}
t.StopTimer()
stats := new(runtime.MemStats)
runtime.ReadMemStats(stats)
// fmt.Println(stats.Sys)
}
func BenchmarkEachNeighbourSync_3_1_0(t *testing.B) {

2
swarm/api/config.go
View file

@ -88,7 +88,7 @@ func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, n
// if not set in function param, then set default for swarm network, will be overwritten by config file if present
if networkId == 0 {
self.NetworkId = network.NetworkId
self.NetworkId = network.NetworkID
}
if err != nil {

104
swarm/network/discovery.go
View file

@ -1,3 +1,19 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package network
import (
@ -19,27 +35,27 @@ type discPeer struct {
}
// NewDiscovery discovery peer contructor
func NewDiscovery(p *bzzPeer, o Overlay) *discPeer {
self := &discPeer{
func newDiscovery(p *bzzPeer, o Overlay) *discPeer {
d := &discPeer{
overlay: o,
bzzPeer: p,
peers: make(map[string]bool),
}
self.seen(self)
return self
d.seen(d)
return d
}
func (self *discPeer) HandleMsg(msg interface{}) error {
func (d *discPeer) HandleMsg(msg interface{}) error {
switch msg := msg.(type) {
case *peersMsg:
return self.handlePeersMsg(msg)
return d.handlePeersMsg(msg)
case *getPeersMsg:
return self.handleGetPeersMsg(msg)
return d.handleGetPeersMsg(msg)
case *subPeersMsg:
return self.handleSubPeersMsg(msg)
return d.handleSubPeersMsg(msg)
default:
return fmt.Errorf("unknown message type: %T", msg)
@ -48,8 +64,8 @@ func (self *discPeer) HandleMsg(msg interface{}) error {
// NotifyPeer notifies the receiver remote end of a peer p or PO po.
// callback for overlay driver
func (self *discPeer) NotifyPeer(a OverlayAddr, po uint8) error {
if po < self.depth || self.seen(a) {
func (d *discPeer) NotifyPeer(a OverlayAddr, po uint8) error {
if po < d.depth || d.seen(a) {
return nil
}
log.Warn(fmt.Sprintf("notification about %x", a.Address()))
@ -57,15 +73,15 @@ func (self *discPeer) NotifyPeer(a OverlayAddr, po uint8) error {
resp := &peersMsg{
Peers: []*bzzAddr{ToAddr(a)}, // perhaps the PeerAddr interface is unnecessary generalization
}
return self.Send(resp)
return d.Send(resp)
}
// NotifyDepth sends a subPeers Msg to the receiver notifying them about
// a change in the prox limit (radius of the set including the nearest X peers
// or first empty row)
// callback for overlay driver
func (self *discPeer) NotifyDepth(po uint8) error {
return self.Send(&subPeersMsg{Depth: po})
func (d *discPeer) NotifyDepth(po uint8) error {
return d.Send(&subPeersMsg{Depth: po})
}
/*
@ -91,8 +107,8 @@ type peersMsg struct {
Peers []*bzzAddr
}
func (self peersMsg) String() string {
return fmt.Sprintf("%T: %v", self, self.Peers)
func (msg peersMsg) String() string {
return fmt.Sprintf("%T: %v", msg, msg.Peers)
}
// getPeersMsg is sent to (random) peers to request (Max) peers of a specific order
@ -101,8 +117,8 @@ type getPeersMsg struct {
Max uint8
}
func (self getPeersMsg) String() string {
return fmt.Sprintf("%T: accept max %v peers of PO%03d", self, self.Max, self.Order)
func (msg getPeersMsg) String() string {
return fmt.Sprintf("%T: accept max %v peers of PO%03d", msg, msg.Max, msg.Order)
}
// subPeers msg is communicating the depth/sharpness/focus of the overlay table of a peer
@ -110,41 +126,41 @@ type subPeersMsg struct {
Depth uint8
}
func (self subPeersMsg) String() string {
return fmt.Sprintf("%T: request peers > PO%02d. ", self, self.Depth)
func (msg subPeersMsg) String() string {
return fmt.Sprintf("%T: request peers > PO%02d. ", msg, msg.Depth)
}
func (self *discPeer) handleSubPeersMsg(msg *subPeersMsg) error {
self.depth = msg.Depth
if !self.sentPeers {
func (d *discPeer) handleSubPeersMsg(msg *subPeersMsg) error {
d.depth = msg.Depth
if !d.sentPeers {
var peers []*bzzAddr
self.overlay.EachConn(self.Over(), 255, func(p OverlayConn, po int, isproxbin bool) bool {
if uint8(po) < self.depth {
d.overlay.EachConn(d.Over(), 255, func(p OverlayConn, po int, isproxbin bool) bool {
if uint8(po) < d.depth {
return false
}
if !self.seen(p) {
if !d.seen(p) {
peers = append(peers, ToAddr(p.Off()))
}
return true
})
log.Warn(fmt.Sprintf("found initial %v peers not farther than %v", len(peers), self.depth))
log.Warn(fmt.Sprintf("found initial %v peers not farther than %v", len(peers), d.depth))
if len(peers) > 0 {
if err := self.Send(&peersMsg{Peers: peers}); err != nil {
if err := d.Send(&peersMsg{Peers: peers}); err != nil {
return err
}
}
}
self.sentPeers = true
d.sentPeers = true
return nil
}
// handlePeersMsg called by the protocol when receiving peerset (for target address)
// list of nodes ([]PeerAddr in peersMsg) is added to the overlay db using the
// Register interface method
func (self *discPeer) handlePeersMsg(msg *peersMsg) error {
func (d *discPeer) handlePeersMsg(msg *peersMsg) error {
// register all addresses
if len(msg.Peers) == 0 {
log.Debug(fmt.Sprintf("whoops, no peers in incoming peersMsg from %v", self))
log.Debug(fmt.Sprintf("whoops, no peers in incoming peersMsg from %v", d))
return nil
}
@ -152,12 +168,12 @@ func (self *discPeer) handlePeersMsg(msg *peersMsg) error {
go func() {
defer close(c)
for _, a := range msg.Peers {
self.seen(a)
d.seen(a)
c <- a
}
}()
log.Info("discovery overlay register")
return self.overlay.Register(c)
return d.overlay.Register(c)
}
// handleGetPeersMsg is called by the protocol when receiving a
@ -165,29 +181,31 @@ func (self *discPeer) handlePeersMsg(msg *peersMsg) error {
// peers suggestions are retrieved from the overlay topology driver
// using the EachConn interface iterator method
// peers sent are remembered throughout a session and not sent twice
func (self *discPeer) handleGetPeersMsg(msg *getPeersMsg) error {
func (d *discPeer) handleGetPeersMsg(msg *getPeersMsg) error {
var peers []*bzzAddr
i := 0
self.overlay.EachConn(self.Over(), int(msg.Order), func(p OverlayConn, po int, isproxbin bool) bool {
d.overlay.EachConn(d.Over(), int(msg.Order), func(p OverlayConn, po int, isproxbin bool) bool {
i++
// only send peers we have not sent before in this session
a := ToAddr(p.Off())
if self.seen(a) {
if d.seen(a) {
peers = append(peers, a)
}
return len(peers) < int(msg.Max)
})
if len(peers) == 0 {
log.Debug(fmt.Sprintf("no peers found for %v", self))
log.Debug(fmt.Sprintf("no peers found for %v", d))
return nil
}
log.Debug(fmt.Sprintf("%v peers sent to %v", len(peers), self))
log.Debug(fmt.Sprintf("%v peers sent to %v", len(peers), d))
resp := &peersMsg{
Peers: peers,
}
return self.Send(resp)
return d.Send(resp)
}
// RequestOrder broadcasts to trageted peers a request for peers of a particular
// proximity order
func RequestOrder(k Overlay, order, broadcastSize, maxPeers uint8) {
req := &getPeersMsg{
Order: uint8(order),
@ -207,13 +225,13 @@ func RequestOrder(k Overlay, order, broadcastSize, maxPeers uint8) {
log.Info(fmt.Sprintf("requesting bees of PO%03d from %v/%v (each max %v)", order, i, broadcastSize, maxPeers))
}
func (self *discPeer) seen(p OverlayPeer) bool {
self.mtx.Lock()
defer self.mtx.Unlock()
func (d *discPeer) seen(p OverlayPeer) bool {
d.mtx.Lock()
defer d.mtx.Unlock()
k := string(p.Address())
if self.peers[k] {
if d.peers[k] {
return true
}
self.peers[k] = true
d.peers[k] = true
return false
}

18
swarm/network/discovery_test.go
View file

@ -1,3 +1,19 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package network
import (
@ -18,7 +34,7 @@ func TestDiscovery(t *testing.T) {
to := NewKademlia(addr.OAddr, NewKadParams())
run := func(p *bzzPeer) error {
dp := NewDiscovery(p, to)
dp := newDiscovery(p, to)
to.On(p)
defer to.Off(p)
log.Trace(fmt.Sprintf("kademlia on %v", p))

118
swarm/network/hive.go
View file

@ -89,9 +89,10 @@ type Hive struct {
tick <-chan time.Time
}
// Hive constructor embeds both arguments
// NewHive constructs a new hive
// HiveParams: config parameters
// Overlay: Topology Driver Interface
// StateStore: to save peers across sessions
func NewHive(params *HiveParams, overlay Overlay, store StateStore) *Hive {
return &Hive{
HiveParams: params,
@ -105,105 +106,98 @@ func NewHive(params *HiveParams, overlay Overlay, store StateStore) *Hive {
// these are called on the p2p.Server which runs on the node
// af() returns an arbitrary ticker channel
// rw is a read writer for json configs
func (self *Hive) Start(server *p2p.Server) error {
if self.store != nil {
if err := self.loadPeers(); err != nil {
func (h *Hive) Start(server *p2p.Server) error {
if h.store != nil {
if err := h.loadPeers(); err != nil {
return err
}
}
self.more = make(chan bool, 1)
self.quit = make(chan bool)
log.Debug("hive started")
h.more = make(chan bool, 1)
h.quit = make(chan bool)
// this loop is doing bootstrapping and maintains a healthy table
go self.keepAlive()
go h.keepAlive()
go func() {
// each iteration, ask kademlia about most preferred peer
for more := range self.more {
for more := range h.more {
if !more {
// receiving false closes the loop while allowing parallel routines
// to attempt to write to more (remove Peer when shutting down)
return
}
log.Debug("hive delegate to overlay driver: suggest addr to connect to")
log.Trace(fmt.Sprintf("%x: hive delegate to overlay driver: suggest addr to connect to", h.BaseAddr()[:4]))
// log.Trace("hive delegate to overlay driver: suggest addr to connect to")
addr, order, want := self.SuggestPeer()
if self.Discovery {
addr, order, want := h.SuggestPeer()
if h.Discovery {
if addr != nil {
log.Info(fmt.Sprintf("========> connect to bee %v", addr))
log.Trace(fmt.Sprintf("%x ========> connect to bee %x", h.BaseAddr()[:4], addr.Address()[:4]))
under, err := discover.ParseNode(string(addr.(Addr).Under()))
if err == nil {
server.AddPeer(under)
} else {
log.Error(fmt.Sprintf("===X====> connect to bee %v failed: invalid node URL: %v", addr, err))
log.Error(fmt.Sprintf("%x ===X====> connect to bee %x failed: invalid node URL: %v", h.BaseAddr()[:4], addr.Address()[:4], err))
}
} else {
log.Trace("cannot suggest peers")
log.Trace(fmt.Sprintf("%x cannot suggest peers", h.BaseAddr()[:4]))
}
if want {
log.Debug(fmt.Sprintf("========> request peers nearest %v", addr))
RequestOrder(self.Overlay, uint8(order), self.PeersBroadcastSetSize, self.MaxPeersPerRequest)
log.Trace(fmt.Sprintf("%x ========> request peers for PO%0d", h.BaseAddr()[:4], order))
RequestOrder(h.Overlay, uint8(order), h.PeersBroadcastSetSize, h.MaxPeersPerRequest)
}
}
log.Info(fmt.Sprintf("%v", self))
select {
case <-self.quit:
return
default:
}
log.Trace(fmt.Sprintf("%v", h))
}
}()
return nil
}
// Stop terminates the updateloop and saves the peers
func (self *Hive) Stop() {
if self.store != nil {
self.savePeers()
func (h *Hive) Stop() {
if h.store != nil {
h.savePeers()
}
// closing toggle channel quits the updateloop
close(self.quit)
close(h.quit)
}
func (self *Hive) Run(p *bzzPeer) error {
dp := NewDiscovery(p, self)
// Run protocol run function
func (h *Hive) Run(p *bzzPeer) error {
dp := newDiscovery(p, h)
log.Debug(fmt.Sprintf("to add new bee %v", p))
self.On(dp)
self.wake()
defer self.wake()
defer self.Off(dp)
h.On(dp)
h.wake()
defer h.wake()
defer h.Off(dp)
return p.Run(dp.HandleMsg)
}
// NodeInfo function is used by the p2p.server RPC interface to display
// protocol specific node information
func (self *Hive) NodeInfo() interface{} {
return self.String()
func (h *Hive) NodeInfo() interface{} {
return h.String()
}
// PeerInfo function is used by the p2p.server RPC interface to display
// protocol specific information any connected peer referred to by their NodeID
func (self *Hive) PeerInfo(id discover.NodeID) interface{} {
self.lock.Lock()
defer self.lock.Unlock()
func (h *Hive) PeerInfo(id discover.NodeID) interface{} {
h.lock.Lock()
defer h.lock.Unlock()
addr := NewAddrFromNodeID(id)
return interface{}(addr)
}
func (self *Hive) Register(peers chan OverlayAddr) error {
defer self.wake()
return self.Overlay.Register(peers)
func (h *Hive) Register(peers chan OverlayAddr) error {
defer h.wake()
return h.Overlay.Register(peers)
}
// wake triggers
func (self *Hive) wake() {
func (h *Hive) wake() {
select {
case self.more <- true:
log.Trace("hive woken up")
case <-self.quit:
case h.more <- true:
case <-h.quit:
default:
log.Trace("hive already awake")
}
}
@ -233,30 +227,30 @@ func (t *timeTicker) Ch() <-chan time.Time {
// keepAlive is a forever loop
// in its awake state it periodically triggers connection attempts
// by writing to self.more until Kademlia Table is saturated
// wake state is toggled by writing to self.toggle
// by writing to h.more until Kademlia Table is saturated
// wake state is toggled by writing to h.toggle
// it goes to sleep mode if table is saturated
// it restarts if the table becomes non-full again due to disconnections
func (self *Hive) keepAlive() {
if self.tick == nil {
ticker := time.NewTicker(self.KeepAliveInterval)
func (h *Hive) keepAlive() {
if h.tick == nil {
ticker := time.NewTicker(h.KeepAliveInterval)
defer ticker.Stop()
self.tick = ticker.C
h.tick = ticker.C
}
for {
select {
case <-self.tick:
log.Debug("wake up: make hive alive")
self.wake()
case <-self.quit:
case <-h.tick:
h.wake()
case <-h.quit:
h.more <- false
return
}
}
}
// loadPeers, savePeer implement persistence callback/
func (self *Hive) loadPeers() error {
data, err := self.store.Load("peers")
func (h *Hive) loadPeers() error {
data, err := h.store.Load("peers")
if err != nil {
return err
}
@ -275,13 +269,13 @@ func (self *Hive) loadPeers() error {
c <- a
}
}()
return self.Overlay.Register(c)
return h.Overlay.Register(c)
}
// savePeers, savePeer implement persistence callback/
func (self *Hive) savePeers() error {
func (h *Hive) savePeers() error {
var peers []*bzzAddr
self.Overlay.EachAddr(nil, 256, func(pa OverlayAddr, i int) bool {
h.Overlay.EachAddr(nil, 256, func(pa OverlayAddr, i int) bool {
if pa == nil {
log.Warn(fmt.Sprintf("empty addr: %v", i))
return true
@ -293,7 +287,7 @@ func (self *Hive) savePeers() error {
if err != nil {
return fmt.Errorf("could not encode peers: %v", err)
}
if err := self.store.Save("peers", data); err != nil {
if err := h.store.Save("peers", data); err != nil {
return fmt.Errorf("could not save peers: %v", err)
}
return nil

22
swarm/network/hive_test.go
View file

@ -1,3 +1,19 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package network
import (
@ -17,6 +33,7 @@ func newHiveTester(t *testing.T, params *HiveParams) (*bzzTester, *Hive) {
}
func TestRegisterAndConnect(t *testing.T) {
//t.Skip("deadlocked")
params := NewHiveParams()
s, pp := newHiveTester(t, params)
defer s.Stop()
@ -46,6 +63,11 @@ func TestRegisterAndConnect(t *testing.T) {
s.TestExchanges(p2ptest.Exchange{
Label: "getPeersMsg message",
Expects: []p2ptest.Expect{
p2ptest.Expect{
Code: 2,
Msg: &subPeersMsg{0},
Peer: id,
},
p2ptest.Expect{
Code: 1,
Msg: &getPeersMsg{uint8(o), 5},

389
swarm/network/kademlia.go
View file

@ -20,6 +20,7 @@ import (
"bytes"
"fmt"
"strings"
"sync"
"time"
"github.com/ethereum/go-ethereum/log"
@ -47,6 +48,8 @@ a guaranteed constant maximum limit on the number of hops needed to reach one
node from the other.
*/
var pof = pot.DefaultPof(256)
// KadParams holds the config params for Kademlia
type KadParams struct {
// adjustable parameters
@ -76,11 +79,12 @@ func NewKadParams() *KadParams {
// Kademlia is a table of live peers and a db of known peers
type Kademlia struct {
*KadParams // Kademlia configuration parameters
base []byte // immutable baseaddress of the table
addrs *pot.Pot // pots container for known peer addresses
conns *pot.Pot // pots container for live peer connections
depth uint8 // stores the last calculated depth
lock sync.RWMutex
*KadParams // Kademlia configuration parameters
base []byte // immutable baseaddress of the table
addrs *pot.Pot // pots container for known peer addresses
conns *pot.Pot // pots container for live peer connections
currentDepth uint8 // stores the last calculated depth
}
// NewKademlia creates a Kademlia table for base address addr
@ -98,6 +102,7 @@ func NewKademlia(addr []byte, params *KadParams) *Kademlia {
}
}
// Notifier interface type for peer allowing / requesting peer and depth notifications
type Notifier interface {
NotifyPeer(OverlayAddr, uint8) error
NotifyDepth(uint8) error
@ -116,16 +121,14 @@ type OverlayConn interface {
Off() OverlayAddr // call to return a persitent OverlayAddr
}
// OverlayAddr represents a kademlia peer record
type OverlayAddr interface {
OverlayPeer
Update(OverlayAddr) OverlayAddr // returns the updated version of the original
}
// entry represents a Kademlia table entry (an extension of OverlayPeer)
// implements the pot.PotVal interface via BytesAddress, so entry can be
// used directly as a pot element
type entry struct {
pot.PotVal
OverlayPeer
seenAt time.Time
retries int
@ -134,51 +137,60 @@ type entry struct {
// newEntry creates a kademlia peer from an OverlayPeer interface
func newEntry(p OverlayPeer) *entry {
return &entry{
PotVal: pot.NewBytesVal(p, nil),
OverlayPeer: p,
seenAt: time.Now(),
}
}
func (self *entry) addr() OverlayAddr {
a, _ := self.OverlayPeer.(OverlayAddr)
// Bin is the binary (bitvector) serialisation of the entry address
func (e *entry) Bin() string {
return pot.ToBin(e.addr().Address())
}
// Label is a short tag for the entry for debug
func Label(e *entry) string {
return fmt.Sprintf("%s (%d)", e.Bin()[:8], e.retries)
}
// Hex is the hexadecimal serialisation of the entry address
func (e *entry) Hex() string {
return fmt.Sprintf("%x", e.addr().Address())
}
// String is the short tag for the entry
func (e *entry) String() string {
return fmt.Sprintf("%s (%d)", e.Hex()[:4], e.retries)
}
// addr returns the kad peer record (OverlayAddr) corresponding to the entry
func (e *entry) addr() OverlayAddr {
a, _ := e.OverlayPeer.(OverlayAddr)
return a
}
func (self *entry) conn() OverlayConn {
c, _ := self.OverlayPeer.(OverlayConn)
// conn returns the connected peer (OverlayPeer) corresponding to the entry
func (e *entry) conn() OverlayConn {
c, _ := e.OverlayPeer.(OverlayConn)
return c
}
func (self *entry) String() string {
return fmt.Sprintf("%x", self.OverlayPeer.Address())
}
// Register enters each OverlayAddr as kademlia peer record into the
// database of known peer addresses
func (self *Kademlia) Register(peers chan OverlayAddr) error {
func (k *Kademlia) Register(peers chan OverlayAddr) error {
np := pot.NewPot(nil, 0)
for p := range peers {
// error if self received, peer should know better
if bytes.Equal(p.Address(), self.base) {
return fmt.Errorf("add peers: %x is self", self.base)
// error if k received, peer should know better
if bytes.Equal(p.Address(), k.base) {
return fmt.Errorf("add peers: %x is k", k.base)
}
np, _, _ = pot.Add(np, newEntry(p))
np, _, _ = pot.Add(np, newEntry(p), pof)
}
com := self.addrs.Merge(np)
log.Debug(fmt.Sprintf("merged %v peers, %v known, total: %v", np.Size(), com, self.addrs.Size()))
// log.Trace(fmt.Sprintf("merged %v peers, %v known", np.Size(), com))
// TODO: remove this check
m := make(map[string]bool)
self.addrs.Each(func(val pot.PotVal, i int) bool {
_, found := m[val.String()]
if found {
panic("duplicate found")
}
m[val.String()] = true
return true
})
var com int
k.lock.Lock()
defer k.lock.Unlock()
k.addrs, com = pot.Union(k.addrs, np, pof)
log.Trace(fmt.Sprintf("%x merged %v peers, %v known, total: %v", k.BaseAddr()[:4], np.Size(), com, k.addrs.Size()))
return nil
}
@ -186,164 +198,179 @@ func (self *Kademlia) Register(peers chan OverlayAddr) error {
// lowest bincount below depth
// naturally if there is an empty row it returns a peer for that
//
func (self *Kademlia) SuggestPeer() (a OverlayAddr, o int, want bool) {
minsize := self.MinBinSize
depth := self.Depth()
empty := self.FirstEmptyBin()
func (k *Kademlia) SuggestPeer() (a OverlayAddr, o int, want bool) {
k.lock.RLock()
defer k.lock.RUnlock()
minsize := k.MinBinSize
depth := k.depth()
// if there is a callable neighbour within the current proxBin, connect
// this makes sure nearest neighbour set is fully connected
log.Debug(fmt.Sprintf("candidate prox peer checking above PO %v", depth))
// log.Trace(fmt.Sprintf("candidate prox peer checking above PO %v", depth))
var ppo int
ba := pot.NewBytesVal(self.base, nil)
self.addrs.EachNeighbour(ba, func(val pot.PotVal, po int) bool {
a = self.callable(val)
log.Trace(fmt.Sprintf("candidate prox peer at %x: %v (%v). a == nil is %v", val.(*entry).Address(), a, po, a == nil))
k.addrs.EachNeighbour(k.base, pof, func(val pot.Val, po int) bool {
if po < depth {
return false
}
a = k.callable(val)
log.Trace(fmt.Sprintf("%x candidate nearest neighbour at %v: %v (%v)", k.BaseAddr()[:4], val.(*entry), a, po))
ppo = po
return a == nil && po >= depth
return a == nil
})
if a != nil {
log.Debug(fmt.Sprintf("candidate prox peer found: %v (%v)", a, ppo))
log.Trace(fmt.Sprintf("%x candidate nearest neighbour found: %v (%v)", k.BaseAddr()[:4], a, ppo))
return a, 0, false
}
log.Debug(fmt.Sprintf("no candidate prox peers to connect to (Depth: %v, minProxSize: %v) %#v", depth, self.MinProxBinSize, a))
log.Trace(fmt.Sprintf("%x no candidate nearest neighbours to connect to (Depth: %v, minProxSize: %v) %#v", k.BaseAddr()[:4], depth, k.MinProxBinSize, a))
var bpo []int
prev := -1
self.conns.EachBin(pot.NewBytesVal(self.base, nil), 0, func(po, size int, f func(func(val pot.PotVal, i int) bool) bool) bool {
log.Trace(fmt.Sprintf("check PO%02d: ", po))
k.conns.EachBin(k.base, pof, 0, func(po, size int, f func(func(val pot.Val, i int) bool) bool) bool {
prev++
if po > prev {
size = 0
po = prev
for ; prev < po; prev++ {
bpo = append(bpo, prev)
minsize = 0
}
if size < minsize {
minsize = size
bpo = append(bpo, po)
minsize = size
}
return size > 0 && po < depth
})
// all buckets are full
// minsize == self.MinBinSize
// minsize == k.MinBinSize
if len(bpo) == 0 {
log.Debug(fmt.Sprintf("%x: all bins saturated", k.BaseAddr()[:4]))
return nil, 0, false
}
// as long as we got candidate peers to connect to
// dont ask for new peers (want = false)
// try to select a candidate peer
for i := len(bpo) - 1; i >= 0; i-- {
// find the first callable peer
self.addrs.EachBin(ba, bpo[i], func(po, size int, f func(func(pot.PotVal, int) bool) bool) bool {
// for each bin we find callable candidate peers
log.Trace(fmt.Sprintf("check PO%02d: ", po))
f(func(val pot.PotVal, j int) bool {
a = self.callable(val)
if po == empty {
log.Debug(fmt.Sprintf("candidate prox peer found: %v (%v)", a, ppo))
}
return a == nil && po <= depth
})
// find the first callable peer
i := 0
nxt := bpo[0]
k.addrs.EachBin(k.base, pof, nxt, func(po, size int, f func(func(pot.Val, int) bool) bool) bool {
// for each bin we find callable candidate peers
if i >= depth {
return false
})
// found a candidate
if a != nil {
break
}
// cannot find a candidate, ask for more for this proximity bin specifically
o = bpo[i]
want = true
if po == nxt {
i++
if i < len(bpo) {
nxt = bpo[i]
}
}
f(func(val pot.Val, j int) bool {
a = k.callable(val)
return a == nil && i < len(bpo) && po <= depth
})
return false
})
// found a candidate
if a != nil {
return a, 0, false
}
return a, o, want
return a, nxt, true
}
// On inserts the peer as a kademlia peer into the live peers
func (self *Kademlia) On(p OverlayConn) {
func (k *Kademlia) On(p OverlayConn) {
k.lock.Lock()
defer k.lock.Unlock()
e := newEntry(p)
self.conns.Swap(p, func(v pot.PotVal) pot.PotVal {
var ins bool
k.conns, _, _, _ = pot.Swap(k.conns, p, pof, func(v pot.Val) pot.Val {
// if not found live
if v == nil {
// insert new online peer into addrs
self.addrs.Swap(p, func(v pot.PotVal) pot.PotVal {
return e
})
ins = true
// insert new online peer into conns
return e
}
// found among live peers, do nothing
return v
})
if ins {
// insert new online peer into addrs
k.addrs, _, _, _ = pot.Swap(k.addrs, p, pof, func(v pot.Val) pot.Val {
return e
})
}
np, ok := p.(Notifier)
if !ok {
return
}
depth := uint8(self.Depth())
if depth != self.depth {
self.depth = depth
} else {
depth = 0
depth := uint8(k.depth())
var depthChanged bool
if depth != k.currentDepth {
depthChanged = true
k.currentDepth = depth
}
go np.NotifyDepth(depth)
f := func(val pot.PotVal, po int) {
f := func(val pot.Val, po int) {
dp := val.(*entry).OverlayPeer.(Notifier)
dp.NotifyPeer(p.Off(), uint8(po))
// log.Trace(fmt.Sprintf("peer %v notified of %v (%v)", dp, p, po))
log.Debug(fmt.Sprintf("peer %v notified of %v (%v)", dp, p, po))
if depth > 0 {
log.Trace(fmt.Sprintf("peer %v notified of %v (%v)", dp, p, po))
if depthChanged {
dp.NotifyDepth(depth)
log.Debug(fmt.Sprintf("peer %v notified of new depth %v", dp, depth))
// log.Trace(fmt.Sprintf("peer %v notified of new depth %v", dp, depth))
log.Trace(fmt.Sprintf("peer %v notified of new depth %v", dp, depth))
}
}
self.conns.EachNeighbourAsync(e, 1024, 255, f, false)
k.conns.EachNeighbourAsync(e, pof, 1024, 255, f, false)
}
// Off removes a peer from among live peers
func (self *Kademlia) Off(p OverlayConn) {
self.addrs.Swap(p, func(v pot.PotVal) pot.PotVal {
func (k *Kademlia) Off(p OverlayConn) {
k.lock.Lock()
defer k.lock.Unlock()
var del bool
k.addrs, _, _, _ = pot.Swap(k.addrs, p, pof, func(v pot.Val) pot.Val {
// v cannot be nil, must check otherwise we overwrite entry
if v == nil {
panic(fmt.Sprintf("connected peer not found %v", p))
}
self.conns.Swap(p, func(v pot.PotVal) pot.PotVal {
del = true
return newEntry(p.Off())
})
if del {
k.conns, _, _, _ = pot.Swap(k.conns, p, pof, func(_ pot.Val) pot.Val {
// v cannot be nil, but no need to check
return nil
})
return newEntry(p.Off())
})
}
}
// EachConn is an iterator with args (base, po, f) applies f to each live peer
// that has proximity order po or less as measured from the base
// if base is nil, kademlia base address is used
func (self *Kademlia) EachConn(base []byte, o int, f func(OverlayConn, int, bool) bool) {
func (k *Kademlia) EachConn(base []byte, o int, f func(OverlayConn, int, bool) bool) {
k.lock.RLock()
defer k.lock.RUnlock()
k.eachConn(base, o, f)
}
func (k *Kademlia) eachConn(base []byte, o int, f func(OverlayConn, int, bool) bool) {
if len(base) == 0 {
base = self.base
base = k.base
}
p := pot.NewBytesVal(base, nil)
self.conns.EachNeighbour(p, func(val pot.PotVal, po int) bool {
depth := k.depth()
k.conns.EachNeighbour(base, pof, func(val pot.Val, po int) bool {
if po > o {
return true
}
isproxbin := false
if l, _ := p.PO(val, 0); l >= self.Depth() {
isproxbin = true
}
return f(val.(*entry).conn(), po, isproxbin)
return f(val.(*entry).conn(), po, po >= depth)
})
}
// EachAddr(base, po, f) is an iterator applying f to each known peer
// EachAddr called with (base, po, f) is an iterator applying f to each known peer
// that has proximity order po or less as measured from the base
// if base is nil, kademlia base address is used
func (self *Kademlia) EachAddr(base []byte, o int, f func(OverlayAddr, int) bool) {
func (k *Kademlia) EachAddr(base []byte, o int, f func(OverlayAddr, int) bool) {
if len(base) == 0 {
base = self.base
base = k.base
}
p := pot.NewBytesVal(base, nil)
self.addrs.EachNeighbour(p, func(val pot.PotVal, po int) bool {
k.lock.RLock()
defer k.lock.RUnlock()
k.addrs.EachNeighbour(base, pof, func(val pot.Val, po int) bool {
if po > o {
return true
}
@ -354,39 +381,45 @@ func (self *Kademlia) EachAddr(base []byte, o int, f func(OverlayAddr, int) bool
// Depth returns the proximity order that defines the distance of
// the nearest neighbour set with cardinality >= MinProxBinSize
// if there is altogether less than MinProxBinSize peers it returns 0
func (self *Kademlia) Depth() (depth int) {
if self.conns.Size() < self.MinProxBinSize {
func (k *Kademlia) Depth() (depth int) {
k.lock.RLock()
defer k.lock.RUnlock()
return k.depth()
}
func (k *Kademlia) depth() (depth int) {
if k.conns.Size() < k.MinProxBinSize {
return 0
}
var size int
f := func(v pot.PotVal, i int) bool {
f := func(v pot.Val, i int) bool {
size++
depth = i
return size < self.MinProxBinSize
return size < k.MinProxBinSize
}
self.conns.EachNeighbour(pot.NewBytesVal(self.base, nil), f)
k.conns.EachNeighbour(k.base, pof, f)
return depth
}
func (self *Kademlia) callable(val pot.PotVal) OverlayAddr {
// calleble when called with val,
func (k *Kademlia) callable(val pot.Val) OverlayAddr {
e := val.(*entry)
// not callable if peer is live or exceeded maxRetries
if e.conn() != nil || e.retries > self.MaxRetries {
if e.conn() != nil || e.retries > k.MaxRetries {
log.Trace(fmt.Sprintf("peer %v (%T) not callable", e, e.OverlayPeer))
return nil
}
// calculate the allowed number of retries based on time lapsed since last seen
timeAgo := time.Since(e.seenAt)
var retries int
for delta := int(timeAgo) / self.RetryInterval; delta > 0; delta /= self.RetryExponent {
log.Trace(fmt.Sprintf("delta: %v", delta))
for delta := int(timeAgo) / k.RetryInterval; delta > 0; delta /= k.RetryExponent {
retries++
}
// this is never called concurrently, so safe to increment
// peer can be retried again
if retries < e.retries {
log.Trace(fmt.Sprintf("long time since last try (at %v) needed before retry %v, wait only warrants %v", timeAgo, e.retries, retries))
log.Trace(fmt.Sprintf("%v long time since last try (at %v) needed before retry %v, wait only warrants %v", e, timeAgo, e.retries, retries))
return nil
}
e.retries++
@ -396,61 +429,57 @@ func (self *Kademlia) callable(val pot.PotVal) OverlayAddr {
}
// BaseAddr return the kademlia base addres
func (self *Kademlia) BaseAddr() []byte {
return self.base
func (k *Kademlia) BaseAddr() []byte {
return k.base
}
// String returns kademlia table + kaddb table displayed with ascii
func (self *Kademlia) String() string {
func (k *Kademlia) String() string {
k.lock.RLock()
defer k.lock.RUnlock()
wsrow := " "
var rows []string
rows = append(rows, "=========================================================================")
rows = append(rows, fmt.Sprintf("%v KΛÐΞMLIΛ hive: queen's address: %x", time.Now().UTC().Format(time.UnixDate), self.BaseAddr()[:3]))
rows = append(rows, fmt.Sprintf("population: %d (%d), MinProxBinSize: %d, MinBinSize: %d, MaxBinSize: %d", self.conns.Size(), self.addrs.Size(), self.MinProxBinSize, self.MinBinSize, self.MaxBinSize))
rows = append(rows, fmt.Sprintf("%v KΛÐΞMLIΛ hive: queen's address: %x", time.Now().UTC().Format(time.UnixDate), k.BaseAddr()[:3]))
rows = append(rows, fmt.Sprintf("population: %d (%d), MinProxBinSize: %d, MinBinSize: %d, MaxBinSize: %d", k.conns.Size(), k.addrs.Size(), k.MinProxBinSize, k.MinBinSize, k.MaxBinSize))
liverows := make([]string, self.MaxProxDisplay)
peersrows := make([]string, self.MaxProxDisplay)
var depth int
prev := -1
var depthSet bool
rest := self.conns.Size()
self.conns.EachBin(pot.NewBytesVal(self.base, nil), 0, func(po, size int, f func(func(val pot.PotVal, i int) bool) bool) bool {
liverows := make([]string, k.MaxProxDisplay)
peersrows := make([]string, k.MaxProxDisplay)
depth := k.depth()
rest := k.conns.Size()
k.conns.EachBin(k.base, pof, 0, func(po, size int, f func(func(val pot.Val, i int) bool) bool) bool {
var rowlen int
if po >= self.MaxProxDisplay {
po = self.MaxProxDisplay - 1
if po >= k.MaxProxDisplay {
po = k.MaxProxDisplay - 1
}
row := []string{fmt.Sprintf("%2d", size)}
rest -= size
f(func(val pot.PotVal, vpo int) bool {
row = append(row, val.(*entry).String()[:6])
f(func(val pot.Val, vpo int) bool {
e := val.(*entry)
row = append(row, fmt.Sprintf("%x", e.Address()[:2]))
rowlen++
return rowlen < 4
})
if !depthSet && (po > prev+1 || rest < self.MinProxBinSize) {
depthSet = true
depth = prev + 1
}
for ; rowlen <= 5; rowlen++ {
row = append(row, " ")
}
liverows[po] = strings.Join(row, " ")
prev = po
r := strings.Join(row, " ")
r = r + wsrow
liverows[po] = r[:31]
return true
})
self.addrs.EachBin(pot.NewBytesVal(self.base, nil), 0, func(po, size int, f func(func(val pot.PotVal, i int) bool) bool) bool {
k.addrs.EachBin(k.base, pof, 0, func(po, size int, f func(func(val pot.Val, i int) bool) bool) bool {
var rowlen int
if po >= self.MaxProxDisplay {
po = self.MaxProxDisplay - 1
if po >= k.MaxProxDisplay {
po = k.MaxProxDisplay - 1
}
if size < 0 {
panic("wtf")
}
row := []string{fmt.Sprintf("%2d", size)}
// we are displaying live peers too
f(func(val pot.PotVal, vpo int) bool {
row = append(row, val.(*entry).String()[:6])
f(func(val pot.Val, vpo int) bool {
row = append(row, val.(*entry).String())
rowlen++
return rowlen < 4
})
@ -458,14 +487,14 @@ func (self *Kademlia) String() string {
return true
})
for i := 0; i < self.MaxProxDisplay; i++ {
for i := 0; i < k.MaxProxDisplay; i++ {
if i == depth {
rows = append(rows, fmt.Sprintf("============ PROX LIMIT: %d ==========================================", i))
rows = append(rows, fmt.Sprintf("============ DEPTH: %d ==========================================", i))
}
left := liverows[i]
right := peersrows[i]
if len(left) == 0 {
left = " 0 "
left = " 0 "
}
if len(right) == 0 {
right = " 0"
@ -483,15 +512,17 @@ func (self *Kademlia) String() string {
// the MaxBinSize parameter it drops the oldest n peers such that
// the bin is reduced to MinBinSize peers thus leaving slots to newly
// connecting peers
func (self *Kademlia) Prune(c <-chan time.Time) {
func (k *Kademlia) Prune(c <-chan time.Time) {
k.lock.RLock()
defer k.lock.RUnlock()
go func() {
for range c {
total := 0
self.conns.EachBin(pot.NewBytesVal(self.base, nil), 0, func(po, size int, f func(func(pot.PotVal, int) bool) bool) bool {
extra := size - self.MinBinSize
if size > self.MaxBinSize {
k.conns.EachBin(k.base, pof, 0, func(po, size int, f func(func(pot.Val, int) bool) bool) bool {
extra := size - k.MinBinSize
if size > k.MaxBinSize {
n := 0
f(func(v pot.PotVal, po int) bool {
f(func(v pot.Val, po int) bool {
v.(*entry).conn().Drop(fmt.Errorf("bucket full"))
n++
return n < extra
@ -500,25 +531,28 @@ func (self *Kademlia) Prune(c <-chan time.Time) {
}
return true
})
log.Debug(fmt.Sprintf("pruned %v peers", total))
log.Trace(fmt.Sprintf("pruned %v peers", total))
}
}()
}
// NewPeerPot just creates a new pot record OverlayAddr
func NewPeerPot(kadMinProxSize int, ids ...discover.NodeID) map[discover.NodeID][][]byte {
// create a table of all nodes for health check
np := pot.NewPot(nil, 0)
for _, id := range ids {
o := ToOverlayAddr(id.Bytes())
np, _, _ = pot.Add(np, pot.NewBytesVal(o, nil))
np, _, _ = pot.Add(np, o, pof)
}
nnmap := make(map[discover.NodeID][][]byte)
for _, id := range ids {
pl := 0
var nns [][]byte
np.EachNeighbour(pot.NewBytesVal(id.Bytes(), nil), func(val pot.PotVal, po int) bool {
a := val.(pot.BytesAddress).Address()
np.EachNeighbour(id.Bytes(), pof, func(val pot.Val, po int) bool {
// a := val.(pot.BytesAddress).Address()
// nns = append(nns, a)
a := val.([]byte)
nns = append(nns, a)
if len(nns) >= kadMinProxSize {
pl = po
@ -530,9 +564,10 @@ func NewPeerPot(kadMinProxSize int, ids ...discover.NodeID) map[discover.NodeID]
return nnmap
}
func (self *Kademlia) FirstEmptyBin() (i int) {
// FirstEmptyBin returns the farthest proximity order (int) that has no peer records
func (k *Kademlia) firstEmptyBin() (i int) {
i = -1
self.conns.EachBin(pot.NewBytesVal(self.base, nil), 0, func(po, size int, f func(func(val pot.PotVal, i int) bool) bool) bool {
k.conns.EachBin(k.base, pof, 0, func(po, size int, f func(func(val pot.Val, i int) bool) bool) bool {
if po > i+1 {
i = po
return false
@ -543,22 +578,24 @@ func (self *Kademlia) FirstEmptyBin() (i int) {
return i
}
func (self *Kademlia) Full() bool {
return self.FirstEmptyBin() >= self.Depth()
// Full returns a bool if the kademlia table is healthy and complete
func (k *Kademlia) full() bool {
return k.firstEmptyBin() >= k.depth()
}
// Healthy reports the health state of the kademlia connectivity
//
func (self *Kademlia) Healthy(peers [][]byte) bool {
return self.gotNearestNeighbours(peers) && self.Full()
func (k *Kademlia) Healthy(peers [][]byte) bool {
k.lock.RLock()
defer k.lock.RUnlock()
return k.gotNearestNeighbours(peers) && k.full()
}
func (self *Kademlia) gotNearestNeighbours(peers [][]byte) (got bool) {
func (k *Kademlia) gotNearestNeighbours(peers [][]byte) (got bool) {
pm := make(map[string]bool)
for _, p := range peers {
pm[string(p)] = true
}
self.EachConn(nil, 255, func(p OverlayConn, po int, nn bool) bool {
k.EachConn(nil, 255, func(p OverlayConn, po int, nn bool) bool {
if !nn {
return false
}

192
swarm/network/kademlia_test.go
View file

@ -13,6 +13,7 @@
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package network
import (
@ -27,13 +28,13 @@ import (
)
func init() {
h := log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))
// h := log.CallerFileHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(true)))
// h := log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))
h := log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))
log.Root().SetHandler(h)
}
func testKadPeerAddr(s string) *bzzAddr {
a := pot.NewHashAddress(s).Bytes()
a := pot.NewAddressFromString(s)
return &bzzAddr{OAddr: a, UAddr: a}
}
@ -54,7 +55,7 @@ type testPeerNotification struct {
po uint8
}
type testProxNotification struct {
type testDepthNotification struct {
rec string
po uint8
}
@ -64,26 +65,26 @@ type dropError struct {
addr string
}
func (self *testDropPeer) Drop(err error) {
err2 := &dropError{err, binStr(self)}
self.dropc <- err2
func (d *testDropPeer) Drop(err error) {
err2 := &dropError{err, binStr(d)}
d.dropc <- err2
}
func (self *testDiscPeer) NotifyDepth(po uint8) error {
key := binStr(self)
self.lock.Lock()
defer self.lock.Unlock()
self.notifications[key] = po
func (d *testDiscPeer) NotifyDepth(po uint8) error {
key := binStr(d)
d.lock.Lock()
defer d.lock.Unlock()
d.notifications[key] = po
return nil
}
func (self *testDiscPeer) NotifyPeer(p OverlayAddr, po uint8) error {
key := binStr(self)
func (d *testDiscPeer) NotifyPeer(p OverlayAddr, po uint8) error {
key := binStr(d)
key += binStr(p)
self.lock.Lock()
defer self.lock.Unlock()
d.lock.Lock()
defer d.lock.Unlock()
log.Trace(fmt.Sprintf("key %v=>%v", key, po))
self.notifications[key] = po
d.notifications[key] = po
return nil
}
@ -99,7 +100,7 @@ func newTestKademlia(b string) *testKademlia {
params := NewKadParams()
params.MinBinSize = 1
params.MinProxBinSize = 2
base := pot.NewHashAddress(b).Bytes()
base := pot.NewAddressFromString(b)
return &testKademlia{
NewKademlia(base, params),
false,
@ -119,8 +120,7 @@ func (k *testKademlia) newTestKadPeer(s string) Peer {
func (k *testKademlia) On(ons ...string) *testKademlia {
for _, s := range ons {
p := k.newTestKadPeer(s)
k.Kademlia.On(p)
k.Kademlia.On(k.newTestKadPeer(s).(OverlayConn))
}
return k
}
@ -158,6 +158,7 @@ func testSuggestPeer(t *testing.T, k *testKademlia, expAddr string, expPo int, e
if want != expWant {
return fmt.Errorf("expected SuggestPeer to want peers: %v", expWant)
}
t.Logf("%v", k)
return nil
}
@ -165,50 +166,49 @@ func binStr(a OverlayPeer) string {
if a == nil {
return "<nil>"
}
return pot.ToBin(a.Address())[:6]
return pot.ToBin(a.Address())[:8]
}
func TestSuggestPeerFindPeers(t *testing.T) {
// 2 row gap, unsaturated proxbin, no callables -> want PO 0
k := newTestKademlia("000000").On("001000")
// k.MinProxBinSize = 2
// k.MinBinSize = 2
k := newTestKademlia("00000000").On("00100000")
err := testSuggestPeer(t, k, "<nil>", 0, true)
if err != nil {
t.Fatal(err.Error())
}
// 2 row gap, saturated proxbin, no callables -> want PO 0
k.On("000100")
k.On("00010000")
err = testSuggestPeer(t, k, "<nil>", 0, true)
if err != nil {
t.Fatal(err.Error())
}
// 1 row gap (1 less), saturated proxbin, no callables -> want PO 1
k.On("100000")
k.On("10000000")
err = testSuggestPeer(t, k, "<nil>", 1, true)
if err != nil {
t.Fatal(err.Error())
}
// no gap (1 less), saturated proxbin, no callables -> do not want more
k.On("010000", "001001")
k.On("01000000", "00100001")
err = testSuggestPeer(t, k, "<nil>", 0, false)
if err != nil {
t.Fatal(err.Error())
}
// oversaturated proxbin, > do not want more
k.On("001001")
k.On("00100001")
err = testSuggestPeer(t, k, "<nil>", 0, false)
if err != nil {
t.Fatal(err.Error())
}
// reintroduce gap, disconnected peer callable
k.Off("010000")
err = testSuggestPeer(t, k, "010000", 0, false)
// log.Info(k.String())
k.Off("01000000")
err = testSuggestPeer(t, k, "01000000", 0, false)
if err != nil {
t.Fatal(err.Error())
}
@ -221,31 +221,33 @@ func TestSuggestPeerFindPeers(t *testing.T) {
}
// on and off again, peer callable again
k.On("010000")
k.Off("010000")
err = testSuggestPeer(t, k, "010000", 0, false)
k.On("01000000")
k.Off("01000000")
err = testSuggestPeer(t, k, "01000000", 0, false)
if err != nil {
t.Fatal(err.Error())
}
k.On("010000")
k.On("01000000")
// new closer peer appears, it is immediately wanted
k.Register("000101")
err = testSuggestPeer(t, k, "000101", 0, false)
k.Register("00010001")
err = testSuggestPeer(t, k, "00010001", 0, false)
if err != nil {
t.Fatal(err.Error())
}
// PO1 disconnects
k.On("000101")
k.Off("010000")
k.On("00010001")
log.Info(k.String())
k.Off("01000000")
log.Info(k.String())
// second time, gap filling
err = testSuggestPeer(t, k, "010000", 0, false)
err = testSuggestPeer(t, k, "01000000", 0, false)
if err != nil {
t.Fatal(err.Error())
}
k.On("010000")
k.On("01000000")
err = testSuggestPeer(t, k, "<nil>", 0, false)
if err != nil {
t.Fatal(err.Error())
@ -257,53 +259,53 @@ func TestSuggestPeerFindPeers(t *testing.T) {
t.Fatal(err.Error())
}
k.Register("010001")
k.Register("01000001")
err = testSuggestPeer(t, k, "<nil>", 0, true)
if err != nil {
t.Fatal(err.Error())
}
k.On("100001")
k.On("10000001")
log.Trace("Kad:\n%v", k.String())
err = testSuggestPeer(t, k, "010001", 0, false)
err = testSuggestPeer(t, k, "01000001", 0, false)
if err != nil {
t.Fatal(err.Error())
}
k.On("100001")
k.On("010001")
k.On("10000001")
k.On("01000001")
err = testSuggestPeer(t, k, "<nil>", 0, false)
if err != nil {
t.Fatal(err.Error())
}
k.MinBinSize = 3
k.Register("100010")
err = testSuggestPeer(t, k, "100010", 0, false)
k.Register("10000010")
err = testSuggestPeer(t, k, "10000010", 0, false)
if err != nil {
t.Fatal(err.Error())
}
k.On("100010")
k.On("10000010")
err = testSuggestPeer(t, k, "<nil>", 1, true)
if err != nil {
t.Fatal(err.Error())
}
k.On("010010")
k.On("01000010")
err = testSuggestPeer(t, k, "<nil>", 2, true)
if err != nil {
t.Fatal(err.Error())
}
k.On("001010")
k.On("00100010")
err = testSuggestPeer(t, k, "<nil>", 3, true)
if err != nil {
log.Trace("Kad:\n%v", k.String())
t.Fatal(err.Error())
}
k.On("000110")
k.On("00010010")
err = testSuggestPeer(t, k, "<nil>", 0, false)
if err != nil {
log.Trace("Kad:\n%v", k.String())
@ -314,14 +316,14 @@ func TestSuggestPeerFindPeers(t *testing.T) {
func TestSuggestPeerRetries(t *testing.T) {
// 2 row gap, unsaturated proxbin, no callables -> want PO 0
k := newTestKademlia("000000")
k := newTestKademlia("00000000")
cycle := 50 * time.Millisecond
k.RetryInterval = int(cycle)
k.MaxRetries = 3
k.RetryExponent = 3
k.Register("010000")
k.On("000001", "000010")
err := testSuggestPeer(t, k, "010000", 0, false)
k.Register("01000000")
k.On("00000001", "00000010")
err := testSuggestPeer(t, k, "01000000", 0, false)
if err != nil {
t.Fatal(err.Error())
}
@ -333,7 +335,7 @@ func TestSuggestPeerRetries(t *testing.T) {
// cycle *= time.Duration(k.RetryExponent)
time.Sleep(cycle)
err = testSuggestPeer(t, k, "010000", 0, false)
err = testSuggestPeer(t, k, "01000000", 0, false)
if err != nil {
t.Fatal(err.Error())
}
@ -345,7 +347,7 @@ func TestSuggestPeerRetries(t *testing.T) {
cycle *= time.Duration(k.RetryExponent)
time.Sleep(cycle)
err = testSuggestPeer(t, k, "010000", 0, false)
err = testSuggestPeer(t, k, "01000000", 0, false)
if err != nil {
t.Fatal(err.Error())
}
@ -357,7 +359,7 @@ func TestSuggestPeerRetries(t *testing.T) {
cycle *= time.Duration(k.RetryExponent)
time.Sleep(cycle)
err = testSuggestPeer(t, k, "010000", 0, false)
err = testSuggestPeer(t, k, "01000000", 0, false)
if err != nil {
t.Fatal(err.Error())
}
@ -378,10 +380,10 @@ func TestSuggestPeerRetries(t *testing.T) {
}
func TestPruning(t *testing.T) {
k := newTestKademlia("000000")
k.On("100000", "110000", "101000", "100100", "100010")
k.On("010000", "011000", "010100", "010010", "010001")
k.On("001000", "001100", "001010", "001001")
k := newTestKademlia("00000000")
k.On("10000000", "11000000", "10100000", "10010000", "10000010")
k.On("01000000", "01100000", "01000100", "01000010", "01000001")
k.On("00100000", "00110000", "00100010", "00100001")
k.MaxBinSize = 4
k.MinBinSize = 3
prune := make(chan time.Time)
@ -411,10 +413,10 @@ func TestPruning(t *testing.T) {
// TODO: this is now based on just taking the first 2 peers
// in order of connecting
expDropped := []string{
"101000",
"110000",
"010100",
"011000",
"10100000",
"11000000",
"01000100",
"01100000",
}
for _, addr := range expDropped {
err := dropped[addr]
@ -428,82 +430,82 @@ func TestPruning(t *testing.T) {
}
func TestKademliaHiveString(t *testing.T) {
k := newTestKademlia("000000").On("010000", "001000").Register("100000", "100001")
k := newTestKademlia("00000000").On("01000000", "00100000").Register("10000000", "10000001")
h := k.String()
expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), MinProxBinSize: 2, MinBinSize: 1, MaxBinSize: 4\n============ PROX LIMIT: 0 ==========================================\n000 0 | 2 840000 800000\n001 1 400000 | 1 400000\n002 1 200000 | 1 200000\n003 0 | 0\n004 0 | 0\n005 0 | 0\n006 0 | 0\n007 0 | 0\n========================================================================="
expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), MinProxBinSize: 2, MinBinSize: 1, MaxBinSize: 4\n000 0 | 2 8100 (0) 8000 (0)\n============ DEPTH: 1 ==========================================\n001 1 4000 | 1 4000 (0)\n002 1 2000 | 1 2000 (0)\n003 0 | 0\n004 0 | 0\n005 0 | 0\n006 0 | 0\n007 0 | 0\n========================================================================="
if expH[100:] != h[100:] {
t.Fatalf("incorrect hive output. expected %v, got %v", expH, h)
}
}
func (self *testKademlia) checkNotifications(npeers []*testPeerNotification, nprox []*testProxNotification) error {
func (k *testKademlia) checkNotifications(npeers []*testPeerNotification, nprox []*testDepthNotification) error {
for _, pn := range npeers {
key := pn.rec + pn.addr
po, found := self.notifications[key]
po, found := k.notifications[key]
if !found || pn.po != po {
return fmt.Errorf("%v, expected to have notified %v about peer %v (%v)", key, pn.rec, pn.addr, pn.po)
}
delete(self.notifications, key)
delete(k.notifications, key)
}
for _, pn := range nprox {
key := pn.rec
po, found := self.notifications[key]
po, found := k.notifications[key]
if !found || pn.po != po {
return fmt.Errorf("expected to have notified %v about new prox limit %v", pn.rec, pn.po)
}
delete(self.notifications, key)
delete(k.notifications, key)
}
if len(self.notifications) > 0 {
return fmt.Errorf("%v unexpected notifications", len(self.notifications))
if len(k.notifications) > 0 {
return fmt.Errorf("%v unexpected notifications", len(k.notifications))
}
return nil
}
func TestNotifications(t *testing.T) {
k := newTestKademlia("000000")
k := newTestKademlia("00000000")
k.Discovery = true
k.MinProxBinSize = 3
k.On("010000", "001000")
k.On("01000000", "00100000")
time.Sleep(1000 * time.Millisecond)
err := k.checkNotifications(
[]*testPeerNotification{
&testPeerNotification{"010000", "001000", 1},
&testPeerNotification{"01000000", "00100000", 1},
},
[]*testProxNotification{
&testProxNotification{"001000", 0},
&testProxNotification{"010000", 0},
[]*testDepthNotification{
&testDepthNotification{"00100000", 0},
&testDepthNotification{"01000000", 0},
},
)
if err != nil {
t.Fatal(err.Error())
}
k = k.On("100000")
k = k.On("10000000")
time.Sleep(100 * time.Millisecond)
k.checkNotifications(
[]*testPeerNotification{
&testPeerNotification{"010000", "100000", 0},
&testPeerNotification{"001000", "100000", 0},
&testPeerNotification{"01000000", "10000000", 0},
&testPeerNotification{"00100000", "10000000", 0},
},
[]*testProxNotification{
&testProxNotification{"100000", 0},
[]*testDepthNotification{
&testDepthNotification{"10000000", 0},
},
)
k = k.On("010001")
k = k.On("01000001")
time.Sleep(100 * time.Millisecond)
k.checkNotifications(
[]*testPeerNotification{
&testPeerNotification{"010000", "010001", 5},
&testPeerNotification{"001000", "010001", 1},
&testPeerNotification{"100000", "010001", 0},
&testPeerNotification{"01000000", "01000001", 5},
&testPeerNotification{"00100000", "01000001", 1},
&testPeerNotification{"10000000", "01000001", 0},
},
[]*testProxNotification{
&testProxNotification{"100000", 0},
&testProxNotification{"010000", 0},
&testProxNotification{"010001", 0},
&testProxNotification{"001000", 0},
[]*testDepthNotification{
&testDepthNotification{"10000000", 0},
&testDepthNotification{"01000000", 0},
&testDepthNotification{"01000001", 0},
&testDepthNotification{"00100000", 0},
},
)
}

45
swarm/network/protocol.go
View file

@ -33,10 +33,11 @@ import (
)
const (
NetworkId = 322 // BZZ in l33t
NetworkID = 322 // BZZ in l33t
ProtocolMaxMsgSize = 10 * 1024 * 1024
)
// BzzHandshakeSpec is the spec of the generic swarm handshake
var BzzHandshakeSpec = &protocols.Spec{
Name: "bzz",
Version: 1,
@ -113,6 +114,14 @@ func NewBzz(config *BzzConfig, kad Overlay, store StateStore) *Bzz {
}
}
func (b *Bzz) UpdateLocalAddr(byteaddr []byte) *bzzAddr {
b.localAddr.Update(&bzzAddr{
UAddr: byteaddr,
OAddr: b.localAddr.OAddr,
})
return b.localAddr
}
// Bzz implements the node.Service interface, offers Protocols
// * handshake/hive
// * discovery
@ -203,7 +212,7 @@ func (b *Bzz) getHandshake(peerID discover.NodeID) *bzzHandshake {
if !ok {
handshake = &bzzHandshake{
Version: uint64(BzzHandshakeSpec.Version),
NetworkId: uint64(NetworkId),
NetworkID: uint64(NetworkID),
Addr: b.localAddr,
done: make(chan struct{}),
}
@ -221,12 +230,12 @@ type bzzPeer struct {
lastActive time.Time // time is updated whenever mutexes are releasing
}
func newBzzPeer(p *protocols.Peer, over, under []byte) *bzzPeer {
return &bzzPeer{
Peer: p,
localAddr: &bzzAddr{over, under},
}
}
//func newBzzPeer(p *protocols.Peer, over, under []byte) *bzzPeer {
// return &bzzPeer{
// Peer: p,
// localAddr: &bzzAddr{over, under},
// }
//}
// Off returns the overlay peer record for offline persistance
func (self *bzzPeer) Off() OverlayAddr {
@ -242,12 +251,12 @@ func (self *bzzPeer) LastActive() time.Time {
Handshake
* Version: 8 byte integer version of the protocol
* NetworkId: 8 byte integer network identifier
* NetworkID: 8 byte integer network identifier
* Addr: the address advertised by the node including underlay and overlay connecctions
*/
type bzzHandshake struct {
Version uint64
NetworkId uint64
NetworkID uint64
Addr *bzzAddr
// peerAddr is the address received in the peer handshake
@ -258,7 +267,7 @@ type bzzHandshake struct {
}
func (self *bzzHandshake) String() string {
return fmt.Sprintf("Handshake: Version: %v, NetworkId: %v, Addr: %v", self.Version, self.NetworkId, self.Addr)
return fmt.Sprintf("Handshake: Version: %v, NetworkID: %v, Addr: %v", self.Version, self.NetworkID, self.Addr)
}
const bzzHandshakeTimeout = time.Second
@ -276,8 +285,8 @@ func (self *bzzHandshake) Perform(p *p2p.Peer, rw p2p.MsgReadWriter) (err error)
return err
}
rhs := hs.(*bzzHandshake)
if rhs.NetworkId != self.NetworkId {
return fmt.Errorf("network id mismatch %d (!= %d)", rhs.NetworkId, self.NetworkId)
if rhs.NetworkID != self.NetworkID {
return fmt.Errorf("network id mismatch %d (!= %d)", rhs.NetworkID, self.NetworkID)
}
if rhs.Version != self.Version {
return fmt.Errorf("version mismatch %d (!= %d)", rhs.Version, self.Version)
@ -333,15 +342,15 @@ func RandomAddr() *bzzAddr {
pubkey := crypto.FromECDSAPub(&key.PublicKey)
var id discover.NodeID
copy(id[:], pubkey[1:])
return &bzzAddr{
OAddr: crypto.Keccak256(pubkey[1:]),
UAddr: id[:],
}
return NewAddrFromNodeID(id)
}
// NewNodeIDFromAddr transforms the underlay address to an adapters.NodeID
func NewNodeIDFromAddr(addr Addr) discover.NodeID {
return discover.MustBytesID(addr.Under())
log.Info(fmt.Sprintf("uaddr=%s", string(addr.Under())))
node := discover.MustParseNode(string(addr.Under()))
// return discover.MustBytesID(addr.Under())
return node.ID
}
// NewAddrFromNodeID constucts a bzzAddr from a discover.NodeID

26
swarm/network/protocol_test.go
View file

@ -1,3 +1,19 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package network
import (
@ -140,12 +156,12 @@ func (s *bzzTester) runHandshakes(ids ...discover.NodeID) {
func correctBzzHandshake(addr *bzzAddr) *bzzHandshake {
return &bzzHandshake{
Version: 0,
NetworkId: 322,
NetworkID: 322,
Addr: addr,
}
}
func TestBzzHandshakeNetworkIdMismatch(t *testing.T) {
func TestBzzHandshakeNetworkIDMismatch(t *testing.T) {
pp := p2ptest.NewTestPeerPool()
addr := RandomAddr()
s := newBzzTester(t, 1, addr, pp, nil, nil)
@ -154,7 +170,7 @@ func TestBzzHandshakeNetworkIdMismatch(t *testing.T) {
id := s.IDs[0]
s.testHandshake(
correctBzzHandshake(addr),
&bzzHandshake{Version: 0, NetworkId: 321, Addr: NewAddrFromNodeID(id)},
&bzzHandshake{Version: 0, NetworkID: 321, Addr: NewAddrFromNodeID(id)},
&p2ptest.Disconnect{Peer: id, Error: fmt.Errorf("network id mismatch 321 (!= 322)")},
)
}
@ -168,7 +184,7 @@ func TestBzzHandshakeVersionMismatch(t *testing.T) {
id := s.IDs[0]
s.testHandshake(
correctBzzHandshake(addr),
&bzzHandshake{Version: 1, NetworkId: 322, Addr: NewAddrFromNodeID(id)},
&bzzHandshake{Version: 1, NetworkID: 322, Addr: NewAddrFromNodeID(id)},
&p2ptest.Disconnect{Peer: id, Error: fmt.Errorf("version mismatch 1 (!= 0)")},
)
}
@ -182,6 +198,6 @@ func TestBzzHandshakeSuccess(t *testing.T) {
id := s.IDs[0]
s.testHandshake(
correctBzzHandshake(addr),
&bzzHandshake{Version: 0, NetworkId: 322, Addr: NewAddrFromNodeID(id)},
&bzzHandshake{Version: 0, NetworkID: 322, Addr: NewAddrFromNodeID(id)},
)
}

39
swarm/network/simulations/discovery/discovery_test.go
View file

@ -2,6 +2,8 @@ package discovery_test
import (
"context"
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"os"
@ -26,13 +28,23 @@ var services = adapters.Services{
serviceName: newService,
}
var (
snapshotFile = flag.String("snapshot", "", "create snapshot")
nodeCount = flag.Int("nodes", 10, "number of nodes to create (default 10)")
verbose = flag.Bool("verbose", false, "output extra logs")
)
func init() {
flag.Parse()
// register the discovery service which will run as a devp2p
// protocol when using the exec adapter
adapters.RegisterServices(services)
// log.Root().SetHandler(log.LvlFilterHandler(log.LvlError, log.StreamHandler(os.Stderr, log.TerminalFormat(false))))
log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(false))))
if *verbose {
log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(false))))
} else {
log.Root().SetHandler(log.LvlFilterHandler(log.LvlError, log.StreamHandler(os.Stderr, log.TerminalFormat(false))))
}
}
func TestDiscoverySimulationDockerAdapter(t *testing.T) {
@ -57,16 +69,15 @@ func TestDiscoverySimulationSimAdapter(t *testing.T) {
}
func testDiscoverySimulation(t *testing.T, adapter adapters.NodeAdapter) {
// create 10 node network
nodeCount := 10
// create network
net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{
ID: "0",
DefaultService: serviceName,
})
defer net.Shutdown()
trigger := make(chan discover.NodeID)
ids := make([]discover.NodeID, nodeCount)
for i := 0; i < nodeCount; i++ {
ids := make([]discover.NodeID, *nodeCount)
for i := 0; i < *nodeCount; i++ {
node, err := net.NewNode()
if err != nil {
t.Fatalf("error starting node: %s", err)
@ -135,6 +146,22 @@ func testDiscoverySimulation(t *testing.T, adapter adapters.NodeAdapter) {
t.Fatalf("simulation failed: %s", result.Error)
}
if *snapshotFile != "" {
snap, err := net.Snapshot()
if err != nil {
t.Fatalf("no shapshot dude")
}
jsonsnapshot, err := json.Marshal(snap)
if err != nil {
t.Fatalf("corrupt json snapshot: %v", err)
}
log.Info("writing snapshot", "file", *snapshotFile)
err = ioutil.WriteFile(*snapshotFile, jsonsnapshot, 0755)
if err != nil {
t.Fatal(err)
}
}
t.Log("Simulation Passed:")
t.Logf("Duration: %s", result.FinishedAt.Sub(result.StartedAt))
for _, id := range ids {

9
swarm/network/simulations/overlay.go
View file

@ -51,7 +51,7 @@ func (s *Simulation) NewService(ctx *adapters.ServiceContext) (node.Service, err
kp.MinBinSize = 1
kp.MaxRetries = 1000
kp.RetryExponent = 2
kp.RetryInterval = 1000
kp.RetryInterval = 1000000
kp.PruneInterval = 2000
kad := network.NewKademlia(addr.Over(), kp)
ticker := time.NewTicker(time.Duration(kad.PruneInterval) * time.Millisecond)
@ -97,7 +97,7 @@ func setupMocker(net *simulations.Network) []discover.NodeID {
conf := net.Config()
conf.DefaultService = "overlay"
nodeCount := 60
nodeCount := 30
ids := make([]discover.NodeID, nodeCount)
for i := 0; i < nodeCount; i++ {
node, err := net.NewNode()
@ -209,8 +209,9 @@ func main() {
config := &simulations.ServerConfig{
NewAdapter: func() adapters.NodeAdapter { return adapters.NewSimAdapter(services) },
DefaultMockerID: "bootNet",
Mockers: mockers,
DefaultMockerID: "randomNodes",
// DefaultMockerID: "bootNet",
Mockers: mockers,
}
log.Info("starting simulation server on 0.0.0.0:8888...")

193
swarm/network/test_overlay.go
View file

@ -1,193 +0,0 @@
package network
//
// import (
// "fmt"
// "strings"
// "sync"
//
// "github.com/ethereum/go-ethereum/log"
// )
//
// const orders = 8
//
// type testOverlay struct {
// mu sync.Mutex
// addr []byte
// pos [][]OverlayAddr
// posMap map[string]OverlayAddr
// }
//
// type testPeerAddr struct {
// Addr
// Peer
// }
//
// func (self *testPeerAddr) Address() []byte {
// return nil
// }
//
// func (self *testPeerAddr) Update(a OverlayAddr) OverlayAddr {
// return self
// }
//
// func (self *testPeerAddr) On(p OverlayConn) OverlayConn {
// return self
// }
//
// func (self *testPeerAddr) Off() OverlayAddr {
// return self
// }
//
// func (self *testOverlay) Register(peers chan OverlayAddr) error {
// self.mu.Lock()
// defer self.mu.Unlock()
// var nas []OverlayAddr
// for a := range peers {
// nas = append(nas, a)
// }
// return self.register(nas...)
// }
//
// func (self *testOverlay) BaseAddr() []byte {
// return nil
// }
//
// func (self *testOverlay) register(nas ...OverlayAddr) error {
// for _, na := range nas {
// addr := na.Address()
// if self.posMap[string(addr)] != nil {
// continue
// }
// self.posMap[string(addr)] = na
// o := order(addr)
// log.Trace(fmt.Sprintf("PO: %v, orders: %v", o, orders))
// self.pos[o] = append(self.pos[o], na)
// }
// return nil
// }
//
// func order(addr []byte) int {
// return int(addr[0]) / 32
// }
//
// func (self *testOverlay) On(n OverlayConn) {
// self.mu.Lock()
// defer self.mu.Unlock()
// addr := n.Address()
// na := self.posMap[string(addr)]
// if na == nil {
// self.register(n)
// na = self.posMap[string(addr)]
// } else if na.Peer != nil {
// return
// }
// log.Trace(fmt.Sprintf("Online: %x", addr[:4]))
// na.Peer = n
// return
// }
//
// func (self *testOverlay) Off(n OverlayConn) {
// self.mu.Lock()
// defer self.mu.Unlock()
// addr := n.Over()
// na := self.posMap[string(addr)]
// if na == nil {
// return
// }
// delete(self.posMap, string(addr))
// na.Peer = nil
// }
//
// // caller must hold the lock
// func (self *testOverlay) on(po []*testPeerAddr) (nodes []OverlayConn) {
// for _, na := range po {
// if na.Peer != nil {
// nodes = append(nodes, na)
// }
// }
// return nodes
// }
//
// // caller must hold the lock
// func (self *testOverlay) off(po []*testPeerAddr) (nas []OverlayAddr) {
// for _, na := range po {
// if na.Peer == (*bzzPeer)(nil) {
// nas = append(nas, Addr(na))
// }
// }
// return nas
// }
//
// func (self *testOverlay) EachConn(base []byte, o int, f func(OverlayConn, int, bool) bool) {
// for i := o; i < len(self.pos); i++ {
// for _, na := range self.pos[i] {
// if na.Peer != nil {
// if !f(na, o, false) {
// return
// }
// }
// }
// }
// }
//
// func (self *testOverlay) EachAddr(base []byte, o int, f func(OverlayAddr, int) bool) {
// for i := o; i < len(self.pos); i++ {
// for _, na := range self.pos[i] {
// if !f(na, i) {
// return
// }
// }
// }
// }
//
// func (self *testOverlay) SuggestPeer() (OverlayAddr, int, bool) {
// self.mu.Lock()
// defer self.mu.Unlock()
// for i, po := range self.pos {
// ons := self.on(po)
// if len(ons) < 2 {
// offs := self.off(po)
// if len(offs) > 0 {
// log.Trace(fmt.Sprintf("node %v is off", offs[0]))
// return offs[0], i, true
// }
// }
// }
// return nil, 0, true
// }
//
// func (self *testOverlay) String() string {
// self.mu.Lock()
// defer self.mu.Unlock()
// var t []string
// var ons, offs int
// var ns []Peer
// var nas []Addr
// for o, po := range self.pos {
// var row []string
// ns = self.on(po)
// nas = self.off(po)
// ons = len(ns)
// for _, n := range ns {
// addr := n.Over()
// row = append(row, fmt.Sprintf("%x", addr[:4]))
// }
// row = append(row, "|")
// offs = len(nas)
// for _, na := range nas {
// addr := na.Over()
// row = append(row, fmt.Sprintf("%x", addr[:4]))
// }
// t = append(t, fmt.Sprintf("%v: (%v/%v) %v", o, ons, offs, strings.Join(row, " ")))
// }
// return strings.Join(t, "\n")
// }
//
// func NewTestOverlay(addr []byte) *testOverlay {
// return &testOverlay{
// addr: addr,
// posMap: make(map[string]*testPeerAddr),
// pos: make([][]*testPeerAddr, orders),
// }
// }

185
swarm/pss/README.md Normal file
View file

@ -0,0 +1,185 @@
# Postal Service over Swarm
pss provides devp2p functionality for swarm nodes without the need for a direct tcp connection between them.
It uses swarm kademlia routing to send and receive messages. Routing is deterministic and will seek the shortest route available on the network.
Messages are encapsulated in a devp2p message structure `PssMsg`. These capsules are forwarded from node to node using ordinary tcp devp2p until it reaches it's destination. The destination address is hinted in `PssMsg.To`
The content of a PssMsg can be anything at all, down to a simple, non-descript byte-slices. But convenience methods are made available to implement devp2p protocol functionality on top of it.
In its final implementation, pss is intended to become "shh over bzz," that is; "whisper over swarm." Specifically, this means that the emphemeral encryption envelopes of whisper will be used to obfuscate the correspondance. Ideally, the unencrypted content of the PssMsg will only contain a part of the address of the recipient, where the final recipient is the one who matches this partial address *and* successfully can encrypt the message.
For the current state and roadmap of pss development please see https://github.com/ethersphere/swarm/wiki/swarm-dev-progress.
Please report issues on https://github.com/ethersphere/go-ethereum
Feel free to ask questions in https://gitter.im/ethersphere/pss
## TL;DR IMPLEMENTATION
Most developers will most probably want to use the protocol-wrapping convenience client in swarm/pss/client. Documentation and a minimal code example for the latter is found in the package documentation. The pss API can of course also be used directly. The client implementation provides a clear illustration of its intended usage.
pss implements the node.Service interface. This means that the API methods will be auto-magically exposed to any RPC layer the node activates. In particular, pss provides subscription to incoming messages using the go-ethereum rpc websocket layer.
The important API methods are:
- Receive() - start a subscription to receive new incoming messages matching specific "topics"
- Send() - send content over pss to a specified recipient
## LOWLEVEL IMPLEMENTATION
code speaks louder than words:
import (
"io/ioutil"
"os"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/pss"
"github.com/ethereum/go-ethereum/swarm/network"
"github.com/ethereum/go-ethereum/swarm/storage"
)
var (
righttopic = pss.NewTopic("foo", 4)
wrongtopic = pss.NewTopic("bar", 2)
)
// if you want to see what's going on
func init() {
hs := log.StreamHandler(os.Stderr, log.TerminalFormat(true))
hf := log.LvlFilterHandler(log.LvlTrace, hs)
h := log.CallerFileHandler(hf)
log.Root().SetHandler(h)
}
// Pss.Handler type
func handler(msg []byte, p *p2p.Peer, from []byte) error {
log.Debug("received", "msg", msg, "from", from, "forwarder", p.ID())
return nil
}
func implementation() {
// bogus addresses for illustration purposes
meaddr := network.RandomAddr()
toaddr := network.RandomAddr()
fwdaddr := network.RandomAddr()
// new kademlia for routing
kp := network.NewKadParams()
to := network.NewKademlia(meaddr.Over(), kp)
// new (local) storage for cache
cachedir, err := ioutil.TempDir("", "pss-cache")
if err != nil {
panic("overlay")
}
dpa, err := storage.NewLocalDPA(cachedir)
if err != nil {
panic("storage")
}
// setup pss
psp := pss.NewPssParams(false)
ps := pss.NewPss(to, dpa, psp)
// does nothing but please include it
ps.Start(nil)
dereg := ps.Register(&righttopic, handler)
// in its simplest form a message is just a byteslice
payload := []byte("foobar")
// send a raw message
err = ps.SendRaw(toaddr.Over(), righttopic, payload)
log.Error("Fails. Not connect, so nothing in kademlia. But it illustrates the point.", "err", err)
// forward a full message
envfwd := pss.NewEnvelope(fwdaddr.Over(), righttopic, payload)
msgfwd := &pss.PssMsg{
To: toaddr.Over(),
Payload: envfwd,
}
err = ps.Forward(msgfwd)
log.Error("Also fails, same reason. I wish, I wish, I wish there was somebody out there.", "err", err)
// process an incoming message
// (this is the first step after the devp2p PssMsg message handler)
envme := pss.NewEnvelope(toaddr.Over(), righttopic, payload)
msgme := &pss.PssMsg{
To: meaddr.Over(),
Payload: envme,
}
err = ps.Process(msgme)
if err == nil {
log.Info("this works :)")
}
// if we don't have a registered topic it fails
dereg() // remove the previously registered topic-handler link
ps.Process(msgme)
log.Error("It fails as we expected", "err", err)
// does nothing but please include it
ps.Stop()
}
## MESSAGE STRUCTURE
NOTE! This part is subject to change. In particular the envelope structure will be re-implemented using whisper.
A pss message has the following layers:
- PssMsg
Contains (eventually only part of) recipient address, and (eventually) encrypted Envelope.
- Envelope
Currently rlp-encoded. Contains the Payload, along with sender address, topic and expiry information.
- Payload
Byte-slice of arbitrary data
- ProtocolMsg
An optional convenience structure for implementation of devp2p protocols. Contains Code, Size and Payload analogous to the p2p.Msg structure, where the payload is a rlp-encoded byteslice. For transport, this struct is serialized and used as the "payload" above.
## TOPICS AND PROTOCOLS
Pure pss is protocol agnostic. Instead it uses the notion of Topic. This is NOT the "subject" of a message. Instead this type is used to internally register handlers for messages matching respective Topics.
Topic in this context virtually mean anything; protocols, chatrooms, or social media groups.
When implementing devp2p protocols, topics are direct mappings to protocols name and version. The pss package provides the PssProtocol convenience structure, and a generic Handler that can be passed to Pss.Register. This makes it possible to use the same message handler code for pss that are used for direct connected peers.
## CONNECTIONS
A "connection" in pss is a purely virtual construct. There is no mechanisms in place to ensure that the remote peer actually is there. In fact, "adding" a peer involves merely the node's opinion that the peer is there. It may issue messages to that remote peer to a directly connected peer, which in turn passes it on. But if it is not present on the network - or if there is no route to it - the message will never reach its destination through mere forwarding.
When implementing the devp2p protocol stack, the "adding" of a remote peer is a prerequisite for the side actually initiating the protocol communication. Adding a peer in effect "runs" the protocol on that peer, and adds an internal mapping between a topic and that peer. It also enables sending and receiving messages using the main io-construct in devp2p - the p2p.MsgReadWriter.
Under the hood, pss implements its own MsgReadWriter, which bridges MsgReadWriter.WriteMsg with Pss.SendRaw, and deftly adds an InjectMsg method which pipes incoming messages to appear on the MsgReadWriter.ReadMsg channel.
An incoming connection is nothing more than an actual PssMsg appearing with a certain Topic. If a Handler har been registered to that Topic, the message will be passed to it. This constitutes a "new" connection if:
- The pss node never called AddPeer with this combination of remote peer address and topic, and
- The pss node never received a PssMsg from this remote peer with this specific Topic before.
If it is a "new" connection, the protocol will be "run" on the remote peer, in the same manner as if it was pre-emptively added.
## ROUTING AND CACHING
(please refer to swarm kademlia routing for an explanation of the routing algorithm used for pss)
pss implements a simple caching mechanism, using the swarm DPA for storage of the messages and generation of the digest keys used in the cache table. The caching is intended to alleviate the following:
- save messages so that they can be delivered later if the recipient was not online at the time of sending.
- drop an identical message to the same recipient if received within a given time interval
- prevent backwards routing of messages
the latter may occur if only one entry is in the receiving node's kademlia. In this case the forwarder will be provided as the "nearest node" to the final recipient. The cache keeps the address of who the message was forwarded from, and if the cache lookup matches, the message will be dropped.

83
swarm/pss/client/README.md Normal file
View file

@ -0,0 +1,83 @@
# pss client
simple abstraction for implementing pss functionality
the pss client library aims to simplify usage of the p2p.protocols package over pss
IO is performed using the ordinary p2p.MsgReadWriter interface, which transparently communicates with a pss node via RPC using websockets as transport layer, using methods in the PssAPI class in the swarm/pss package
## USAGE
Minimal-ish usage example. It needs a running pss-enabled swarm node to work. Please refer to the test files for more details.
import (
"context"
"fmt"
"os"
pss "github.com/ethereum/go-ethereum/swarm/pss/client"
"github.com/ethereum/go-ethereum/p2p/protocols"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/pot"
"github.com/ethereum/go-ethereum/log"
)
type FooMsg struct {
Bar int
}
func fooHandler (msg interface{}) error {
foomsg, ok := msg.(*FooMsg)
if ok {
log.Debug("Yay, just got a message", "msg", foomsg)
}
return fmt.Errorf("Unknown message")
}
spec := &protocols.Spec{
Name: "foo",
Version: 1,
MaxMsgSize: 1024,
Messages: []interface{}{
FooMsg{},
},
}
proto := &p2p.Protocol{
Name: spec.Name,
Version: spec.Version,
Length: uint64(len(spec.Messages)),
Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error {
pp := protocols.NewPeer(p, rw, spec)
return pp.Run(fooHandler)
},
}
func implementation() {
cfg := pss.NewClientConfig()
psc := pss.NewClient(context.Background(), nil, cfg)
err := psc.Start()
if err != nil {
log.Crit("can't start pss client")
os.Exit(1)
}
log.Debug("connected to pss node", "bzz addr", psc.BaseAddr)
err = psc.RunProtocol(proto)
if err != nil {
log.Crit("can't start protocol on pss websocket")
os.Exit(1)
}
addr := pot.RandomAddress() // should be a real address, of course
psc.AddPssPeer(addr, spec)
// use the protocol for something
psc.Stop()
}
BUG(test): TestIncoming test times out due to deadlock issues in the swarm hive

186
swarm/pss/client/client.go
View file

@ -6,7 +6,6 @@ import (
"sync"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/discover"
@ -20,44 +19,30 @@ import (
const (
inboxCapacity = 3000
outboxCapacity = 100
defaultWSHost = 8546
addrLen = common.HashLength
)
// RemoteHost: hostname of node running websockets proxy to pss (default localhost)
// RemotePort: port of node running websockets proxy to pss (0 = go-ethereum node default)
// Secure: whether or not to use secure connection
// SelfHost: local if host to connect from
type ClientConfig struct {
SelfHost string
RemoteHost string
RemotePort int
Secure bool
}
func NewClientConfig() *ClientConfig {
return &ClientConfig{
SelfHost: "localhost",
RemoteHost: "localhost",
RemotePort: 8546,
}
}
// After a successful connection with Client.Start, BaseAddr contains the swarm overlay address of the pss node
type Client struct {
localuri string
remoteuri string
ctx context.Context
cancel func()
subscription *rpc.ClientSubscription
topicsC chan []byte
msgC chan pss.APIMsg
quitC chan struct{}
ws *rpc.Client
lock sync.Mutex
peerPool map[pss.Topic]map[pot.Address]*pssRPCRW
protos map[pss.Topic]*p2p.Protocol
BaseAddr []byte
// peers
peerPool map[pss.Topic]map[pot.Address]*pssRPCRW
protos map[pss.Topic]*p2p.Protocol
// rpc connections
rpc *rpc.Client
sub *rpc.ClientSubscription
// channels
topicsC chan []byte
msgC chan pss.APIMsg
quitC chan struct{}
lock sync.Mutex
}
// implements p2p.MsgReadWriter
type pssRPCRW struct {
*Client
topic *pss.Topic
@ -97,81 +82,63 @@ func (rw *pssRPCRW) WriteMsg(msg p2p.Msg) error {
if err != nil {
return err
}
return rw.Client.ws.CallContext(rw.Client.ctx, nil, "pss_send", rw.topic, pss.APIMsg{
return rw.Client.rpc.Call(nil, "pss_send", rw.topic, pss.APIMsg{
Addr: rw.addr.Bytes(),
Msg: pmsg,
})
}
func NewClient(ctx context.Context, cancel func(), config *ClientConfig) *Client {
prefix := "ws"
if ctx == nil {
ctx = context.Background()
}
if cancel == nil {
cancel = func() { return }
}
pssc := &Client{
msgC: make(chan pss.APIMsg),
quitC: make(chan struct{}),
peerPool: make(map[pss.Topic]map[pot.Address]*pssRPCRW),
protos: make(map[pss.Topic]*p2p.Protocol),
ctx: ctx,
cancel: cancel,
}
if config.Secure {
prefix = "wss"
}
pssc.remoteuri = fmt.Sprintf("%s://%s:%d", prefix, config.RemoteHost, config.RemotePort)
pssc.localuri = fmt.Sprintf("%s://%s", prefix, config.SelfHost)
return pssc
}
func NewClientWithRPC(ctx context.Context, client *rpc.Client) *Client {
return &Client{
msgC: make(chan pss.APIMsg),
quitC: make(chan struct{}),
peerPool: make(map[pss.Topic]map[pot.Address]*pssRPCRW),
protos: make(map[pss.Topic]*p2p.Protocol),
ws: client,
ctx: ctx,
}
}
func (self *Client) shutdown() {
self.cancel()
}
func (self *Client) Start() error {
if self.ws != nil {
return nil
}
log.Debug("Dialing ws", "src", self.localuri, "dst", self.remoteuri)
ws, err := rpc.DialWebsocket(self.ctx, self.remoteuri, self.localuri)
func NewClient(rpcurl string) (*Client, error) {
rpcclient, err := rpc.Dial(rpcurl)
if err != nil {
return fmt.Errorf("Couldnt dial pss websocket: %v", err)
return nil, err
}
self.ws = ws
return nil
client, err := NewClientWithRPC(rpcclient)
if err != nil {
return nil, err
}
return client, nil
}
func (self *Client) RunProtocol(proto *p2p.Protocol) error {
// Constructor for test implementations
// The 'rpcclient' parameter allows passing a in-memory rpc client to act as the remote websocket RPC.
func NewClientWithRPC(rpcclient *rpc.Client) (*Client, error) {
client := newClient()
client.rpc = rpcclient
err := client.rpc.Call(&client.BaseAddr, "pss_baseAddr")
if err != nil {
return nil, fmt.Errorf("cannot get pss node baseaddress: %v", err)
}
return client, nil
}
func newClient() (client *Client) {
client = &Client{
msgC: make(chan pss.APIMsg),
quitC: make(chan struct{}),
peerPool: make(map[pss.Topic]map[pot.Address]*pssRPCRW),
protos: make(map[pss.Topic]*p2p.Protocol),
}
return
}
// Mounts a new devp2p protcool on the pss connection
// the protocol is aliased as a "pss topic"
// uses normal devp2p Send and incoming message handler routines from the p2p/protocols package
//
// when an incoming message is received from a peer that is not yet known to the client, this peer object is instantiated, and the protocol is run on it.
func (self *Client) RunProtocol(ctx context.Context, proto *p2p.Protocol) error {
topic := pss.NewTopic(proto.Name, int(proto.Version))
msgC := make(chan pss.APIMsg)
self.peerPool[topic] = make(map[pot.Address]*pssRPCRW)
sub, err := self.ws.Subscribe(self.ctx, "pss", msgC, "receive", topic)
sub, err := self.rpc.Subscribe(ctx, "pss", msgC, "receive", topic)
if err != nil {
return fmt.Errorf("pss event subscription failed: %v", err)
}
self.subscription = sub
self.sub = sub
// dispatch incoming messages
go func() {
@ -190,7 +157,6 @@ func (self *Client) RunProtocol(proto *p2p.Protocol) error {
self.peerPool[topic][addr].msgC <- msg.Msg
}()
case <-self.quitC:
self.shutdown()
return
}
}
@ -200,13 +166,18 @@ func (self *Client) RunProtocol(proto *p2p.Protocol) error {
return nil
}
// Always call this to ensure that we exit cleanly
func (self *Client) Stop() error {
self.cancel()
return nil
}
// Preemptively add a remote pss peer
func (self *Client) AddPssPeer(addr pot.Address, spec *protocols.Spec) {
topic := pss.NewTopic(spec.Name, int(spec.Version))
if self.peerPool[topic] == nil {
log.Error("addpeer on unset topic")
return
}
if self.peerPool[topic][addr] == nil {
self.peerPool[topic][addr] = self.newpssRPCRW(addr, &topic)
nid, _ := discover.HexID("0x00")
@ -215,31 +186,10 @@ func (self *Client) AddPssPeer(addr pot.Address, spec *protocols.Spec) {
}
}
// Remove a remote pss peer
//
// Note this doesn't actually currently drop the peer, but only remmoves the reference from the client's peer lookup table
func (self *Client) RemovePssPeer(addr pot.Address, spec *protocols.Spec) {
topic := pss.NewTopic(spec.Name, int(spec.Version))
delete(self.peerPool[topic], addr)
}
func (self *Client) SubscribeEvents(ch chan *p2p.PeerEvent) event.Subscription {
log.Error("PSS client handles events internally, use the read functions instead")
return nil
}
func (self *Client) PeerCount() int {
return len(self.peerPool)
}
func (self *Client) NodeInfo() *p2p.NodeInfo {
return nil
}
func (self *Client) PeersInfo() []*p2p.PeerInfo {
return nil
}
func (self *Client) AddPeer(node *discover.Node) {
log.Error("Cannot add peer in PSS with discover.Node, need swarm overlay address")
}
func (self *Client) RemovePeer(node *discover.Node) {
log.Error("Cannot remove peer in PSS with discover.Node, need swarm overlay address")
}

49
swarm/pss/client/client_test.go
View file

@ -30,7 +30,7 @@ func TestRunProtocol(t *testing.T) {
C: make(chan struct{}),
}
proto := newProtocol(ping)
_, err := baseTester(t, proto, ps, nil, nil, quitC)
_, err := baseTester(t, proto, ps, nil, quitC)
if err != nil {
t.Fatalf(err.Error())
}
@ -40,18 +40,18 @@ func TestRunProtocol(t *testing.T) {
func TestIncoming(t *testing.T) {
quitC := make(chan struct{})
ps := pss.NewTestPss(nil)
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
ctx, _ := context.WithTimeout(context.Background(), time.Second*5)
var addr []byte
ping := &pss.Ping{
C: make(chan struct{}),
}
proto := newProtocol(ping)
client, err := baseTester(t, proto, ps, ctx, cancel, quitC)
client, err := baseTester(t, proto, ps, ctx, quitC)
if err != nil {
t.Fatalf(err.Error())
}
client.ws.Call(&addr, "psstest_baseAddr")
client.rpc.Call(&addr, "psstest_baseAddr")
code, _ := pss.PingProtocol.GetCode(&pss.PingMsg{})
rlpbundle, err := pss.NewProtocolMsg(code, &pss.PingMsg{
@ -69,11 +69,7 @@ func TestIncoming(t *testing.T) {
ps.Process(&pssmsg)
select {
case <-client.ctx.Done():
t.Fatalf("outgoing timed out or canceled")
case <-ping.C:
}
<-ping.C
quitC <- struct{}{}
}
@ -81,7 +77,7 @@ func TestIncoming(t *testing.T) {
func TestOutgoing(t *testing.T) {
quitC := make(chan struct{})
ps := pss.NewTestPss(nil)
ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*250)
ctx, _ := context.WithTimeout(context.Background(), time.Millisecond*250)
var addr []byte
var potaddr pot.Address
@ -89,12 +85,12 @@ func TestOutgoing(t *testing.T) {
C: make(chan struct{}),
}
proto := newProtocol(ping)
client, err := baseTester(t, proto, ps, ctx, cancel, quitC)
client, err := baseTester(t, proto, ps, ctx, quitC)
if err != nil {
t.Fatalf(err.Error())
}
client.ws.Call(&addr, "psstest_baseAddr")
client.rpc.Call(&addr, "psstest_baseAddr")
copy(potaddr[:], addr)
msg := &pss.PingMsg{
@ -107,25 +103,16 @@ func TestOutgoing(t *testing.T) {
p := p2p.NewPeer(nid, fmt.Sprintf("%v", potaddr), []p2p.Cap{})
pp := protocols.NewPeer(p, client.peerPool[topic][potaddr], pss.PingProtocol)
pp.Send(msg)
select {
case <-client.ctx.Done():
t.Fatalf("outgoing timed out or canceled")
case <-ping.C:
}
<-ping.C
quitC <- struct{}{}
}
func baseTester(t *testing.T, proto *p2p.Protocol, ps *pss.Pss, ctx context.Context, cancel func(), quitC chan struct{}) (*Client, error) {
func baseTester(t *testing.T, proto *p2p.Protocol, ps *pss.Pss, ctx context.Context, quitC chan struct{}) (*Client, error) {
var err error
client := newClient(t, ctx, cancel, quitC)
client := newTestclient(t, quitC)
err = client.Start()
if err != nil {
return nil, err
}
err = client.RunProtocol(proto)
err = client.RunProtocol(context.Background(), proto)
if err != nil {
return nil, err
@ -148,11 +135,7 @@ func newProtocol(ping *pss.Ping) *p2p.Protocol {
}
}
func newClient(t *testing.T, ctx context.Context, cancel func(), quitC chan struct{}) *Client {
conf := NewClientConfig()
pssclient := NewClient(ctx, cancel, conf)
func newTestclient(t *testing.T, quitC chan struct{}) *Client {
ps := pss.NewTestPss(nil)
srv := rpc.NewServer()
@ -174,5 +157,11 @@ func newClient(t *testing.T, ctx context.Context, cancel func(), quitC chan stru
<-quitC
sock.Close()
}()
pssclient, err := NewClient("ws://localhost:8546")
if err != nil {
t.Fatalf(err.Error())
}
return pssclient
}

80
swarm/pss/client/doc.go Normal file
View file

@ -0,0 +1,80 @@
// simple abstraction for implementing pss functionality
//
// the pss client library aims to simplify usage of the p2p.protocols package over pss
//
// IO is performed using the ordinary p2p.MsgReadWriter interface, which transparently communicates with a pss node via RPC using websockets as transport layer, using methods in the PssAPI class in the swarm/pss package
//
//
// Minimal-ish usage example (requires a running pss node with websocket RPC):
//
//
// import (
// "context"
// "fmt"
// "os"
// pss "github.com/ethereum/go-ethereum/swarm/pss/client"
// "github.com/ethereum/go-ethereum/p2p/protocols"
// "github.com/ethereum/go-ethereum/p2p"
// "github.com/ethereum/go-ethereum/pot"
// "github.com/ethereum/go-ethereum/log"
// )
//
// type FooMsg struct {
// Bar int
// }
//
//
// func fooHandler (msg interface{}) error {
// foomsg, ok := msg.(*FooMsg)
// if ok {
// log.Debug("Yay, just got a message", "msg", foomsg)
// }
// return fmt.Errorf("Unknown message")
// }
//
// spec := &protocols.Spec{
// Name: "foo",
// Version: 1,
// MaxMsgSize: 1024,
// Messages: []interface{}{
// FooMsg{},
// },
// }
//
// proto := &p2p.Protocol{
// Name: spec.Name,
// Version: spec.Version,
// Length: uint64(len(spec.Messages)),
// Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error {
// pp := protocols.NewPeer(p, rw, spec)
// return pp.Run(fooHandler)
// },
// }
//
// func implementation() {
// cfg := pss.NewClientConfig()
// psc := pss.NewClient(context.Background(), nil, cfg)
// err := psc.Start()
// if err != nil {
// log.Crit("can't start pss client")
// os.Exit(1)
// }
//
// log.Debug("connected to pss node", "bzz addr", psc.BaseAddr)
//
// err = psc.RunProtocol(proto)
// if err != nil {
// log.Crit("can't start protocol on pss websocket")
// os.Exit(1)
// }
//
// addr := pot.RandomAddress() // should be a real address, of course
// psc.AddPssPeer(addr, spec)
//
// // use the protocol for something
//
// psc.Stop()
// }
//
// BUG(test): TestIncoming test times out due to deadlock issues in the swarm hive
package client

183
swarm/pss/doc.go Normal file
View file

@ -0,0 +1,183 @@
// pss provides devp2p functionality for swarm nodes without the need for a direct tcp connection between them.
//
// It uses swarm kademlia routing to send and receive messages. Routing is deterministic and will seek the shortest route available on the network.
//
// Messages are encapsulated in a devp2p message structure `PssMsg`. These capsules are forwarded from node to node using ordinary tcp devp2p until it reaches it's destination. The destination address is hinted in `PssMsg.To`
//
// The content of a PssMsg can be anything at all, down to a simple, non-descript byte-slices. But convenience methods are made available to implement devp2p protocol functionality on top of it.
//
// In its final implementation, pss is intended to become "shh over bzz," that is; "whisper over swarm." Specifically, this means that the emphemeral encryption envelopes of whisper will be used to obfuscate the correspondance. Ideally, the unencrypted content of the PssMsg will only contain a part of the address of the recipient, where the final recipient is the one who matches this partial address *and* successfully can encrypt the message.
//
// For the current state and roadmap of pss development please see https://github.com/ethersphere/swarm/wiki/swarm-dev-progress.
//
// Please report issues on https://github.com/ethersphere/go-ethereum
//
// Feel free to ask questions in https://gitter.im/ethersphere/pss
//
// TLDR IMPLEMENTATION
//
// Most developers will most probably want to use the protocol-wrapping convenience client in swarm/pss/client. Documentation and a minimal code example for the latter is found in the package documentation. The pss API can of course also be used directly. The client implementation provides a clear illustration of its intended usage.
//
// pss implements the node.Service interface. This means that the API methods will be auto-magically exposed to any RPC layer the node activates. In particular, pss provides subscription to incoming messages using the go-ethereum rpc websocket layer.
//
// The important API methods are:
// - Receive() - start a subscription to receive new incoming messages matching specific "topics"
// - Send() - send content over pss to a specified recipient
//
//
// LOWLEVEL IMPLEMENTATION
//
// code speaks louder than words:
//
// import (
// "io/ioutil"
// "os"
// "github.com/ethereum/go-ethereum/p2p"
// "github.com/ethereum/go-ethereum/log"
// "github.com/ethereum/go-ethereum/swarm/pss"
// "github.com/ethereum/go-ethereum/swarm/network"
// "github.com/ethereum/go-ethereum/swarm/storage"
// )
//
// var (
// righttopic = pss.NewTopic("foo", 4)
// wrongtopic = pss.NewTopic("bar", 2)
// )
//
// func init() {
// hs := log.StreamHandler(os.Stderr, log.TerminalFormat(true))
// hf := log.LvlFilterHandler(log.LvlTrace, hs)
// h := log.CallerFileHandler(hf)
// log.Root().SetHandler(h)
// }
//
//
// // Pss.Handler type
// func handler(msg []byte, p *p2p.Peer, from []byte) error {
// log.Debug("received", "msg", msg, "from", from, "forwarder", p.ID())
// return nil
// }
//
// func implementation() {
//
// // bogus addresses for illustration purposes
// meaddr := network.RandomAddr()
// toaddr := network.RandomAddr()
// fwdaddr := network.RandomAddr()
//
// // new kademlia for routing
// kp := network.NewKadParams()
// to := network.NewKademlia(meaddr.Over(), kp)
//
// // new (local) storage for cache
// cachedir, err := ioutil.TempDir("", "pss-cache")
// if err != nil {
// panic("overlay")
// }
// dpa, err := storage.NewLocalDPA(cachedir)
// if err != nil {
// panic("storage")
// }
//
// // setup pss
// psp := pss.NewPssParams(false)
// ps := pss.NewPss(to, dpa, psp)
//
// // does nothing but please include it
// ps.Start(nil)
//
// dereg := ps.Register(&righttopic, handler)
//
// // in its simplest form a message is just a byteslice
// payload := []byte("foobar")
//
// // send a raw message
// err = ps.SendRaw(toaddr.Over(), righttopic, payload)
// log.Error("Fails. Not connect, so nothing in kademlia. But it illustrates the point.", "err", err)
//
// // forward a full message
// envfwd := pss.NewEnvelope(fwdaddr.Over(), righttopic, payload)
// msgfwd := &pss.PssMsg{
// To: toaddr.Over(),
// Payload: envfwd,
// }
// err = ps.Forward(msgfwd)
// log.Error("Also fails, same reason. I wish, I wish, I wish there was somebody out there.", "err", err)
//
// // process an incoming message
// // (this is the first step after the devp2p PssMsg message handler)
// envme := pss.NewEnvelope(toaddr.Over(), righttopic, payload)
// msgme := &pss.PssMsg{
// To: meaddr.Over(),
// Payload: envme,
// }
// err = ps.Process(msgme)
// if err == nil {
// log.Info("this works :)")
// }
//
// // if we don't have a registered topic it fails
// dereg() // remove the previously registered topic-handler link
// ps.Process(msgme)
// log.Error("It fails as we expected", "err", err)
//
// // does nothing but please include it
// ps.Stop()
// }
//
// MESSAGE STRUCTURE
//
// NOTE! This part is subject to change. In particular the envelope structure will be re-implemented using whisper.
//
// A pss message has the following layers:
//
// PssMsg
// Contains (eventually only part of) recipient address, and (eventually) encrypted Envelope.
//
// Envelope
// Currently rlp-encoded. Contains the Payload, along with sender address, topic and expiry information.
//
// Payload
// Byte-slice of arbitrary data
//
// ProtocolMsg
// An optional convenience structure for implementation of devp2p protocols. Contains Code, Size and Payload analogous to the p2p.Msg structure, where the payload is a rlp-encoded byteslice. For transport, this struct is serialized and used as the "payload" above.
//
// TOPICS AND PROTOCOLS
//
// Pure pss is protocol agnostic. Instead it uses the notion of Topic. This is NOT the "subject" of a message. Instead this type is used to internally register handlers for messages matching respective Topics.
//
// Topic in this context virtually mean anything; protocols, chatrooms, or social media groups.
//
// When implementing devp2p protocols, topics are direct mappings to protocols name and version. The pss package provides the PssProtocol convenience structure, and a generic Handler that can be passed to Pss.Register. This makes it possible to use the same message handler code for pss that are used for direct connected peers.
//
// CONNECTIONS
//
// A "connection" in pss is a purely virtual construct. There is no mechanisms in place to ensure that the remote peer actually is there. In fact, "adding" a peer involves merely the node's opinion that the peer is there. It may issue messages to that remote peer to a directly connected peer, which in turn passes it on. But if it is not present on the network - or if there is no route to it - the message will never reach its destination through mere forwarding.
//
// When implementing the devp2p protocol stack, the "adding" of a remote peer is a prerequisite for the side actually initiating the protocol communication. Adding a peer in effect "runs" the protocol on that peer, and adds an internal mapping between a topic and that peer. It also enables sending and receiving messages using the main io-construct in devp2p - the p2p.MsgReadWriter.
//
// Under the hood, pss implements its own MsgReadWriter, which bridges MsgReadWriter.WriteMsg with Pss.SendRaw, and deftly adds an InjectMsg method which pipes incoming messages to appear on the MsgReadWriter.ReadMsg channel.
//
// An incoming connection is nothing more than an actual PssMsg appearing with a certain Topic. If a Handler har been registered to that Topic, the message will be passed to it. This constitutes a "new" connection if:
//
// - The pss node never called AddPeer with this combination of remote peer address and topic, and
//
// - The pss node never received a PssMsg from this remote peer with this specific Topic before.
//
// If it is a "new" connection, the protocol will be "run" on the remote peer, in the same manner as if it was pre-emptively added.
//
// ROUTING AND CACHING
//
// (please refer to swarm kademlia routing for an explanation of the routing algorithm used for pss)
//
// pss implements a simple caching mechanism, using the swarm DPA for storage of the messages and generation of the digest keys used in the cache table. The caching is intended to alleviate the following:
//
// - save messages so that they can be delivered later if the recipient was not online at the time of sending.
//
// - drop an identical message to the same recipient if received within a given time interval
//
// - prevent backwards routing of messages
//
// the latter may occur if only one entry is in the receiving node's kademlia. In this case the forwarder will be provided as the "nearest node" to the final recipient. The cache keeps the address of who the message was forwarded from, and if the cache lookup matches, the message will be dropped.
package pss

1
swarm/pss/common.go → swarm/pss/ping.go
View file

@ -27,6 +27,7 @@ func (self *Ping) PingHandler(msg interface{}) error {
return nil
}
// Sample protocol used for tests
var PingProtocol = &protocols.Spec{
Name: "psstest",
Version: 1,

134
swarm/pss/pss.go
View file

@ -20,22 +20,24 @@ import (
)
const (
TopicResolverLength = 8
PssPeerCapacity = 256
PssPeerTopicDefaultCapacity = 8
digestLength = 32
digestCapacity = 256
PssPeerCapacity = 256 // limit of peers kept in cache. (not implemented)
PssPeerTopicDefaultCapacity = 8 // limit of topics kept per peer. (not implemented)
digestLength = 32 // byte length of digest used for pss cache (currently same as swarm chunk hash)
digestCapacity = 256 // cache entry limit (not implement)
)
var (
errorForwardToSelf = errors.New("forward to self")
)
// abstraction to enable access to p2p.protocols.Peer.Send
type senderPeer interface {
ID() discover.NodeID
Address() []byte
Send(interface{}) error
}
// protocol specification of the pss capsule
var pssSpec = &protocols.Spec{
Name: "pss",
Version: 1,
@ -52,24 +54,13 @@ type pssCacheEntry struct {
type pssDigest [digestLength]byte
// implements node.Service
// Toplevel pss object, taking care of message sending and receiving, message handler dispatchers and message forwarding.
//
// pss provides sending messages to nodes without having to be directly connected to them.
//
// The messages are wrapped in a PssMsg structure and routed using the swarm kademlia routing.
//
// The top-level Pss object provides:
//
// - access to the swarm overlay and routing (kademlia)
// - a collection of remote overlay addresses mapped to MsgReadWriters, representing the virtually connected peers
// - a collection of remote underlay address, mapped to the overlay addresses above
// - a method to send a message to specific overlayaddr
// - a dispatcher lookup, mapping protocols to topics
// - a message cache to spot messages that previously have been forwarded
// Implements node.Service
type Pss struct {
network.Overlay // we can get the overlayaddress from this
peerPool map[pot.Address]map[Topic]p2p.MsgReadWriter // keep track of all virtual p2p.Peers we are currently speaking to
fwdPool map[pot.Address]*protocols.Peer // keep track of all peers sitting on the pssmsg routing layer
fwdPool map[discover.NodeID]*protocols.Peer // keep track of all peers sitting on the pssmsg routing layer
handlers map[Topic]map[*Handler]bool // topic and version based pss payload handlers
fwdcache map[pssDigest]pssCacheEntry // checksum of unique fields from pssmsg mapped to expiry, cache to determine whether to drop msg
cachettl time.Duration // how long to keep messages in fwdcache
@ -81,7 +72,7 @@ type Pss struct {
func (self *Pss) storeMsg(msg *PssMsg) (pssDigest, error) {
swg := &sync.WaitGroup{}
wwg := &sync.WaitGroup{}
buf := bytes.NewReader(msg.Serialize())
buf := bytes.NewReader(msg.serialize())
key, err := self.dpa.Store(buf, int64(buf.Len()), swg, wwg)
if err != nil {
log.Warn("Could not store in swarm", "err", err)
@ -93,12 +84,14 @@ func (self *Pss) storeMsg(msg *PssMsg) (pssDigest, error) {
return digest, nil
}
// Creates a new Pss instance. A node should only need one of these
// Creates a new Pss instance.
//
// Needs a swarm network overlay, a DPA storage for message cache storage.
func NewPss(k network.Overlay, dpa *storage.DPA, params *PssParams) *Pss {
return &Pss{
Overlay: k,
peerPool: make(map[pot.Address]map[Topic]p2p.MsgReadWriter, PssPeerCapacity),
fwdPool: make(map[pot.Address]*protocols.Peer),
fwdPool: make(map[discover.NodeID]*protocols.Peer),
handlers: make(map[Topic]map[*Handler]bool),
fwdcache: make(map[pssDigest]pssCacheEntry),
cachettl: params.Cachettl,
@ -107,18 +100,24 @@ func NewPss(k network.Overlay, dpa *storage.DPA, params *PssParams) *Pss {
}
}
// Convenience accessor to the swarm overlay address of the pss node
func (self *Pss) BaseAddr() []byte {
return self.Overlay.BaseAddr()
}
// For node.Service implementation. Does nothing for now, but should be included in the code for backwards compatibility.
func (self *Pss) Start(srv *p2p.Server) error {
return nil
}
// For node.Service implementation. Does nothing for now, but should be included in the code for backwards compatibility.
func (self *Pss) Stop() error {
return nil
}
// devp2p protocol object for the PssMsg struct.
//
// This represents the PssMsg capsule, and is the entry point for processing, receiving and sending pss messages between directly connected peers.
func (self *Pss) Protocols() []p2p.Protocol {
return []p2p.Protocol{
p2p.Protocol{
@ -130,14 +129,21 @@ func (self *Pss) Protocols() []p2p.Protocol {
}
}
// Starts the PssMsg protocol
func (self *Pss) Run(p *p2p.Peer, rw p2p.MsgReadWriter) error {
self.lock.Lock()
defer self.lock.Unlock()
pp := protocols.NewPeer(p, rw, pssSpec)
id := p.ID()
h := pot.NewHashAddressFromBytes(network.ToOverlayAddr(id[:]))
self.fwdPool[h.Address] = pp
//addr := network.NewAddrFromNodeID(id)
//potaddr := pot.NewHashAddressFromBytes(addr.OAddr)
self.fwdPool[p.ID()] = pp
return pp.Run(self.handlePssMsg)
}
// Exposes the API methods
//
// If the debug-parameter was given to the top Pss object, the TestAPI methods will also be included
func (self *Pss) APIs() []rpc.API {
apis := []rpc.API{
rpc.API{
@ -158,12 +164,11 @@ func (self *Pss) APIs() []rpc.API {
return apis
}
// Takes the generated Topic of a protocol/chatroom etc, and links a handler function to it
// This allows the implementer to retrieve the right handler functions (invoke the right protocol)
// for an incoming message by inspecting the topic on it.
// a topic allows for multiple handlers
// returns a deregister function which needs to be called to deregister the handler
// (similar to event.Subscription.Unsubscribe())
// Links a handler function to a Topic
//
// After calling this, all incoming messages with an envelope Topic matching the Topic specified will be passed to the given Handler function.
//
// Returns a deregister function which needs to be called to deregister the handler,
func (self *Pss) Register(topic *Topic, handler Handler) func() {
self.lock.Lock()
defer self.lock.Unlock()
@ -187,10 +192,7 @@ func (self *Pss) deregister(topic *Topic, h *Handler) {
delete(handlers, h)
}
// enables to set address of node, to avoid backwards forwarding
//
// currently not in use as forwarder address is not known in the handler function hooked to the pss dispatcher.
// it is included as a courtesy to custom transport layers that may want to implement this
// Adds an address/message pair to the cache
func (self *Pss) AddToCache(addr []byte, msg *PssMsg) error {
digest, err := self.storeMsg(msg)
if err != nil {
@ -258,13 +260,13 @@ func (self *Pss) handlePssMsg(msg interface{}) error {
return self.Process(pssmsg)
}
// processes a message with self as recipient
// Entry point to processing a message for which the current node is the intended recipient.
func (self *Pss) Process(pssmsg *PssMsg) error {
env := pssmsg.Payload
payload := env.Payload
handlers := self.getHandlers(env.Topic)
if len(handlers) == 0 {
return fmt.Errorf("No registered handler for topic '%s'", env.Topic)
return fmt.Errorf("No registered handler for topic '%x'", env.Topic)
}
nid, _ := discover.HexID("0x00")
p := p2p.NewPeer(nid, fmt.Sprintf("%x", env.From), []p2p.Cap{})
@ -277,9 +279,11 @@ func (self *Pss) Process(pssmsg *PssMsg) error {
return nil
}
// Sends a message using The message could be anything at all, and will be handled by whichever handler function is mapped to Topic using *Pss.Register()
// Sends a message using Pss.
//
// The to address is a swarm overlay address
// This method is payload agnostic, and will accept any arbitrary byte slice as the payload for a message.
//
// It generates an envelope for the specified recipient and topic, and wraps the message payload in it.
func (self *Pss) SendRaw(to []byte, topic Topic, msg []byte) error {
sender := self.Overlay.BaseAddr()
pssenv := NewEnvelope(sender, topic, msg)
@ -292,18 +296,20 @@ func (self *Pss) SendRaw(to []byte, topic Topic, msg []byte) error {
// Forwards a pss message to the peer(s) closest to the to address
//
// Handlers that want to pass on a message should call this directly
// Handlers that are merely passing on the PssMsg to its final recipient should call this directly
func (self *Pss) Forward(msg *PssMsg) error {
if self.isSelfRecipient(msg) {
return errorForwardToSelf
}
// cache it
digest, err := self.storeMsg(msg)
if err != nil {
log.Warn(fmt.Sprintf("could not store message %v to cache: %v", msg, err))
}
// flood guard
if self.checkFwdCache(nil, digest) {
log.Trace(fmt.Sprintf("pss relay block-cache match: FROM %x TO %x", common.ByteLabel(self.Overlay.BaseAddr()), common.ByteLabel(msg.To)))
return nil
@ -315,11 +321,14 @@ func (self *Pss) Forward(msg *PssMsg) error {
// send with kademlia
// find the closest peer to the recipient and attempt to send
self.Overlay.EachConn(msg.To, 256, func(op network.OverlayConn, po int, isproxbin bool) bool {
//p, ok := op.(senderPeer)
h := pot.NewHashAddressFromBytes(op.Address())
pp := self.fwdPool[h.Address]
addr := self.Overlay.BaseAddr()
sendMsg := fmt.Sprintf("%x: msg to %x via %x", common.ByteLabel(addr), common.ByteLabel(msg.To), common.ByteLabel(op.Address()))
sp, ok := op.(senderPeer)
if !ok {
log.Crit("Pss cannot use kademlia peer type")
return false
}
//sendMsg := fmt.Sprintf("MSG %x TO %x FROM %x VIA %x", digest, common.ByteLabel(msg.To), common.ByteLabel(self.BaseAddr()), common.ByteLabel(op.Address()))
sendMsg := fmt.Sprintf("TO %x FROM %x VIA %x", common.ByteLabel(msg.To), common.ByteLabel(self.BaseAddr()), common.ByteLabel(op.Address()))
pp := self.fwdPool[sp.ID()]
if self.checkFwdCache(op.Address(), digest) {
log.Info(fmt.Sprintf("%v: peer already forwarded to", sendMsg))
return true
@ -329,7 +338,7 @@ func (self *Pss) Forward(msg *PssMsg) error {
log.Warn(fmt.Sprintf("%v: failed forwarding: %v", sendMsg, err))
return true
}
log.Trace(fmt.Sprintf("%v: successfully forwarded", sendMsg))
log.Debug(fmt.Sprintf("%v: successfully forwarded", sendMsg))
sent++
// if equality holds, p is always the first peer given in the iterator
if bytes.Equal(msg.To, op.Address()) || !isproxbin {
@ -341,14 +350,16 @@ func (self *Pss) Forward(msg *PssMsg) error {
if sent == 0 {
log.Error("PSS: unable to forward to any peers")
return nil
return fmt.Errorf("unable to forward to any peers")
}
self.addFwdCacheExpire(digest)
return nil
}
// Links a pss peer address and topic to a dedicated p2p.MsgReadWriter in the pss peerpool, and runs the specificed protocol on this p2p.MsgReadWriter and the specified peer
// For devp2p protocol integration only. Analogous to an outgoing devp2p connection.
//
// Links a remote peer and Topic to a dedicated p2p.MsgReadWriter in the pss peerpool, and runs the specificed protocol using these resources.
//
// The effect is that now we have a "virtual" protocol running on an artificial p2p.Peer, which can be looked up and piped to through Pss using swarm overlay address and topic
func (self *Pss) AddPeer(p *p2p.Peer, addr pot.Address, run func(*p2p.Peer, p2p.MsgReadWriter) error, topic Topic, rw p2p.MsgReadWriter) error {
@ -393,10 +404,9 @@ func (self *Pss) isActive(id pot.Address, topic Topic) bool {
return self.peerPool[id][topic] != nil
}
// Convenience object that:
// For devp2p protocol integration only.
//
// - allows passing of the unwrapped PssMsg payload to the p2p level message handlers
// - interprets outgoing p2p.Msg from the p2p level to pass in to *Pss.Send()
// Bridges pss send/receive with devp2p protocol send/receive
//
// Implements p2p.MsgReadWriter
type PssReadWriter struct {
@ -417,7 +427,7 @@ func (prw PssReadWriter) ReadMsg() (p2p.Msg, error) {
// Implements p2p.MsgWriter
func (prw PssReadWriter) WriteMsg(msg p2p.Msg) error {
log.Warn("got writemsg pssclient", "msg", msg)
log.Trace("pssrw writemsg", "msg", msg)
rlpdata := make([]byte, msg.Size)
msg.Payload.Read(rlpdata)
pmsg, err := rlp.EncodeToBytes(ProtocolMsg{
@ -438,6 +448,8 @@ func (prw PssReadWriter) injectMsg(msg p2p.Msg) error {
return nil
}
// For devp2p protocol integration only.
//
// Convenience object for passing messages in and out of the p2p layer
type PssProtocol struct {
*Pss
@ -446,20 +458,26 @@ type PssProtocol struct {
spec *protocols.Spec
}
// Constructor
func RegisterPssProtocol(ps *Pss, topic *Topic, spec *protocols.Spec, targetprotocol *p2p.Protocol) error {
// For devp2p protocol integration only.
//
// Maps a Topic to a devp2p protocol.
func RegisterPssProtocol(ps *Pss, topic *Topic, spec *protocols.Spec, targetprotocol *p2p.Protocol) *PssProtocol {
pp := &PssProtocol{
Pss: ps,
proto: targetprotocol,
topic: topic,
spec: spec,
}
ps.Register(topic, pp.handle)
return nil
return pp
}
func (self *PssProtocol) handle(msg []byte, p *p2p.Peer, senderAddr []byte) error {
hashoaddr := pot.NewHashAddressFromBytes(senderAddr).Address
// For devp2p protocol integration only.
//
// Generic handler for initiating devp2p-like protocol connections
//
// This handler should be passed to Pss.Register with the associated ropic.
func (self *PssProtocol) Handle(msg []byte, p *p2p.Peer, senderAddr []byte) error {
hashoaddr := pot.NewAddressFromBytes(senderAddr)
if !self.isActive(hashoaddr, *self.topic) {
rw := &PssReadWriter{
Pss: self.Pss,

315
swarm/pss/pss_test.go
View file

@ -4,14 +4,17 @@ import (
"bytes"
"context"
"encoding/hex"
"encoding/json"
"fmt"
"io/ioutil"
"math/rand"
"os"
"sync"
"testing"
"time"
"flag"
"github.com/ethereum/go-ethereum/common"
// "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/p2p"
@ -20,7 +23,6 @@ import (
"github.com/ethereum/go-ethereum/p2p/simulations"
"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
p2ptest "github.com/ethereum/go-ethereum/p2p/testing"
"github.com/ethereum/go-ethereum/pot"
"github.com/ethereum/go-ethereum/swarm/network"
"github.com/ethereum/go-ethereum/swarm/storage"
)
@ -30,14 +32,35 @@ const (
bzzServiceName = "bzz"
)
var (
snapshotfile string
debugflag = flag.Bool("v", false, "verbose")
// custom logging
psslogmain log.Logger
)
var services = newServices()
func init() {
flag.Parse()
rand.Seed(time.Now().Unix())
adapters.RegisterServices(services)
loglevel := log.LvlDebug
if *debugflag {
loglevel = log.LvlTrace
}
psslogmain = log.New("psslog", "*")
hs := log.StreamHandler(os.Stderr, log.TerminalFormat(true))
hf := log.LvlFilterHandler(log.LvlTrace, hs)
hf := log.LvlFilterHandler(loglevel, hs)
h := log.CallerFileHandler(hf)
log.Root().SetHandler(h)
}
func TestCache(t *testing.T) {
@ -187,7 +210,7 @@ func TestSimpleLinear(t *testing.T) {
C: make(chan struct{}),
}
err = RegisterPssProtocol(ps, &PingTopic, PingProtocol, NewPingProtocol(ping.PingHandler))
ps.Register(&PingTopic, RegisterPssProtocol(ps, &PingTopic, PingProtocol, NewPingProtocol(ping.PingHandler)).Handle)
if err != nil {
t.Fatalf("Failed to register virtual protocol in pss: %v", err)
@ -199,8 +222,9 @@ func TestSimpleLinear(t *testing.T) {
Peer: pp,
addr: network.ToOverlayAddr(id[:]),
}
h := pot.NewHashAddressFromBytes(bp.addr)
ps.fwdPool[h.Address] = pp
//a := pot.NewAddressFromBytes(bp.addr)
//ps.fwdPool[a] = pp
ps.fwdPool[id] = pp
ps.Overlay.On(bp)
defer ps.Overlay.Off(bp)
log.Debug(fmt.Sprintf("%v", ps.Overlay))
@ -234,132 +258,145 @@ func TestSimpleLinear(t *testing.T) {
}
}
func TestFullRandom50n(t *testing.T) {
adapter := adapters.NewSimAdapter(services)
testFullRandom(t, adapter, 50, 50, 50)
func TestSnapshot_50_5(t *testing.T) {
testSnapshot(t, "testdata/snapshot_50.json", 5, true)
}
func TestFullRandom25n(t *testing.T) {
adapter := adapters.NewSimAdapter(services)
testFullRandom(t, adapter, 25, 25, 25)
func TestSnapshot_5_50(t *testing.T) {
testSnapshot(t, "testdata/snapshot_5.json", 50, true)
}
func TestFullRandom10n(t *testing.T) {
adapter := adapters.NewSimAdapter(services)
testFullRandom(t, adapter, 10, 10, 10)
func TestSnapshot_5_5(t *testing.T) {
testSnapshot(t, "testdata/snapshot_5.json", 5, true)
}
func TestFullRandom5n(t *testing.T) {
adapter := adapters.NewSimAdapter(services)
testFullRandom(t, adapter, 5, 5, 5)
}
func testSnapshot(t *testing.T, snapshotfile string, msgcount int, sim bool) {
func testFullRandom(t *testing.T, adapter adapters.NodeAdapter, nodecount int, fullnodecount int, msgcount int) {
var i int
var msgfromids []discover.NodeID
var msgreceived []discover.NodeID
var cancelmain func()
var triggerptr *chan discover.NodeID
msgtoids := make([]discover.NodeID, msgcount)
// choose the adapter to use
var adapter adapters.NodeAdapter
if sim {
adapter = adapters.NewSimAdapter(services)
} else {
baseDir, err := ioutil.TempDir("", "swarm-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(baseDir)
adapter = adapters.NewExecAdapter(baseDir)
}
wg := sync.WaitGroup{}
wg.Add(msgcount)
psslog := make(map[discover.NodeID]log.Logger)
psslogmain := log.New("psslog", "*")
// process shapshot
jsonsnapshot, err := ioutil.ReadFile(snapshotfile)
if err != nil {
t.Fatalf("cant read snapshot: %s", snapshotfile)
}
snapshot := &simulations.Snapshot{}
err = json.Unmarshal(jsonsnapshot, snapshot)
if err != nil {
t.Fatalf("snapshot file unreadable: %v", err)
}
for _, node := range snapshot.Nodes {
node.Config.Services = []string{"bzz", "pss"}
}
// setup network with snapshot
net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{
ID: "0",
})
defer net.Shutdown()
err = net.Load(snapshot)
if err != nil {
t.Fatalf("invalid snapshot: %v", err)
}
timeout := 15 * time.Second
ctx, cancelmain := context.WithTimeout(context.Background(), timeout)
defer cancelmain()
// nodes expecting messages
recvids := make([]discover.NodeID, msgcount)
// the overlay address map to recvids
recvaddrs := make(map[discover.NodeID][]byte)
// messages actually received (registered through trigger and test check)
var msgreceived []discover.NodeID
// trigger for expect in test
trigger := make(chan discover.NodeID)
triggerptr = &trigger
ids := make([]discover.NodeID, nodecount)
fullids := ids[0:fullnodecount]
fullpeers := make(map[discover.NodeID][]byte)
// one wait for every message
wg := sync.WaitGroup{}
wg.Add(msgcount)
for i = 0; i < nodecount; i++ {
nodeconfig := adapters.RandomNodeConfig()
nodeconfig.Services = []string{"bzz", "pss"}
node, err := net.NewNodeWithConfig(nodeconfig)
if err != nil {
t.Fatalf("error starting node: %s", err)
}
if err := net.Start(node.ID()); err != nil {
t.Fatalf("error starting node %s: %s", node.ID().TerminalString(), err)
}
if err := triggerChecks(ctx, &wg, triggerptr, net, node.ID()); err != nil {
t.Fatal("error triggering checks for node %s: %s", node.ID().TerminalString(), err)
}
ids[i] = node.ID()
if i < fullnodecount {
fullpeers[ids[i]] = network.ToOverlayAddr(node.ID().Bytes())
psslog[ids[i]] = log.New("psslog", fmt.Sprintf("%x", fullpeers[ids[i]]))
}
log.Debug("psslog starting node", "id", nodeconfig.ID)
}
for i, id := range fullids {
msgfromids = append(msgfromids, id)
msgtoids[i] = fullids[(i+(len(fullids)/2)+1)%len(fullids)]
}
// run a simulation which connects the 10 nodes in a ring and waits
// for full peer discovery
action := func(ctx context.Context) error {
for i, id := range ids {
peerID := ids[(i+1)%len(ids)]
if net.GetConn(id, peerID) != nil {
continue
var rpcerr error
var rpcbyte []byte
for _, simnode := range net.Nodes {
if simnode == nil {
return fmt.Errorf("unknown node: %s", simnode.ID())
}
if err := net.Connect(id, peerID); err != nil {
return err
client, err := simnode.Client()
if err != nil {
return fmt.Errorf("error getting recp node client: %s", err)
}
err = client.Call(&rpcbyte, "pss_baseAddr")
if err != nil {
t.Fatalf("cant get overlayaddr: %v", err)
}
recvaddrs[simnode.ID()] = rpcbyte
err = client.Call(&rpcbyte, "pss_baseAddr")
if err != nil {
t.Fatalf("cant get overlayaddr: %v", err)
}
err = triggerChecks(ctx, &wg, &trigger, net, simnode.ID())
if err != nil {
t.Fatalf("trigger setup failed: %v", err)
}
}
for i := 0; i < msgcount; i++ {
idx := rand.Intn(len(net.Nodes))
sendernode := net.Nodes[idx]
toidx := rand.Intn(len(net.Nodes)-1)
if toidx >= idx {
toidx++
}
recvnode := net.Nodes[toidx]
recvids[i] = recvnode.ID()
msg := PingMsg{Created: time.Now()}
code, _ := PingProtocol.GetCode(&PingMsg{})
pmsg, _ := NewProtocolMsg(code, msg)
client, err := sendernode.Client()
if err != nil {
return fmt.Errorf("error getting sendernode client: %s", err)
}
client.CallContext(ctx, &rpcerr, "pss_send", PingTopic, APIMsg{
Addr: recvaddrs[recvnode.ID()],
Msg: pmsg,
})
if rpcerr != nil {
return fmt.Errorf("error rpc send id %x: %v", sendernode.ID(), rpcerr)
}
psslog[id].Debug("conn ok", "one", id, "other", peerID)
}
return nil
}
check := func(ctx context.Context, id discover.NodeID) (bool, error) {
var tgt []byte
var fwd struct {
Addr []byte
Count int
}
select {
case <-ctx.Done():
wg.Done()
psslog[id].Error("conn failed!", "id", id)
return false, ctx.Err()
default:
case <-ctx.Done():
wg.Done()
return false, ctx.Err()
default:
}
for i, fid := range msgfromids {
if id == fid {
tgt = network.ToOverlayAddr(msgtoids[(i+(len(msgtoids)/2)+1)%len(msgtoids)].Bytes())
break
}
}
p := net.GetNode(id)
if p == nil {
return false, fmt.Errorf("Unknown node: %v", id)
}
c, err := p.Client()
if err != nil {
return false, err
}
for fwd.Count < 2 {
c.CallContext(context.Background(), &fwd, "pss_getForwarder", tgt)
time.Sleep(time.Microsecond * 250)
}
psslog[id].Debug("fwd check ok", "topaddr", fmt.Sprintf("%x", common.ByteLabel(fwd.Addr)), "kadcount", fwd.Count)
msgreceived = append(msgreceived, id)
psslogmain.Info("trigger received", "id", id, "len", len(msgreceived))
wg.Done()
return true, nil
}
@ -367,61 +404,7 @@ func testFullRandom(t *testing.T, adapter adapters.NodeAdapter, nodecount int, f
Action: action,
Trigger: trigger,
Expect: &simulations.Expectation{
Nodes: ids,
Check: check,
},
})
if result.Error != nil {
t.Fatalf("simulation failed: %s", result.Error)
cancelmain()
}
trigger = make(chan discover.NodeID)
triggerptr = &trigger
action = func(ctx context.Context) error {
var rpcerr error
for ii, id := range msgfromids {
node := net.GetNode(id)
if node == nil {
return fmt.Errorf("unknown node: %s", id)
}
client, err := node.Client()
if err != nil {
return fmt.Errorf("error getting node client: %s", err)
}
msg := PingMsg{Created: time.Now()}
code, _ := PingProtocol.GetCode(&PingMsg{})
pmsg, _ := NewProtocolMsg(code, msg)
client.CallContext(ctx, &rpcerr, "pss_send", PingTopic, APIMsg{
Addr: fullpeers[msgtoids[ii]],
Msg: pmsg,
})
if rpcerr != nil {
return fmt.Errorf("error rpc send id %x: %v", id, rpcerr)
}
}
return nil
}
check = func(ctx context.Context, id discover.NodeID) (bool, error) {
select {
case <-ctx.Done():
wg.Done()
return false, ctx.Err()
default:
}
msgreceived = append(msgreceived, id)
psslog[id].Info("trigger received", "id", id, "len", len(msgreceived))
wg.Done()
return true, nil
}
result = simulations.NewSimulation(net).Run(ctx, &simulations.Step{
Action: action,
Trigger: trigger,
Expect: &simulations.Expectation{
Nodes: msgtoids,
Nodes: recvids,
Check: check,
},
})
@ -431,13 +414,14 @@ func testFullRandom(t *testing.T, adapter adapters.NodeAdapter, nodecount int, f
t.Fatalf("simulation failed: %s", result.Error)
}
if len(msgreceived) != len(msgtoids) {
t.Fatalf("Simulation Failed, got %d of %d msgs", len(msgreceived), len(msgtoids))
wg.Wait()
if len(msgreceived) != msgcount {
t.Fatalf("Simulation Failed, got %d of %d msgs", len(msgreceived), msgcount)
}
wg.Wait()
psslogmain.Info("done!")
t.Logf("Simulation Passed, got %d of %d msgs", len(msgreceived), len(msgtoids))
t.Logf("Simulation Passed, got %d of %d msgs", len(msgreceived), msgcount)
//t.Logf("Duration: %s", result.FinishedAt.Sub(result.StartedAt))
}
@ -447,7 +431,6 @@ func testFullRandom(t *testing.T, adapter adapters.NodeAdapter, nodecount int, f
func triggerChecks(ctx context.Context, wg *sync.WaitGroup, trigger *chan discover.NodeID, net *simulations.Network, id discover.NodeID) error {
quitC := make(chan struct{})
got := false
node := net.GetNode(id)
if node == nil {
@ -475,12 +458,8 @@ func triggerChecks(ctx context.Context, wg *sync.WaitGroup, trigger *chan discov
defer peersub.Unsubscribe()
for {
select {
case event := <-peerevents:
if event.Type == "add" && !got {
got = true
*trigger <- id
}
case <-msgevents:
psslogmain.Debug("incoming msg", "node", id)
*trigger <- id
case err := <-peersub.Err():
if err != nil {
@ -543,7 +522,7 @@ func newServices() adapters.Services {
ping := &Ping{
C: make(chan struct{}),
}
err = RegisterPssProtocol(ps, &PingTopic, PingProtocol, NewPingProtocol(ping.PingHandler))
ps.Register(&PingTopic, RegisterPssProtocol(ps, &PingTopic, PingProtocol, NewPingProtocol(ping.PingHandler)).Handle)
if err != nil {
log.Error("Couldnt register pss protocol", "err", err)
os.Exit(1)
@ -555,7 +534,7 @@ func newServices() adapters.Services {
"bzz": func(ctx *adapters.ServiceContext) (node.Service, error) {
addr := network.NewAddrFromNodeID(ctx.Config.ID)
hp := network.NewHiveParams()
hp.Discovery = true
hp.Discovery = false
config := &network.BzzConfig{
OverlayAddr: addr.Over(),
UnderlayAddr: addr.Under(),

42
swarm/pss/pssapi.go
View file

@ -11,17 +11,20 @@ import (
"github.com/ethereum/go-ethereum/swarm/network"
)
// API is the RPC API module for Pss
// Pss API services
type API struct {
*Pss
}
// NewAPI constructs a PssAPI instance
func NewAPI(ps *Pss) *API {
return &API{Pss: ps}
}
// NewMsg API endpoint creates an RPC subscription
// Creates a new subscription for the caller. Enables external handling of incoming messages.
//
// A new handler is registered in pss for the supplied topic
//
// All incoming messages to the node matching this topic will be encapsulated in the APIMsg struct and sent to the subscriber
func (pssapi *API) Receive(ctx context.Context, topic Topic) (*rpc.Subscription, error) {
notifier, supported := rpc.NotifierFromContext(ctx)
if !supported {
@ -54,31 +57,47 @@ func (pssapi *API) Receive(ctx context.Context, topic Topic) (*rpc.Subscription,
return psssub, nil
}
// SendRaw sends the message (serialized into byte slice) to a peer with topic
// Sends the message wrapped in APIMsg through pss
//
// Wrapper method for the pss.SendRaw function.
//
// The method will pass on the error received from pss.
//
// Note that normally pss will report an error if an attempt is made to send a pss to oneself. However, if the debug flag has been set, and the address specified in APIMsg is the node's own, this method implements a short-circuit which injects the message as an incoming message (using Pss.Process). This can be useful for testing purposes, when only operating with one node.
func (pssapi *API) Send(topic Topic, msg APIMsg) error {
if pssapi.debug && bytes.Equal(msg.Addr, pssapi.BaseAddr()) {
if pssapi.debug && bytes.Equal(msg.Addr, pssapi.Pss.BaseAddr()) {
log.Warn("Pss debug enabled; send to self shortcircuit", "apimsg", msg, "topic", topic)
env := NewEnvelope(msg.Addr, topic, msg.Msg)
return pssapi.Process(&PssMsg{
To: pssapi.BaseAddr(),
To: pssapi.Pss.BaseAddr(),
Payload: env,
})
}
return pssapi.SendRaw(msg.Addr, topic, msg.Msg)
}
// BaseAddr returns the pss node's swarm overlay address
//
// Note that the overlay address is NOT inferable. To really know the node's overlay address it must reveal it itself.
func (pssapi *API) BaseAddr() ([]byte, error) {
return pssapi.Pss.BaseAddr(), nil
}
// PssAPITest are temporary API calls for development use only
// These symbols should not be included in production environment
//
// These symbols should NOT be included in production environment
type APITest struct {
*Pss
}
// NewAPI constructs a API instance
// Include these methods to the node.Service if test symbols should be used
func NewAPITest(ps *Pss) *APITest {
return &APITest{Pss: ps}
}
// temporary for access to overlay while faking kademlia healthy routines
// Get the current nearest swarm node to the specified address
//
// (Can be used for diagnosing kademlia state)
func (pssapitest *APITest) GetForwarder(addr []byte) (fwd struct {
Addr []byte
Count int
@ -92,8 +111,3 @@ func (pssapitest *APITest) GetForwarder(addr []byte) (fwd struct {
})
return
}
// BaseAddr gets our own overlayaddress
func (pssapitest *APITest) BaseAddr() ([]byte, error) {
return pssapitest.Pss.BaseAddr(), nil
}

6
swarm/pss/simulations/service_test.go
View file

@ -139,9 +139,13 @@ func (t *testWrapper) newTestService(ctx *adapters.ServiceContext) (node.Service
if err != nil {
panic(err)
}
pssclient, err := client.NewClientWithRPC(rpcClient)
if err != nil {
return nil, err
}
return &testService{
id: ctx.Config.ID,
pss: client.NewClientWithRPC(context.Background(), rpcClient),
pss: pssclient,
handshakes: make(chan *testHandshake),
}, nil
}

1
swarm/pss/testdata/snapshot_10.json vendored Executable file
View file

File diff suppressed because one or more lines are too long

1
swarm/pss/testdata/snapshot_5.json vendored Executable file
View file

@ -0,0 +1 @@
{"nodes":[{"config":{"id":"78c9669ee7b6b66642c69547501bddc4ffb5b3887c2b889b30b761fe245fee05db56bc892c6e6502a4d036c9c86ce7f9f0975d7b8b1ffebfdb8109335f364b4a","private_key":"d264be17451dbbeec8cffa553fdc9717dcc76b4ad895db7bbc672016470b6e3a","name":"node01","services":["discovery"]},"up":true},{"config":{"id":"0eec1942b5c7bea584e03d9d0f8b3219161daa68635457fd33e703b1b1512b1dd2ae27bd575a72b4748f6c85b00006dd5a08be0b8eacd6cbd8d90124b3874c7a","private_key":"b5057fc754be217f79b594072c0ef2d35c306435c8b1de75358852e92ab33ca7","name":"node02","services":["discovery"]},"up":true},{"config":{"id":"8d84d276b03161f58af3911b10da2dc33cfc93391130ed89db0f7a84bd6323b0237dab2f3c8b5c7d9953cdc745125dd67ee1122cb73a583caa98359f01afb415","private_key":"a00eea5fae5de6ca8e8d7b622ab1ede0f8adeb6fbb55a00dcd9783b760812cdd","name":"node03","services":["discovery"]},"up":true},{"config":{"id":"fdeaa376c517fe1b4574eb859d92df4272d344409ae9ec42f1c3f21e386c88b8dddd5d2febb9851e450c7e2a72113d419a6e52bf63e6bfbdde00259062ed3098","private_key":"3de00ba717baa58589e0562051f4ce6a2650c0bdab330d9eb55786fecbab2ec7","name":"node04","services":["discovery"]},"up":true},{"config":{"id":"a26035f22d085ad5c8f3892f16ed4867b6a09952c589d0177af8372666e44a454a3e49b43456c25a8365015f450432f4a6b328810e740dd2eadb898857b83e7b","private_key":"1aa2ca63a68c03f8dbc1a5a50cd7994af15fe860e1d2c7e97c0b99cd9f295532","name":"node05","services":["discovery"]},"up":true}],"conns":[{"one":"78c9669ee7b6b66642c69547501bddc4ffb5b3887c2b889b30b761fe245fee05db56bc892c6e6502a4d036c9c86ce7f9f0975d7b8b1ffebfdb8109335f364b4a","other":"a26035f22d085ad5c8f3892f16ed4867b6a09952c589d0177af8372666e44a454a3e49b43456c25a8365015f450432f4a6b328810e740dd2eadb898857b83e7b","up":true,"reverse":true,"distance":65},{"one":"0eec1942b5c7bea584e03d9d0f8b3219161daa68635457fd33e703b1b1512b1dd2ae27bd575a72b4748f6c85b00006dd5a08be0b8eacd6cbd8d90124b3874c7a","other":"78c9669ee7b6b66642c69547501bddc4ffb5b3887c2b889b30b761fe245fee05db56bc892c6e6502a4d036c9c86ce7f9f0975d7b8b1ffebfdb8109335f364b4a","up":true,"reverse":false,"distance":69},{"one":"8d84d276b03161f58af3911b10da2dc33cfc93391130ed89db0f7a84bd6323b0237dab2f3c8b5c7d9953cdc745125dd67ee1122cb73a583caa98359f01afb415","other":"0eec1942b5c7bea584e03d9d0f8b3219161daa68635457fd33e703b1b1512b1dd2ae27bd575a72b4748f6c85b00006dd5a08be0b8eacd6cbd8d90124b3874c7a","up":true,"reverse":true,"distance":68},{"one":"fdeaa376c517fe1b4574eb859d92df4272d344409ae9ec42f1c3f21e386c88b8dddd5d2febb9851e450c7e2a72113d419a6e52bf63e6bfbdde00259062ed3098","other":"8d84d276b03161f58af3911b10da2dc33cfc93391130ed89db0f7a84bd6323b0237dab2f3c8b5c7d9953cdc745125dd67ee1122cb73a583caa98359f01afb415","up":true,"reverse":false,"distance":65},{"one":"a26035f22d085ad5c8f3892f16ed4867b6a09952c589d0177af8372666e44a454a3e49b43456c25a8365015f450432f4a6b328810e740dd2eadb898857b83e7b","other":"fdeaa376c517fe1b4574eb859d92df4272d344409ae9ec42f1c3f21e386c88b8dddd5d2febb9851e450c7e2a72113d419a6e52bf63e6bfbdde00259062ed3098","up":true,"reverse":true,"distance":69}]}

1
swarm/pss/testdata/snapshot_50.json vendored Executable file
View file

File diff suppressed because one or more lines are too long

68
swarm/pss/types.go
View file

@ -18,13 +18,13 @@ const (
defaultDigestCacheTTL = time.Second
)
// Defines params for Pss
// Pss configuration parameters
type PssParams struct {
Cachettl time.Duration
Debug bool
}
// Initializes default params for Pss
// Sane defaults for Pss
func NewPssParams(debug bool) *PssParams {
return &PssParams{
Cachettl: defaultDigestCacheTTL,
@ -32,13 +32,14 @@ func NewPssParams(debug bool) *PssParams {
}
}
// Encapsulates the message transported over pss.
// Encapsulates messages transported over pss.
type PssMsg struct {
To []byte
Payload *Envelope
}
func (msg *PssMsg) Serialize() []byte {
// serializes the message for use in cache
func (msg *PssMsg) serialize() []byte {
rlpdata, _ := rlp.EncodeToBytes(msg)
return rlpdata
}
@ -48,16 +49,7 @@ func (self *PssMsg) String() string {
return fmt.Sprintf("PssMsg: Recipient: %x", common.ByteLabel(self.To))
}
// Topic defines the context of a message being transported over pss
// It is used by pss to determine what action is to be taken on an incoming message
// Typically, one can map protocol handlers for the message payloads by mapping topic to them; see *Pss.Register()
type Topic [TopicLength]byte
func (self *Topic) String() string {
return fmt.Sprintf("%x", self)
}
// Pre-Whisper placeholder, payload of PssMsg
// Pre-Whisper placeholder, payload of PssMsg, sender address, Topic
type Envelope struct {
Topic Topic
TTL uint16
@ -65,7 +57,7 @@ type Envelope struct {
From []byte
}
// creates Pss envelope from sender address, topic and raw payload
// Creates A Pss envelope from sender address, topic and raw payload
func NewEnvelope(addr []byte, topic Topic, payload []byte) *Envelope {
return &Envelope{
From: addr,
@ -75,7 +67,7 @@ func NewEnvelope(addr []byte, topic Topic, payload []byte) *Envelope {
}
}
// encapsulates a protocol msg as PssEnvelope data
// Convenience wrapper for devp2p protocol messages for transport over pss
type ProtocolMsg struct {
Code uint64
Size uint32
@ -83,13 +75,7 @@ type ProtocolMsg struct {
ReceivedAt time.Time
}
// PssAPIMsg is the type for messages, it extends the rlp encoded protocol Msg
// with the Sender's overlay address
type APIMsg struct {
Msg []byte
Addr []byte
}
// Creates a ProtocolMsg
func NewProtocolMsg(code uint64, msg interface{}) ([]byte, error) {
rlpdata, err := rlp.EncodeToBytes(msg)
@ -97,8 +83,6 @@ func NewProtocolMsg(code uint64, msg interface{}) ([]byte, error) {
return nil, err
}
// previous attempts corrupted nested structs in the payload iself upon deserializing
// therefore we use two separate []byte fields instead of peerAddr
// TODO verify that nested structs cannot be used in rlp
smsg := &ProtocolMsg{
Code: code,
@ -109,12 +93,35 @@ func NewProtocolMsg(code uint64, msg interface{}) ([]byte, error) {
return rlp.EncodeToBytes(smsg)
}
// Message handler func for a topic
// Convenience wrapper for sending and receiving pss messages when using the pss API
type APIMsg struct {
Msg []byte
Addr []byte
}
// for debugging, show nice hex version
func (self *APIMsg) String() string {
return fmt.Sprintf("APIMsg: from: %s..., msg: %s...", common.ByteLabel(self.Msg), common.ByteLabel(self.Addr))
}
// Signature for a message handler function for a PssMsg
//
// Implementations of this type are passed to Pss.Register together with a topic,
type Handler func(msg []byte, p *p2p.Peer, from []byte) error
// constructs a new PssTopic from a given name and version.
// Topic defines the context of a message being transported over pss
// It is used by pss to determine what action is to be taken on an incoming message
// Typically, one can map protocol handlers for the message payloads by mapping topic to them; see Pss.Register
type Topic [TopicLength]byte
// String representation of Topic
func (self *Topic) String() string {
return fmt.Sprintf("%x", self)
}
// Constructs a new PssTopic from a given name and version.
//
// Analogous to the name and version members of p2p.Protocol
// Analogous to the name and version members of p2p.Protocol.
func NewTopic(s string, v int) (topic Topic) {
h := sha3.NewKeccak256()
h.Write([]byte(s))
@ -125,6 +132,11 @@ func NewTopic(s string, v int) (topic Topic) {
return topic
}
// For devp2p protocol integration only
//
// Creates a serialized (non-buffered) version of a p2p.Msg, used in the specialized p2p.MsgReadwriter implementations used internally by pss
//
// Should not normally be called outside the pss package hierarchy
func ToP2pMsg(msg []byte) (p2p.Msg, error) {
payload := &ProtocolMsg{}
if err := rlp.DecodeBytes(msg, payload); err != nil {

64
swarm/swarm.go
View file

@ -21,6 +21,7 @@ import (
"context"
"crypto/ecdsa"
"fmt"
"io/ioutil"
"github.com/ethereum/go-ethereum/accounts/abi/bind"
"github.com/ethereum/go-ethereum/common"
@ -31,6 +32,7 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/discover"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/swarm/api"
httpapi "github.com/ethereum/go-ethereum/swarm/api/http"
@ -42,13 +44,12 @@ import (
// the swarm stack
type Swarm struct {
config *api.Config // swarm configuration
api *api.Api // high level api layer (fs/manifest)
dns api.Resolver // DNS registrar
storage storage.ChunkStore // internal access to storage, common interface to cloud storage backends
dpa *storage.DPA // distributed preimage archive, the local API to the storage with document level storage/retrieval support
cloud storage.CloudStore // procurement, cloud storage backend (can multi-cloud)
//hive *network.Hive // the logistic manager
config *api.Config // swarm configuration
api *api.Api // high level api layer (fs/manifest)
dns api.Resolver // DNS registrar
storage storage.ChunkStore // internal access to storage, common interface to cloud storage backends
dpa *storage.DPA // distributed preimage archive, the local API to the storage with document level storage/retrieval support
cloud storage.CloudStore // procurement, cloud storage backend (can multi-cloud)
bzz *network.Bzz // hive and bzz protocol
backend chequebook.Backend // simple blockchain Backend
privateKey *ecdsa.PrivateKey
@ -106,21 +107,17 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api.
kp,
)
config.HiveParams.Discovery = true
nodeid := discover.PubkeyID(crypto.ToECDSAPub(common.FromHex(config.PublicKey)))
addr := network.NewAddrFromNodeID(nodeid)
bzzconfig := &network.BzzConfig{
UnderlayAddr: common.FromHex(config.PublicKey),
OverlayAddr: common.FromHex(config.BzzKey),
UnderlayAddr: addr.UAddr,
HiveParams: config.HiveParams,
}
self.bzz = network.NewBzz(bzzconfig, to, nil)
// set up the kademlia hive
// self.hive = network.NewHive(
// config.HiveParams, // configuration parameters
// self.Kademlia,
// stateStore,
// )
// log.Debug(fmt.Sprintf("Set up swarm network with Kademlia hive"))
//
// setup cloud storage internal access layer
self.storage = storage.NewNetStore(hash, self.lstore, nil, config.StoreParams)
log.Debug("-> swarm net store shared access layer to Swarm Chunk Store")
@ -133,6 +130,16 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api.
self.dpa = storage.NewDPA(dpaChunkStore, self.config.ChunkerParams)
log.Debug(fmt.Sprintf("-> Content Store API"))
// netstore is broken, temp workaround to fiddle with pss
cachedir, err := ioutil.TempDir("", "bzz-tmp")
if err != nil {
return nil, err
}
self.dpa, err = storage.NewLocalDPA(cachedir)
if err != nil {
return nil, err
}
// Pss = postal service over swarm (devp2p over bzz)
if pssEnabled {
pssparams := pss.NewPssParams(false)
@ -175,6 +182,11 @@ Start is called when the stack is started
*/
// implements the node.Service interface
func (self *Swarm) Start(net *p2p.Server) error {
// update uaddr to correct enode
newaddr := self.bzz.UpdateLocalAddr([]byte(net.Self().String()))
log.Warn("Updated bzz local addr", "oaddr", fmt.Sprintf("%x", newaddr.OAddr), "uaddr", fmt.Sprintf("%x", newaddr.UAddr))
// set chequebook
if self.swapEnabled {
ctx := context.Background() // The initial setup has no deadline.
@ -189,16 +201,16 @@ func (self *Swarm) Start(net *p2p.Server) error {
log.Warn(fmt.Sprintf("Starting Swarm service"))
// self.hive.Start(net)
err := self.bzz.Start(net)
if err != nil {
log.Error("bzz failed", "err", err)
return err
}
log.Info(fmt.Sprintf("Swarm network started on bzz address: %v", self.bzz.Hive.Overlay.BaseAddr()))
log.Info(fmt.Sprintf("Swarm network started on bzz address: %x", self.bzz.Hive.Overlay.BaseAddr()))
if self.pss != nil {
self.pss.Start(net)
log.Info("Pss started")
}
self.dpa.Start()
@ -226,7 +238,6 @@ func (self *Swarm) Start(net *p2p.Server) error {
// stops all component services.
func (self *Swarm) Stop() error {
self.dpa.Stop()
//self.hive.Stop()
self.bzz.Stop()
if self.pss != nil {
self.pss.Stop()
@ -251,22 +262,11 @@ func (self *Swarm) Protocols() (protos []p2p.Protocol) {
}
if self.pss != nil {
log.Warn("adding pss protos")
for _, p := range self.pss.Protocols() {
protos = append(protos, p)
}
}
// ct := network.BzzCodeMap()
// ct.Register(2, network.DiscoveryMsgs...)
// proto := network.Bzz(
// self.hive.Overlay.GetAddr().Over(),
// self.hive.Overlay.GetAddr().Under(),
// ct,
// nil,
// nil,
// nil,
// )
//
return
}
@ -354,7 +354,7 @@ func NewLocalSwarm(datadir, port string) (self *Swarm, err error) {
return
}
config, err := api.NewConfig(datadir, common.Address{}, prvKey, network.NetworkId)
config, err := api.NewConfig(datadir, common.Address{}, prvKey, network.NetworkID)
if err != nil {
return
}