From 77b5ff3ff1c9ed2b0018eedc06f685bcb31de339 Mon Sep 17 00:00:00 2001 From: zelig Date: Tue, 6 Jun 2017 00:02:29 +0200 Subject: [PATCH 01/17] temp commit, kademlia changes --- swarm/network/kademlia.go | 83 ++++++++++++++++------------ swarm/network/simulations/overlay.go | 4 +- 2 files changed, 49 insertions(+), 38 deletions(-) diff --git a/swarm/network/kademlia.go b/swarm/network/kademlia.go index 8b2b3f07ea..69e2ef6f8e 100644 --- a/swarm/network/kademlia.go +++ b/swarm/network/kademlia.go @@ -192,68 +192,79 @@ func (self *Kademlia) SuggestPeer() (a OverlayAddr, o int, want bool) { empty := self.FirstEmptyBin() // if there is a callable neighbour within the current proxBin, connect // this makes sure nearest neighbour set is fully connected - log.Debug(fmt.Sprintf("candidate prox peer checking above PO %v", depth)) - // log.Trace(fmt.Sprintf("candidate prox peer checking above PO %v", depth)) + log.Debug(fmt.Sprintf("candidate nearest neighbour checking above PO %v", depth)) + // log.Trace(fmt.Sprintf("candidate nearest neighbour checking above PO %v", depth)) var ppo int ba := pot.NewBytesVal(self.base, nil) self.addrs.EachNeighbour(ba, func(val pot.PotVal, po int) bool { a = self.callable(val) - log.Trace(fmt.Sprintf("candidate prox peer at %x: %v (%v). a == nil is %v", val.(*entry).Address(), a, po, a == nil)) + log.Trace(fmt.Sprintf("candidate nearest neighbour at %x: %v (%v). a == nil is %v", val.(*entry).Address(), a, po, a == nil)) ppo = po return a == nil && po >= depth }) if a != nil { - log.Debug(fmt.Sprintf("candidate prox peer found: %v (%v)", a, ppo)) + log.Debug(fmt.Sprintf("candidate nearest neighbour found: %v (%v)", a, ppo)) return a, 0, false } - log.Debug(fmt.Sprintf("no candidate prox peers to connect to (Depth: %v, minProxSize: %v) %#v", depth, self.MinProxBinSize, a)) + log.Debug(fmt.Sprintf("no candidate nearest neighbours to connect to (Depth: %v, minProxSize: %v) %#v", depth, self.MinProxBinSize, a)) var bpo []int prev := -1 self.conns.EachBin(pot.NewBytesVal(self.base, nil), 0, func(po, size int, f func(func(val pot.PotVal, i int) bool) bool) bool { log.Trace(fmt.Sprintf("check PO%02d: ", po)) prev++ - if po > prev { - size = 0 - po = prev + for ; prev < po; prev++ { + bpo = append(bpo, prev) + minsize = 0 } if size < minsize { - minsize = size bpo = append(bpo, po) + minsize = size } return size > 0 && po < depth }) // all buckets are full // minsize == self.MinBinSize if len(bpo) == 0 { + log.Trace(fmt.Sprintf("all bins saturated")) return nil, 0, false } // as long as we got candidate peers to connect to // dont ask for new peers (want = false) // try to select a candidate peer - for i := len(bpo) - 1; i >= 0; i-- { - // find the first callable peer - self.addrs.EachBin(ba, bpo[i], func(po, size int, f func(func(pot.PotVal, int) bool) bool) bool { - // for each bin we find callable candidate peers - log.Trace(fmt.Sprintf("check PO%02d: ", po)) - f(func(val pot.PotVal, j int) bool { - a = self.callable(val) - if po == empty { - log.Debug(fmt.Sprintf("candidate prox peer found: %v (%v)", a, ppo)) - } - return a == nil && po <= depth - }) - return false - }) - // found a candidate - if a != nil { - break + // for i := len(bpo) - 1; i >= 0; i-- { + // find the first callable peer + i := 0 + nxt := bpo[0] + self.addrs.EachBin(ba, nxt, func(po, size int, f func(func(pot.PotVal, int) bool) bool) bool { + // for each bin we find callable candidate peers + if po == nxt { + if i == len(bpo)-1 { + return false + } + i++ + nxt = bpo[i] } - // cannot find a candidate, ask for more for this proximity bin specifically - o = bpo[i] - want = true + log.Trace(fmt.Sprintf("check PO%02d: ", po)) + f(func(val pot.PotVal, j int) bool { + a = self.callable(val) + if po == empty { + log.Debug(fmt.Sprintf("candidate nearest neighbour found: %v (%v)", a, ppo)) + } + return a == nil && po <= depth + }) + return false + }) + // found a candidate + if a != nil { + return a, 0, false } - return a, o, want + return a, bpo[i], true + // cannot find a candidate, ask for more for this proximity bin specifically + // o = bpo[i] + // want = true + // // } + // return a, o, want } // On inserts the peer as a kademlia peer into the live peers @@ -323,15 +334,12 @@ func (self *Kademlia) EachConn(base []byte, o int, f func(OverlayConn, int, bool base = self.base } p := pot.NewBytesVal(base, nil) + depth := self.Depth() self.conns.EachNeighbour(p, func(val pot.PotVal, po int) bool { if po > o { return true } - isproxbin := false - if l, _ := p.PO(val, 0); l >= self.Depth() { - isproxbin = true - } - return f(val.(*entry).conn(), po, isproxbin) + return f(val.(*entry).conn(), po, po >= depth) }) } @@ -423,6 +431,9 @@ func (self *Kademlia) String() string { row := []string{fmt.Sprintf("%2d", size)} rest -= size f(func(val pot.PotVal, vpo int) bool { + e := val.(*entry) + label := e.String()[:6] + row = append(row, fmt.Sprintf("%s (%d)", label, e.retries)) row = append(row, val.(*entry).String()[:6]) rowlen++ return rowlen < 4 @@ -460,7 +471,7 @@ func (self *Kademlia) String() string { for i := 0; i < self.MaxProxDisplay; i++ { if i == depth { - rows = append(rows, fmt.Sprintf("============ PROX LIMIT: %d ==========================================", i)) + rows = append(rows, fmt.Sprintf("============ DEPTH: %d ==========================================", i)) } left := liverows[i] right := peersrows[i] diff --git a/swarm/network/simulations/overlay.go b/swarm/network/simulations/overlay.go index 319eeec5c4..ede71057c6 100644 --- a/swarm/network/simulations/overlay.go +++ b/swarm/network/simulations/overlay.go @@ -51,7 +51,7 @@ func (s *Simulation) NewService(ctx *adapters.ServiceContext) (node.Service, err kp.MinBinSize = 1 kp.MaxRetries = 1000 kp.RetryExponent = 2 - kp.RetryInterval = 1000 + kp.RetryInterval = 1000000 kp.PruneInterval = 2000 kad := network.NewKademlia(addr.Over(), kp) ticker := time.NewTicker(time.Duration(kad.PruneInterval) * time.Millisecond) @@ -97,7 +97,7 @@ func setupMocker(net *simulations.Network) []discover.NodeID { conf := net.Config() conf.DefaultService = "overlay" - nodeCount := 60 + nodeCount := 30 ids := make([]discover.NodeID, nodeCount) for i := 0; i < nodeCount; i++ { node, err := net.NewNode() From 82150d65cb65a826118c65fbfc5a16806c3b27bc Mon Sep 17 00:00:00 2001 From: zelig Date: Fri, 9 Jun 2017 13:41:29 +0200 Subject: [PATCH 02/17] swarm/network, pot, swarm/pss: kademlia fixes --- p2p/simulations/network.go | 2 +- pot/address.go | 214 ++++--------- pot/address_test.go | 249 +++++++-------- pot/doc.go | 18 +- pot/pot.go | 198 ++++++------ pot/pot_test.go | 220 ++++++------- swarm/network/hive.go | 118 ++++--- swarm/network/kademlia.go | 294 +++++++++--------- swarm/network/kademlia_test.go | 149 ++++----- swarm/network/protocol.go | 10 +- .../simulations/discovery/discovery_test.go | 2 +- swarm/network/simulations/overlay.go | 5 +- swarm/pss/pss.go | 11 +- swarm/pss/pss_test.go | 4 +- 14 files changed, 693 insertions(+), 801 deletions(-) diff --git a/p2p/simulations/network.go b/p2p/simulations/network.go index 706c77dab2..752103de7d 100644 --- a/p2p/simulations/network.go +++ b/p2p/simulations/network.go @@ -255,7 +255,7 @@ func (self *Network) newConn(oneID, otherID discover.NodeID) (*Conn, error) { if other == nil { return nil, fmt.Errorf("other %v does not exist", other) } - distance, _ := pot.NewBytesVal(one.Addr(), nil).PO(pot.NewBytesVal(other.Addr(), nil), 0) + distance, _ := pot.DefaultPof(256)(one.Addr(), other.Addr(), 0) return &Conn{ One: oneID, Other: otherID, diff --git a/pot/address.go b/pot/address.go index 05f2f82167..8a396dac3b 100644 --- a/pot/address.go +++ b/pot/address.go @@ -13,6 +13,8 @@ // // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . + +// Package pot see doc.go package pot import ( @@ -26,35 +28,40 @@ import ( ) var ( - zeroAddr = &common.Hash{} - zerosHex = zeroAddr.Hex()[2:] zerosBin = Address{}.Bin() ) -var ( - addrlen = keylen -) - +// Address is an alias for common.Hash type Address common.Hash +// NewAddressFromBytes constructs an Address from a byte slice +func NewAddressFromBytes(b []byte) Address { + h := common.Hash{} + copy(h[:], b) + return Address(h) +} + func (a Address) String() string { return fmt.Sprintf("%x", a[:]) } +// MarshalJSON Address serialisation func (a *Address) MarshalJSON() (out []byte, err error) { return []byte(`"` + a.String() + `"`), nil } +// UnmarshalJSON Address deserialisation func (a *Address) UnmarshalJSON(value []byte) error { *a = Address(common.HexToHash(string(value[1 : len(value)-1]))) return nil } -// the string form of the binary representation of an address (only first 8 bits) +// Bin returns the string form of the binary representation of an address (only first 8 bits) func (a Address) Bin() string { return ToBin(a[:]) } +// ToBin converts a byteslice to the string binary representation func ToBin(a []byte) string { var bs []string for _, b := range a { @@ -63,6 +70,7 @@ func ToBin(a []byte) string { return strings.Join(bs, "") } +// Bytes returns the Address as a byte slice func (a Address) Bytes() []byte { return a[:] } @@ -107,23 +115,23 @@ func posProximity(one, other Address, pos int) (ret int, eq bool) { return len(one) * 8, true } -// Address.ProxCmp compares the distances a->target and b->target. +// ProxCmp compares the distances a->target and b->target. // Returns -1 if a is closer to target, 1 if b is closer to target // and 0 if they are equal. -func (target Address) ProxCmp(a, b Address) int { - for i := range target { - da := a[i] ^ target[i] - db := b[i] ^ target[i] - if da > db { +func (a Address) ProxCmp(x, y Address) int { + for i := range a { + dx := x[i] ^ a[i] + dy := y[i] ^ a[i] + if dx > dy { return 1 - } else if da < db { + } else if dx < dy { return -1 } } return 0 } -// randomAddressAt(address, prox) generates a random address +// RandomAddressAt (address, prox) generates a random address // at proximity order prox relative to address // if prox is negative a random address is generated func RandomAddressAt(self Address, prox int) (addr Address) { @@ -148,71 +156,12 @@ func RandomAddressAt(self Address, prox int) (addr Address) { return } -// KeyRange(a0, a1, proxLimit) returns the address inclusive address -// range that contain addresses closer to one than other -// func KeyRange(one, other Address, proxLimit int) (start, stop Address) { -// prox := proximity(one, other) -// if prox >= proxLimit { -// prox = proxLimit -// } -// start = CommonBitsAddrByte(one, other, byte(0x00), prox) -// stop = CommonBitsAddrByte(one, other, byte(0xff), prox) -// return -// } - -func CommonBitsAddrF(self, other Address, f func() byte, p int) (addr Address) { - prox, _ := proximity(self, other) - var pos int - if p <= prox { - prox = p - } - pos = prox / 8 - addr = self - trans := byte(prox % 8) - var transbytea byte - if p > prox { - transbytea = byte(0x7f) - } else { - transbytea = byte(0xff) - } - transbytea >>= trans - transbyteb := transbytea ^ byte(0xff) - addrpos := addr[pos] - addrpos &= transbyteb - if p > prox { - addrpos ^= byte(0x80 >> trans) - } - addrpos |= transbytea & f() - addr[pos] = addrpos - for i := pos + 1; i < len(addr); i++ { - addr[i] = f() - } - - return -} - -func CommonBitsAddr(self, other Address, prox int) (addr Address) { - return CommonBitsAddrF(self, other, func() byte { return byte(rand.Intn(255)) }, prox) -} - -func CommonBitsAddrByte(self, other Address, b byte, prox int) (addr Address) { - return CommonBitsAddrF(self, other, func() byte { return b }, prox) -} - -// randomAddressAt() generates a random address +// RandomAddress generates a random address func RandomAddress() Address { return RandomAddressAt(Address{}, -1) } -// wraps an Address to implement the PotVal interface -type HashAddress struct { - Address -} - -func (a *HashAddress) String() string { - return a.Address.Bin() -} - +// NewAddressFromString creates a byte slice from a string in binary representation func NewAddressFromString(s string) []byte { ha := [32]byte{} @@ -227,85 +176,16 @@ func NewAddressFromString(s string) []byte { return ha[:] } -func NewHashAddress(s string) *HashAddress { - ha := NewAddressFromString(s) - h := common.Hash{} - copy(h[:], ha) - return &HashAddress{Address(h)} -} - -func NewHashAddressFromBytes(b []byte) *HashAddress { - h := common.Hash{} - copy(h[:], b) - return &HashAddress{Address(h)} -} - -// PO(addr, pos) return the proximity order of addr wrt to -// the pinned address of the tree -// assuming it is greater than or equal to pos -func (self *HashAddress) PO(val PotVal, pos int) (po int, eq bool) { - return posProximity(self.Address, val.(*HashAddress).Address, pos) -} - -type BoolAddress struct { - addr []bool -} - -func NewBoolAddress(s string) *BoolAddress { - return NewBoolAddressXOR(s, zerosBin[:len(s)]) -} - -func NewBoolAddressXOR(s, t string) *BoolAddress { - if len(s) != len(t) { - panic("lengths do not match") - } - addr := make([]bool, len(s)) - for i, _ := range addr { - addr[i] = s[i] != t[i] - } - return &BoolAddress{addr} -} - -func (self *BoolAddress) String() string { - a := self.addr - s := []byte(zerosBin)[:len(a)] - for i, one := range a { - if one { - s[i] = byte('1') - } - } - return string(s) -} - -func (self *BoolAddress) PO(val PotVal, pos int) (po int, eq bool) { - a := self.addr - b := val.(*BoolAddress).addr - for po = pos; po < len(b); po++ { - if a[po] != b[po] { - return po, false - } - } - return po, true -} - +// BytesAddress is an interface for elements addressable by a byte slice type BytesAddress interface { Address() []byte } -type bytesAddress struct { - bytes []byte - toBytes func(v AnyVal) []byte -} - -func NewBytesVal(v AnyVal, f func(v AnyVal) []byte) *bytesAddress { - if f == nil { - f = ToBytes +// ToBytes turns the Val into bytes +func ToBytes(v Val) []byte { + if v == nil { + return nil } - b := f(v) - return &bytesAddress{b, f} -} - -func ToBytes(v AnyVal) []byte { b, ok := v.([]byte) if !ok { ba, ok := v.(BytesAddress) @@ -317,15 +197,17 @@ func ToBytes(v AnyVal) []byte { return b } -func (a *bytesAddress) String() string { - return fmt.Sprintf("%08b", a.bytes) -} -func (a *bytesAddress) Address() []byte { - return a.bytes -} - -func (a *bytesAddress) PO(val PotVal, i int) (int, bool) { - return proximityOrder(a.bytes, a.toBytes(val), i) +// DefaultPof returns a proximity order operator function +// where all +func DefaultPof(max int) func(one, other Val, pos int) (int, bool) { + return func(one, other Val, pos int) (int, bool) { + po, eq := proximityOrder(ToBytes(one), ToBytes(other), pos) + if po >= max { + eq = true + po = max + } + return po, eq + } } func proximityOrder(one, other []byte, pos int) (int, bool) { @@ -346,3 +228,17 @@ func proximityOrder(one, other []byte, pos int) (int, bool) { } return len(one) * 8, true } + +// Label displays the node's key in binary format +func Label(v Val) string { + if v == nil { + return "" + } + if s, ok := v.(fmt.Stringer); ok { + return s.String() + } + if b, ok := v.([]byte); ok { + return ToBin(b) + } + panic(fmt.Sprintf("unsupported value type %T", v)) +} diff --git a/pot/address_test.go b/pot/address_test.go index b590201f41..b9b3f1f4e3 100644 --- a/pot/address_test.go +++ b/pot/address_test.go @@ -15,127 +15,128 @@ // along with the go-ethereum library. If not, see . package pot -import ( - "math/rand" - "reflect" - "testing" - - "github.com/ethereum/go-ethereum/common" -) - -func (Address) Generate(rand *rand.Rand, size int) reflect.Value { - var id Address - for i := 0; i < len(id); i++ { - id[i] = byte(uint8(rand.Intn(255))) - } - return reflect.ValueOf(id) -} - -func TestCommonBitsAddrF(t *testing.T) { - a := Address(common.HexToHash("0x0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef")) - b := Address(common.HexToHash("0x8123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef")) - c := Address(common.HexToHash("0x4123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef")) - d := Address(common.HexToHash("0x0023456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef")) - e := Address(common.HexToHash("0x01A3456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef")) - ab := CommonBitsAddrF(a, b, func() byte { return byte(0x00) }, 10) - expab := Address(common.HexToHash("0x8000000000000000000000000000000000000000000000000000000000000000")) - - if ab != expab { - t.Fatalf("%v != %v", ab, expab) - } - ac := CommonBitsAddrF(a, c, func() byte { return byte(0x00) }, 10) - expac := Address(common.HexToHash("0x4000000000000000000000000000000000000000000000000000000000000000")) - - if ac != expac { - t.Fatalf("%v != %v", ac, expac) - } - ad := CommonBitsAddrF(a, d, func() byte { return byte(0x00) }, 10) - expad := Address(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")) - - if ad != expad { - t.Fatalf("%v != %v", ad, expad) - } - ae := CommonBitsAddrF(a, e, func() byte { return byte(0x00) }, 10) - expae := Address(common.HexToHash("0x0180000000000000000000000000000000000000000000000000000000000000")) - - if ae != expae { - t.Fatalf("%v != %v", ae, expae) - } - acf := CommonBitsAddrF(a, c, func() byte { return byte(0xff) }, 10) - expacf := Address(common.HexToHash("0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")) - - if acf != expacf { - t.Fatalf("%v != %v", acf, expacf) - } - aeo := CommonBitsAddrF(a, e, func() byte { return byte(0x00) }, 2) - expaeo := Address(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")) - - if aeo != expaeo { - t.Fatalf("%v != %v", aeo, expaeo) - } - aep := CommonBitsAddrF(a, e, func() byte { return byte(0xff) }, 2) - expaep := Address(common.HexToHash("0x3fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")) - - if aep != expaep { - t.Fatalf("%v != %v", aep, expaep) - } - -} - -func TestRandomAddressAt(t *testing.T) { - var a Address - for i := 0; i < 100; i++ { - a = RandomAddress() - prox := rand.Intn(255) - b := RandomAddressAt(a, prox) - p, _ := proximity(a, b) - if p != prox { - t.Fatalf("incorrect address prox(%v, %v) == %v (expected %v)", a, b, p, prox) - } - } -} - -const ( - maxTestPOs = 1000 - testPOkeylen = 9 -) - -func TestPOs(t *testing.T) { - for i := 0; i < maxTestPOs; i++ { - length := rand.Intn(256) + 1 - v0 := RandomAddress().Bin()[:length] - v1 := RandomAddress().Bin()[:length] - a0 := NewBoolAddress(v0) - a1 := NewBoolAddress(v1) - b0 := NewHashAddress(v0) - b1 := NewHashAddress(v1) - pos := rand.Intn(length) + 1 - apo, aeq := a0.PO(a1, pos) - bpo, beq := b0.PO(b1, pos) - if bpo == 256 { - bpo = length - } - a0s := a0.String() - if a0s != v0 { - t.Fatalf("incorrect bool address. expected %v, got %v", v0, a0s) - } - a1s := a1.String() - if a1s != v1 { - t.Fatalf("incorrect bool address. expected %v, got %v", v1, a1s) - } - b0s := b0.String()[:length] - if b0s != v0 { - t.Fatalf("incorrect hash address. expected %v, got %v", v0, b0s) - } - b1s := b1.String()[:length] - if b1s != v1 { - t.Fatalf("incorrect hash address. expected %v, got %v", v1, b1s) - } - if apo != bpo { - t.Fatalf("PO does not match for %v X %v (pos: %v): expected %v, got %v", v0, v1, pos, apo, bpo) - } - if aeq != beq { - t.Fatalf("PO equality does not match for %v X %v (pos: %v): expected %v, got %v", v0, v1, pos, aeq, beq) - } - } -} +// +// import ( +// "math/rand" +// "reflect" +// "testing" +// +// "github.com/ethereum/go-ethereum/common" +// ) +// // +// // func (Address) Generate(rand *rand.Rand, size int) reflect.Value { +// // var id Address +// // for i := 0; i < len(id); i++ { +// // id[i] = byte(uint8(rand.Intn(255))) +// // } +// // return reflect.ValueOf(id) +// // } +// +// func TestCommonBitsAddrF(t *testing.T) { +// a := Address(common.HexToHash("0x0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef")) +// b := Address(common.HexToHash("0x8123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef")) +// c := Address(common.HexToHash("0x4123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef")) +// d := Address(common.HexToHash("0x0023456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef")) +// e := Address(common.HexToHash("0x01A3456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef")) +// ab := CommonBitsAddrF(a, b, func() byte { return byte(0x00) }, 10) +// expab := Address(common.HexToHash("0x8000000000000000000000000000000000000000000000000000000000000000")) +// +// if ab != expab { +// t.Fatalf("%v != %v", ab, expab) +// } +// ac := CommonBitsAddrF(a, c, func() byte { return byte(0x00) }, 10) +// expac := Address(common.HexToHash("0x4000000000000000000000000000000000000000000000000000000000000000")) +// +// if ac != expac { +// t.Fatalf("%v != %v", ac, expac) +// } +// ad := CommonBitsAddrF(a, d, func() byte { return byte(0x00) }, 10) +// expad := Address(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")) +// +// if ad != expad { +// t.Fatalf("%v != %v", ad, expad) +// } +// ae := CommonBitsAddrF(a, e, func() byte { return byte(0x00) }, 10) +// expae := Address(common.HexToHash("0x0180000000000000000000000000000000000000000000000000000000000000")) +// +// if ae != expae { +// t.Fatalf("%v != %v", ae, expae) +// } +// acf := CommonBitsAddrF(a, c, func() byte { return byte(0xff) }, 10) +// expacf := Address(common.HexToHash("0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")) +// +// if acf != expacf { +// t.Fatalf("%v != %v", acf, expacf) +// } +// aeo := CommonBitsAddrF(a, e, func() byte { return byte(0x00) }, 2) +// expaeo := Address(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")) +// +// if aeo != expaeo { +// t.Fatalf("%v != %v", aeo, expaeo) +// } +// aep := CommonBitsAddrF(a, e, func() byte { return byte(0xff) }, 2) +// expaep := Address(common.HexToHash("0x3fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")) +// +// if aep != expaep { +// t.Fatalf("%v != %v", aep, expaep) +// } +// +// } +// +// func TestRandomAddressAt(t *testing.T) { +// var a Address +// for i := 0; i < 100; i++ { +// a = RandomAddress() +// prox := rand.Intn(255) +// b := RandomAddressAt(a, prox) +// p, _ := proximity(a, b) +// if p != prox { +// t.Fatalf("incorrect address prox(%v, %v) == %v (expected %v)", a, b, p, prox) +// } +// } +// } +// +// const ( +// maxTestPOs = 1000 +// testPOkeylen = 9 +// ) +// +// func TestPOs(t *testing.T) { +// for i := 0; i < maxTestPOs; i++ { +// length := rand.Intn(256) + 1 +// v0 := RandomAddress().Bin()[:length] +// v1 := RandomAddress().Bin()[:length] +// a0 := NewBoolAddress(v0) +// a1 := NewBoolAddress(v1) +// b0 := NewHashAddress(v0) +// b1 := NewHashAddress(v1) +// pos := rand.Intn(length) + 1 +// apo, aeq := a0.PO(a1, pos) +// bpo, beq := b0.PO(b1, pos) +// if bpo == 256 { +// bpo = length +// } +// a0s := a0.String() +// if a0s != v0 { +// t.Fatalf("incorrect bool address. expected %v, got %v", v0, a0s) +// } +// a1s := a1.String() +// if a1s != v1 { +// t.Fatalf("incorrect bool address. expected %v, got %v", v1, a1s) +// } +// b0s := b0.String()[:length] +// if b0s != v0 { +// t.Fatalf("incorrect hash address. expected %v, got %v", v0, b0s) +// } +// b1s := b1.String()[:length] +// if b1s != v1 { +// t.Fatalf("incorrect hash address. expected %v, got %v", v1, b1s) +// } +// if apo != bpo { +// t.Fatalf("PO does not match for %v X %v (pos: %v): expected %v, got %v", v0, v1, pos, apo, bpo) +// } +// if aeq != beq { +// t.Fatalf("PO equality does not match for %v X %v (pos: %v): expected %v, got %v", v0, v1, pos, aeq, beq) +// } +// } +// } diff --git a/pot/doc.go b/pot/doc.go index 484590175f..5f01a91c17 100644 --- a/pot/doc.go +++ b/pot/doc.go @@ -1,5 +1,21 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + /* -POT: proximity order tree implements a container similar to a binary tree. +Package pot (proximity order tree) implements a container similar to a binary tree. Value types implement the PoVal interface which provides the PO (proximity order) comparison operator. Each fork in the trie is itself a value. Values of the subtree contained under diff --git a/pot/pot.go b/pot/pot.go index 6d7b15fce9..2ebce82cec 100644 --- a/pot/pot.go +++ b/pot/pot.go @@ -13,6 +13,8 @@ // // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . + +// Package pot see doc.go package pot import ( @@ -33,66 +35,69 @@ type Pot struct { // pot is the node type (same for root, branching node and leaf) type pot struct { - pin PotVal + pin Val bins []*pot size int po int + pof Pof } -// PotVal is the interface the generic container item should implement -type PotVal interface { - PO(PotVal, int) (po int, eq bool) - String() string -} +// Val is the element type for pots +type Val interface{} -type AnyVal interface{} +// Pof is the proximity order function +type Pof func(Val, Val, int) (int, bool) -// Pot constructor. Requires value of type PotVal to pin -// and po to point to a span in the PotVal key +// NewPot constructor. Requires value of type Val to pin +// and po to point to a span in the Val key // The pinned item counts towards the size -func NewPot(v PotVal, po int) *Pot { +func NewPot(v Val, po int, pof Pof) *Pot { var size int if v != nil { size++ } + if pof == nil { + pof = DefaultPof(keylen) + } return &Pot{ pot: &pot{ pin: v, po: po, size: size, + pof: pof, }, } } -// Pin() returns the pinned element (key) of the Pot -func (t *Pot) Pin() PotVal { +// Pin returns the pinned element (key) of the Pot +func (t *Pot) Pin() Val { return t.pin } -// Size() returns the number of values in the Pot +// Size returns the number of values in the Pot func (t *Pot) Size() int { t.lock.RLock() defer t.lock.RUnlock() return t.size } -// Add(v) inserts v into the Pot and +// Add inserts v into the Pot and // returns the proximity order of v and a boolean // indicating if the item was found // Add locks the Pot while using applicative add on its pot -func (t *Pot) Add(val PotVal) (po int, found bool) { +func (t *Pot) Add(val Val) (po int, found bool) { t.lock.Lock() defer t.lock.Unlock() t.pot, po, found = add(t.pot, val) return po, found } -// Add(t, v) returns a new Pot that contains all the elements of t +// Add called on (t, v) returns a new Pot that contains all the elements of t // plus the value v, using the applicative add // the second return value is the proximity order of the inserted element // the third is boolean indicating if the item was found // it only readlocks the Pot while reading its pot -func Add(t *Pot, val PotVal) (*Pot, int, bool) { +func Add(t *Pot, val Val) (*Pot, int, bool) { t.lock.RLock() n := t.pot t.lock.RUnlock() @@ -100,25 +105,28 @@ func Add(t *Pot, val PotVal) (*Pot, int, bool) { return &Pot{pot: r}, po, found } -func add(t *pot, val PotVal) (*pot, int, bool) { +func (t *pot) clone() *pot { + return &pot{ + pin: t.pin, + size: t.size, + po: t.po, + bins: t.bins, + pof: t.pof, + } +} + +func add(t *pot, val Val) (*pot, int, bool) { var r *pot if t == nil || t.pin == nil { - r = &pot{ - pin: val, - size: t.size + 1, - po: t.po, - bins: t.bins, - } + r = t.clone() + r.pin = val + r.size++ return r, 0, false } - po, found := t.pin.PO(val, t.po) + po, found := t.pof(t.pin, val, t.po) if found { - r = &pot{ - pin: val, - size: t.size, - po: t.po, - bins: t.bins, - } + r = t.clone() + r.pin = val return r, po, true } @@ -147,6 +155,7 @@ func add(t *pot, val PotVal) (*pot, int, bool) { pin: val, size: 1, po: po, + pof: t.pof, } } @@ -158,28 +167,29 @@ func add(t *pot, val PotVal) (*pot, int, bool) { size: size, po: t.po, bins: bins, + pof: t.pof, } return r, po, found } -// T.Re move(v) deletes v from the Pot and returns +// Remove called on (v) deletes v from the Pot and returns // the proximity order of v and a boolean value indicating // if the value was found // Remove locks Pot while using applicative remove on its pot -func (t *Pot) Remove(val PotVal) (po int, found bool) { +func (t *Pot) Remove(val Val) (po int, found bool) { t.lock.Lock() defer t.lock.Unlock() t.pot, po, found = remove(t.pot, val) return po, found } -// Remove(t, v) returns a new Pot that contains all the elements of t +// Remove called on (t, v) returns a new Pot that contains all the elements of t // minus the value v, using the applicative remove // the second return value is the proximity order of the inserted element // the third is boolean indicating if the item was found // it only readlocks the Pot while reading its pot -func Remove(t *Pot, v PotVal) (*Pot, int, bool) { +func Remove(t *Pot, v Val) (*Pot, int, bool) { t.lock.RLock() n := t.pot t.lock.RUnlock() @@ -187,14 +197,15 @@ func Remove(t *Pot, v PotVal) (*Pot, int, bool) { return &Pot{pot: r}, po, found } -func remove(t *pot, val PotVal) (r *pot, po int, found bool) { +func remove(t *pot, val Val) (r *pot, po int, found bool) { size := t.size - po, found = t.pin.PO(val, t.po) + po, found = t.pof(t.pin, val, t.po) if found { size-- if size == 0 { r = &pot{ - po: t.po, + po: t.po, + pof: t.pof, } return r, po, true } @@ -205,6 +216,7 @@ func remove(t *pot, val PotVal) (r *pot, po int, found bool) { bins: append(t.bins[:i], last.bins...), size: size, po: t.po, + pof: t.pof, } return r, t.po, true } @@ -237,58 +249,56 @@ func remove(t *pot, val PotVal) (r *pot, po int, found bool) { size: size, po: t.po, bins: bins, + pof: t.pof, } return r, po, found } -// Swap(k, f) looks up the item at k +// Swap called on (k, f) looks up the item at k // and applies the function f to the value v at k or nil if the item is not found // if f returns nil, the element is removed // if f returns v' <> v then v' is inserted into the Pot // if v' == v the pot is not changed // it panics if v'.PO(k, 0) says v and k are not equal -func (t *Pot) Swap(val AnyVal, f func(v PotVal) PotVal) (po int, found bool, change bool) { +func (t *Pot) Swap(val Val, f func(v Val) Val) (po int, found bool, change bool) { t.lock.Lock() defer t.lock.Unlock() - ba := NewBytesVal(val, nil) var t0 *pot - t0, po, found, change = swap(t.pot, ba, f) + t0, po, found, change = swap(t.pot, val, f) if change { t.pot = t0 } return po, found, change } -func swap(t *pot, k PotVal, f func(v PotVal) PotVal) (r *pot, po int, found bool, change bool) { - var val PotVal +func swap(t *pot, k Val, f func(v Val) Val) (r *pot, po int, found bool, change bool) { + var val Val if t == nil || t.pin == nil { val = f(nil) if val == nil { return t, t.po, false, false } - if _, eq := val.PO(k, t.po); !eq { - panic("value key mismatch") - } - r = &pot{ - pin: val, - size: t.size + 1, - po: t.po, - bins: t.bins, - } + // if _, eq := t.pof(k, t.pin, t.po); !eq { + // panic("value key mismatch") + // } + r = t.clone() + r.pin = val + r.size++ return r, t.po, false, true } size := t.size if k == nil { panic("k is nil") } - po, found = k.PO(t.pin, t.po) + po, found = t.pof(k, t.pin, t.po) if found { val = f(t.pin) if val == nil { size-- if size == 0 { r = &pot{ - po: t.po, + po: t.po, + pof: t.pof, } return r, po, true, true } @@ -299,18 +309,15 @@ func swap(t *pot, k PotVal, f func(v PotVal) PotVal) (r *pot, po int, found bool bins: append(t.bins[:i], last.bins...), size: size, po: t.po, + pof: t.pof, } return r, t.po, true, true // remove element } else if val == t.pin { return nil, po, true, false } else { // add element - r = &pot{ - pin: val, - size: t.size, - po: t.po, - bins: t.bins, - } + r = t.clone() + r.pin = val return r, po, true, true } } @@ -344,6 +351,7 @@ func swap(t *pot, k PotVal, f func(v PotVal) PotVal) (r *pot, po int, found bool pin: val, size: 1, po: po, + pof: t.pof, } } @@ -357,12 +365,13 @@ func swap(t *pot, k PotVal, f func(v PotVal) PotVal) (r *pot, po int, found bool size: size, po: t.po, bins: bins, + pof: t.pof, } return r, po, found, true } -// t0.Merge(t1) changes t0 to contain all the elements of t1 +// Merge called on (t1) changes t0 to contain all the elements of t1 // it locks t0, but only readlocks t1 while taking its pot // uses applicative union func (t *Pot) Merge(t1 *Pot) (c int) { @@ -388,9 +397,7 @@ func Union(t0, t1 *Pot) (*Pot, int) { t1.lock.RUnlock() p, c := union(n0, n1) - return &Pot{ - pot: p, - }, c + return &Pot{pot: p}, c } func union(t0, t1 *pot) (*pot, int) { @@ -400,7 +407,7 @@ func union(t0, t1 *pot) (*pot, int) { if t1 == nil || t1.size == 0 { return t0, 0 } - var pin PotVal + var pin Val var bins []*pot var mis []int wg := &sync.WaitGroup{} @@ -411,7 +418,7 @@ func union(t0, t1 *pot) (*pot, int) { var i0, i1 int var common int - po, eq := pin0.PO(pin1, 0) + po, eq := t0.pof(pin0, pin1, 0) for { l0 := len(bins0) @@ -486,6 +493,7 @@ func union(t0, t1 *pot) (*pot, int) { bins: bins0[i:], size: size0 + 1, po: po, + pof: t0.pof, } bins2 := []*pot{np} @@ -499,7 +507,7 @@ func union(t0, t1 *pot) (*pot, int) { bins2 = append(bins2, n0.bins...) pin0 = pin1 pin1 = n0.pin - po, eq = pin0.PO(pin1, n0.po) + po, eq = t0.pof(pin0, pin1, n0.po) } bins0 = bins1 @@ -518,21 +526,22 @@ func union(t0, t1 *pot) (*pot, int) { bins: bins, size: t0.size + t1.size - common, po: t0.po, + pof: t0.pof, } return n, common } -// Each(f) is a synchronous iterator over the bins of a node +// Each called with (f) is a synchronous iterator over the bins of a node // respecting an ordering // proximity > pinnedness -func (t *Pot) Each(f func(PotVal, int) bool) bool { +func (t *Pot) Each(f func(Val, int) bool) bool { t.lock.RLock() n := t.pot t.lock.RUnlock() return n.each(f) } -func (t *pot) each(f func(PotVal, int) bool) bool { +func (t *pot) each(f func(Val, int) bool) bool { var next bool for _, n := range t.bins { if n == nil { @@ -546,7 +555,7 @@ func (t *pot) each(f func(PotVal, int) bool) bool { return f(t.pin, t.po) } -// EachFrom(f, start) is a synchronous iterator over the elements of a pot +// EachFrom called with (f, start) is a synchronous iterator over the elements of a pot // within the inclusive range starting from proximity order start // the function argument is passed the value and the proximity order wrt the root pin // it does NOT include the pinned item of the root @@ -554,14 +563,14 @@ func (t *pot) each(f func(PotVal, int) bool) bool { // proximity > pinnedness // the iteration ends if the function return false or there are no more elements // end of a po range can be implemented since po is passed to the function -func (t *Pot) EachFrom(f func(PotVal, int) bool, po int) bool { +func (t *Pot) EachFrom(f func(Val, int) bool, po int) bool { t.lock.RLock() n := t.pot t.lock.RUnlock() return n.eachFrom(f, po) } -func (t *pot) eachFrom(f func(PotVal, int) bool, po int) bool { +func (t *pot) eachFrom(f func(Val, int) bool, po int) bool { var next bool _, lim := t.getPos(po) for i := lim; i < len(t.bins); i++ { @@ -578,18 +587,18 @@ func (t *pot) eachFrom(f func(PotVal, int) bool, po int) bool { // subtree passing the proximity order and the size // the iteration continues until the function's return value is false // or there are no more subtries -func (t *Pot) EachBin(val PotVal, po int, f func(int, int, func(func(val PotVal, i int) bool) bool) bool) { +func (t *Pot) EachBin(val Val, po int, f func(int, int, func(func(val Val, i int) bool) bool) bool) { t.lock.RLock() n := t.pot t.lock.RUnlock() n.eachBin(val, po, f) } -func (t *pot) eachBin(val PotVal, po int, f func(int, int, func(func(val PotVal, i int) bool) bool) bool) { +func (t *pot) eachBin(val Val, po int, f func(int, int, func(func(val Val, i int) bool) bool) bool) { if t == nil || t.size == 0 { return } - spr, _ := t.pin.PO(val, t.po) + spr, _ := t.pof(t.pin, val, t.po) _, lim := t.getPos(spr) var size int var n *pot @@ -604,7 +613,7 @@ func (t *pot) eachBin(val PotVal, po int, f func(int, int, func(func(val PotVal, } } if lim == len(t.bins) { - f(spr, 1, func(g func(PotVal, int) bool) bool { + f(spr, 1, func(g func(Val, int) bool) bool { return g(t.pin, spr) }) return @@ -616,8 +625,8 @@ func (t *pot) eachBin(val PotVal, po int, f func(int, int, func(func(val PotVal, spo++ size += n.size } - if !f(spr, t.size-size, func(g func(PotVal, int) bool) bool { - return t.eachFrom(func(v PotVal, j int) bool { + if !f(spr, t.size-size, func(g func(Val, int) bool) bool { + return t.eachFrom(func(v Val, j int) bool { return g(v, spr) }, spo) }) { @@ -628,17 +637,17 @@ func (t *pot) eachBin(val PotVal, po int, f func(int, int, func(func(val PotVal, } } -// syncronous iterator over neighbours of any target val +// EachNeighbour is a syncronous iterator over neighbours of any target val // the order of elements retrieved reflect proximity order to the target // TODO: add maximum proxbin to start range of iteration -func (t *Pot) EachNeighbour(val PotVal, f func(PotVal, int) bool) bool { +func (t *Pot) EachNeighbour(val Val, f func(Val, int) bool) bool { t.lock.RLock() n := t.pot t.lock.RUnlock() return n.eachNeighbour(val, f) } -func (t *pot) eachNeighbour(val PotVal, f func(PotVal, int) bool) bool { +func (t *pot) eachNeighbour(val Val, f func(Val, int) bool) bool { if t == nil || t.size == 0 { return false } @@ -647,7 +656,7 @@ func (t *pot) eachNeighbour(val PotVal, f func(PotVal, int) bool) bool { var n *pot ir := l il := l - po, eq := t.pin.PO(val, t.po) + po, eq := t.pof(t.pin, val, t.po) if !eq { n, il = t.getPos(po) if n != nil { @@ -667,7 +676,7 @@ func (t *pot) eachNeighbour(val PotVal, f func(PotVal, int) bool) bool { } for i := l - 1; i > ir; i-- { - next = t.bins[i].each(func(v PotVal, _ int) bool { + next = t.bins[i].each(func(v Val, _ int) bool { return f(v, po) }) if !next { @@ -677,7 +686,7 @@ func (t *pot) eachNeighbour(val PotVal, f func(PotVal, int) bool) bool { for i := il - 1; i >= 0; i-- { n := t.bins[i] - next = n.each(func(v PotVal, _ int) bool { + next = n.each(func(v Val, _ int) bool { return f(v, n.po) }) if !next { @@ -687,7 +696,7 @@ func (t *pot) eachNeighbour(val PotVal, f func(PotVal, int) bool) bool { return true } -// EachNeighnbourAsync(val, max, maxPos, f, wait) is an asyncronous iterator +// EachNeighbourAsync called on (val, max, maxPos, f, wait) is an asyncronous iterator // over elements not closer than maxPos wrt val. // val does not need to be match an element of the pot, but if it does, and // maxPos is keylength than it is included in the iteration @@ -698,7 +707,7 @@ func (t *pot) eachNeighbour(val PotVal, f func(PotVal, int) bool) bool { // or if the entire there are no nodes not closer than maxPos that is not visited // if wait is true, the iterator returns only if all calls to f are finished // TODO: implement minPos for proper prox range iteration -func (t *Pot) EachNeighbourAsync(val PotVal, max int, maxPos int, f func(PotVal, int), wait bool) { +func (t *Pot) EachNeighbourAsync(val Val, max int, maxPos int, f func(Val, int), wait bool) { t.lock.RLock() n := t.pot t.lock.RUnlock() @@ -715,15 +724,14 @@ func (t *Pot) EachNeighbourAsync(val PotVal, max int, maxPos int, f func(PotVal, } } -func (t *pot) eachNeighbourAsync(val PotVal, max int, maxPos int, f func(PotVal, int), wg *sync.WaitGroup) (extra int) { - +func (t *pot) eachNeighbourAsync(val Val, max int, maxPos int, f func(Val, int), wg *sync.WaitGroup) (extra int) { l := len(t.bins) var n *pot il := l ir := l // ic := l - po, eq := t.pin.PO(val, t.po) + po, eq := t.pof(t.pin, val, t.po) // if po is too close, set the pivot branch (pom) to maxPos pom := po @@ -799,7 +807,7 @@ func (t *pot) eachNeighbourAsync(val PotVal, max int, maxPos int, f func(PotVal, wg.Add(m) } go func(pn *pot, pm int) { - pn.each(func(v PotVal, _ int) bool { + pn.each(func(v Val, _ int) bool { if wg != nil { defer wg.Done() } @@ -826,7 +834,7 @@ func (t *pot) eachNeighbourAsync(val PotVal, max int, maxPos int, f func(PotVal, wg.Add(m) } go func(pn *pot, pm int) { - pn.each(func(v PotVal, _ int) bool { + pn.each(func(v Val, _ int) bool { if wg != nil { defer wg.Done() } @@ -840,7 +848,7 @@ func (t *pot) eachNeighbourAsync(val PotVal, max int, maxPos int, f func(PotVal, return max + extra } -// getPos(n) returns the forking node at PO n and its index if it exists +// getPos called on (n) returns the forking node at PO n and its index if it exists // otherwise nil // caller is suppoed to hold the lock func (t *pot) getPos(po int) (n *pot, i int) { @@ -856,7 +864,7 @@ func (t *pot) getPos(po int) (n *pot, i int) { return nil, len(t.bins) } -// need(m, max, extra) uses max m out of extra, and then max +// need called on (m, max, extra) uses max m out of extra, and then max // if needed, returns the adjusted counts func need(m, max, extra int) (int, int, int) { if m <= extra { diff --git a/pot/pot_test.go b/pot/pot_test.go index 263dd7942d..1d4dd76966 100644 --- a/pot/pot_test.go +++ b/pot/pot_test.go @@ -37,55 +37,45 @@ func init() { log.Root().SetHandler(log.LvlFilterHandler(log.LvlError, log.StreamHandler(os.Stderr, log.TerminalFormat(false)))) } -type testBVAddr struct { - *HashAddress - i int -} - -func NewTestBVAddr(s string, i int) *testBVAddr { - return &testBVAddr{NewHashAddress(s), i} -} - -func (a *testBVAddr) String() string { - return a.HashAddress.String()[:keylen] -} - -func (self *testBVAddr) PO(val PotVal, po int) (int, bool) { - return self.HashAddress.PO(val.(*testBVAddr).HashAddress, po) -} - type testAddr struct { - *BoolAddress + a []byte i int } -func NewTestAddr(s string, i int) *testAddr { - return &testAddr{NewBoolAddress(s), i} +// +// func newTestAddr(s string, i int) *testAddr { +// b := NewAddressFromString(s) +// h := &common.Hash{} +// copy((*h)[:], b) +// a := Address(*h) +// return &testAddr{&a, i} +// } + +func newTestAddr(s string, i int) *testAddr { + return &testAddr{NewAddressFromString(s), i} } -func (self *testAddr) PO(val PotVal, po int) (int, bool) { - return self.BoolAddress.PO(val.(*testAddr).BoolAddress, po) +func (a *testAddr) Address() []byte { + return a.a } -func str(v PotVal) string { - if v == nil { - return "" - } - return v.(*testAddr).String() +func (a *testAddr) String() string { + return Label(a.a) + // return a.Address.String()[:keylen] } func randomTestAddr(n int, i int) *testAddr { v := RandomAddress().Bin()[:n] - return NewTestAddr(v, i) + return newTestAddr(v, i) } -func randomTestBVAddr(n int, i int) *testBVAddr { +func randomtestAddr(n int, i int) *testAddr { v := RandomAddress().Bin()[:n] - return NewTestBVAddr(v, i) + return newTestAddr(v, i) } func indexes(t *Pot) (i []int, po []int) { - t.Each(func(v PotVal, p int) bool { + t.Each(func(v Val, p int) bool { a := v.(*testAddr) i = append(i, a.i) po = append(po, p) @@ -96,16 +86,16 @@ func indexes(t *Pot) (i []int, po []int) { func testAdd(t *Pot, n int, values ...string) { for i, val := range values { - t.Add(NewTestAddr(val, i+n)) + t.Add(newTestAddr(val, i+n)) } } // func RandomBoolAddress() func TestPotAdd(t *testing.T) { - n := NewPot(NewTestAddr("001111", 0), 0) + n := NewPot(newTestAddr("00111100", 0), 0, DefaultPof(8)) // Pin set correctly - exp := "001111" - got := str(n.Pin())[:6] + exp := "00111100" + got := Label(n.Pin())[:8] if got != exp { t.Fatalf("incorrect pinned value. Expected %v, got %v", exp, got) } @@ -116,7 +106,7 @@ func TestPotAdd(t *testing.T) { t.Fatalf("incorrect number of elements in Pot. Expected %v, got %v", expi, goti) } - testAdd(n, 1, "011111", "001111", "011111", "000111") + testAdd(n, 1, "01111100", "00111100", "01111100", "00011100") // check size goti = n.Size() expi = 3 @@ -138,15 +128,15 @@ func TestPotAdd(t *testing.T) { // func RandomBoolAddress() func TestPotRemove(t *testing.T) { - n := NewPot(NewTestAddr("001111", 0), 0) - n.Remove(NewTestAddr("001111", 0)) - exp := "" - got := str(n.Pin()) + n := NewPot(newTestAddr("00111100", 0), 0, DefaultPof(8)) + n.Remove(newTestAddr("00111100", 0)) + exp := "" + got := Label(n.Pin()) if got != exp { t.Fatalf("incorrect pinned value. Expected %v, got %v", exp, got) } - testAdd(n, 1, "000000", "011111", "001111", "000111") - n.Remove(NewTestAddr("001111", 0)) + testAdd(n, 1, "00000000", "01111100", "00111100", "00011100") + n.Remove(newTestAddr("00111100", 0)) goti := n.Size() expi := 3 if goti != expi { @@ -164,7 +154,7 @@ func TestPotRemove(t *testing.T) { t.Fatalf("incorrect po-s in iteration over Pot. Expected %v, got %v", exp, got) } // remove again - n.Remove(NewTestAddr("001111", 0)) + n.Remove(newTestAddr("00111100", 0)) inds, po = indexes(n) got = fmt.Sprintf("%v", inds) exp = "[2 4]" @@ -175,31 +165,32 @@ func TestPotRemove(t *testing.T) { } func TestPotSwap(t *testing.T) { + // t.Skip("") max := maxEachNeighbour - n := NewPot(nil, 0) - var m []*testBVAddr + n := NewPot(nil, 0, nil) + var m []*testAddr for j := 0; j < 2*max; { - v := randomTestBVAddr(keylen, j) + v := randomtestAddr(keylen, j) _, found := n.Add(v) if !found { m = append(m, v) j++ } } - k := make(map[string]*testBVAddr) + k := make(map[string]*testAddr) for j := 0; j < max; { - v := randomTestBVAddr(keylen, 1) - _, found := k[v.String()] + v := randomtestAddr(keylen, 1) + _, found := k[Label(v)] if !found { - k[v.String()] = v + k[Label(v)] = v j++ } } for _, v := range k { m = append(m, v) } - f := func(v PotVal) PotVal { - tv := v.(*testBVAddr) + f := func(v Val) Val { + tv := v.(*testAddr) if tv.i < max { return nil } @@ -207,7 +198,7 @@ func TestPotSwap(t *testing.T) { return v } for _, val := range m { - n.Swap(val, func(v PotVal) PotVal { + n.Swap(val, func(v Val) Val { if v == nil { return val } @@ -215,9 +206,9 @@ func TestPotSwap(t *testing.T) { }) } sum := 0 - n.Each(func(v PotVal, i int) bool { + n.Each(func(v Val, i int) bool { sum++ - tv := v.(*testBVAddr) + tv := v.(*testAddr) if tv.i > 1 { t.Fatalf("item value incorrect, expected 0, got %v", tv.i) } @@ -231,10 +222,10 @@ func TestPotSwap(t *testing.T) { } } -func checkPo(val PotVal) func(PotVal, int) error { - return func(v PotVal, po int) error { +func checkPo(val Val, pof Pof) func(Val, int) error { + return func(v Val, po int) error { // check the po - exp, _ := val.PO(v, 0) + exp, _ := pof(val, v, 0) if po != exp { return fmt.Errorf("incorrect prox order for item %v in neighbour iteration for %v. Expected %v, got %v", v, val, exp, po) } @@ -242,9 +233,9 @@ func checkPo(val PotVal) func(PotVal, int) error { } } -func checkOrder(val PotVal) func(PotVal, int) error { +func checkOrder(val Val) func(Val, int) error { var po int = keylen - return func(v PotVal, p int) error { + return func(v Val, p int) error { if po < p { return fmt.Errorf("incorrect order for item %v in neighbour iteration for %v. PO %v > %v (previous max)", v, val, p, po) } @@ -253,26 +244,26 @@ func checkOrder(val PotVal) func(PotVal, int) error { } } -func checkValues(m map[string]bool, val PotVal) func(PotVal, int) error { - return func(v PotVal, po int) error { - duplicate, ok := m[v.String()] +func checkValues(m map[string]bool, val Val) func(Val, int) error { + return func(v Val, po int) error { + duplicate, ok := m[Label(v)] if !ok { return fmt.Errorf("alien value %v", v) } if duplicate { return fmt.Errorf("duplicate value returned: %v", v) } - m[v.String()] = true + m[Label(v)] = true return nil } } var errNoCount = errors.New("not count") -func testPotEachNeighbour(n *Pot, val PotVal, expCount int, fs ...func(PotVal, int) error) error { +func testPotEachNeighbour(n *Pot, val Val, expCount int, fs ...func(Val, int) error) error { var err error var count int - n.EachNeighbour(val, func(v PotVal, po int) bool { + n.EachNeighbour(val, func(v Val, po int) bool { for _, f := range fs { err = f(v, po) if err != nil { @@ -297,15 +288,14 @@ const ( ) func TestPotMergeOne(t *testing.T) { - pot1 := NewPot(nil, 0) - pot1.Add(NewTestAddr("10", 0)) - pot1.Add(NewTestAddr("00", 0)) - pot2 := NewPot(nil, 0) - pot2.Add(NewTestAddr("01", 0)) - log.Debug(fmt.Sprintf("\n%v\n%v", pot2, pot1)) + pot1 := NewPot(nil, 0, DefaultPof(2)) + pot1.Add(newTestAddr("10", 0)) + pot1.Add(newTestAddr("00", 0)) + pot2 := NewPot(nil, 0, DefaultPof(2)) + pot2.Add(newTestAddr("01", 0)) pot1.Merge(pot2) count := 0 - pot1.Each(func(val PotVal, i int) bool { + pot1.Each(func(val Val, i int) bool { count++ return true }) @@ -315,25 +305,25 @@ func TestPotMergeOne(t *testing.T) { } func TestPotMergeCommon(t *testing.T) { - vs := make([]*testBVAddr, mergeTestCount) + vs := make([]*testAddr, mergeTestCount) + // vs := make([]*testAddr, mergeTestCount) for i := 0; i < maxEachNeighbourTests; i++ { for i := 0; i < len(vs); i++ { - vs[i] = randomTestBVAddr(keylen, i) + vs[i] = randomtestAddr(keylen, i) } max0 := rand.Intn(mergeTestChoose) + 1 max1 := rand.Intn(mergeTestChoose) + 1 - n0 := NewPot(nil, 0) - n1 := NewPot(nil, 0) + n0 := NewPot(nil, 0, nil) + n1 := NewPot(nil, 0, nil) log.Trace(fmt.Sprintf("round %v: %v - %v", i, max0, max1)) m := make(map[string]bool) for j := 0; j < max0; { r := rand.Intn(max0) v := vs[r] - // v := randomTestBVAddr(keylen, j) _, found := n0.Add(v) if !found { - m[v.String()] = false + m[Label(v)] = false j++ } } @@ -346,10 +336,10 @@ func TestPotMergeCommon(t *testing.T) { if !found { j++ } - _, found = m[v.String()] + _, found = m[Label(v)] if !found { expAdded++ - m[v.String()] = false + m[Label(v)] = false } } if i < 6 { @@ -372,8 +362,8 @@ func TestPotMergeCommon(t *testing.T) { if !checkDuplicates(n.pot) { t.Fatalf("%v: merged pot contains duplicates: \n%v", i, n) } - for k, _ := range m { - _, found := n.Add(NewTestBVAddr(k, 0)) + for k := range m { + _, found := n.Add(newTestAddr(k, 0)) if !found { t.Fatalf("%v: merged pot (size:%v, added: %v) missing element %v\n%v", i, size, added, k, n) } @@ -385,32 +375,32 @@ func TestPotMergeScale(t *testing.T) { for i := 0; i < maxEachNeighbourTests; i++ { max0 := rand.Intn(maxEachNeighbour) + 1 max1 := rand.Intn(maxEachNeighbour) + 1 - n0 := NewPot(nil, 0) - n1 := NewPot(nil, 0) + n0 := NewPot(nil, 0, nil) + n1 := NewPot(nil, 0, nil) log.Trace(fmt.Sprintf("round %v: %v - %v", i, max0, max1)) m := make(map[string]bool) for j := 0; j < max0; { - v := randomTestBVAddr(keylen, j) - // v := randomTestBVAddr(keylen, j) + v := randomtestAddr(keylen, j) + // v := randomtestAddr(keylen, j) _, found := n0.Add(v) if !found { - m[v.String()] = false + m[Label(v)] = false j++ } } expAdded := 0 for j := 0; j < max1; { - v := randomTestBVAddr(keylen, j) - // v := randomTestBVAddr(keylen, j) + v := randomtestAddr(keylen, j) + // v := randomtestAddr(keylen, j) _, found := n1.Add(v) if !found { j++ } - _, found = m[v.String()] + _, found = m[Label(v)] if !found { expAdded++ - m[v.String()] = false + m[Label(v)] = false } } if i < 6 { @@ -433,8 +423,8 @@ func TestPotMergeScale(t *testing.T) { if !checkDuplicates(n.pot) { t.Fatalf("%v: merged pot contains duplicates: \n%v", i, n) } - for k, _ := range m { - _, found := n.Add(NewTestBVAddr(k, 0)) + for k := range m { + _, found := n.Add(newTestAddr(k, 0)) if !found { t.Fatalf("%v: merged pot (size:%v, added: %v) missing element %v\n%v", i, size, added, k, n) } @@ -443,7 +433,7 @@ func TestPotMergeScale(t *testing.T) { } func checkDuplicates(t *pot) bool { - var po int = -1 + po := -1 for _, c := range t.bins { if c == nil { return false @@ -460,13 +450,13 @@ func TestPotEachNeighbourSync(t *testing.T) { for i := 0; i < maxEachNeighbourTests; i++ { max := rand.Intn(maxEachNeighbour/2) + maxEachNeighbour/2 pin := randomTestAddr(keylen, 0) - n := NewPot(pin, 0) + n := NewPot(pin, 0, nil) m := make(map[string]bool) - m[pin.String()] = false + m[Label(pin)] = false for j := 1; j <= max; j++ { v := randomTestAddr(keylen, j) n.Add(v) - m[v.String()] = false + m[Label(v)] = false } size := n.Size() @@ -476,14 +466,14 @@ func TestPotEachNeighbourSync(t *testing.T) { count := rand.Intn(size/2) + size/2 val := randomTestAddr(keylen, max+1) log.Trace(fmt.Sprintf("%v: pin: %v, size: %v, val: %v, count: %v", i, n.Pin(), size, val, count)) - err := testPotEachNeighbour(n, val, count, checkPo(val), checkOrder(val), checkValues(m, val)) + err := testPotEachNeighbour(n, val, count, checkPo(val, n.pof), checkOrder(val), checkValues(m, val)) if err != nil { t.Fatal(err) } minPoFound := keylen maxPoNotFound := 0 for k, found := range m { - po, _ := val.PO(NewTestAddr(k, 0), 0) + po, _ := n.pof(val, newTestAddr(k, 0), 0) if found { if po < minPoFound { minPoFound = po @@ -503,8 +493,8 @@ func TestPotEachNeighbourSync(t *testing.T) { func TestPotEachNeighbourAsync(t *testing.T) { for i := 0; i < maxEachNeighbourTests; i++ { max := rand.Intn(maxEachNeighbour/2) + maxEachNeighbour/2 - n := NewPot(randomTestAddr(keylen, 0), 0) - var size int = 1 + n := NewPot(randomTestAddr(keylen, 0), 0, nil) + size := 1 for j := 1; j <= max; j++ { v := randomTestAddr(keylen, j) _, found := n.Add(v) @@ -526,32 +516,24 @@ func TestPotEachNeighbourAsync(t *testing.T) { maxPos := rand.Intn(keylen) log.Trace(fmt.Sprintf("%v: pin: %v, size: %v, val: %v, count: %v, maxPos: %v", i, n.Pin(), size, val, count, maxPos)) msize := 0 - remember := func(v PotVal, po int) error { - // mu.Lock() - // defer mu.Unlock() + remember := func(v Val, po int) error { if po > maxPos { - // log.Trace(fmt.Sprintf("NOT ADD %v", v)) return errNoCount } - // log.Trace(fmt.Sprintf("ADD %v, %v", v, msize)) - m[v.String()] = true + m[Label(v)] = true msize++ return nil } if i == 0 { continue } - err := testPotEachNeighbour(n, val, count, remember) - if err != nil { - log.Error(err.Error()) - } + testPotEachNeighbour(n, val, count, remember) d := 0 - forget := func(v PotVal, po int) { + forget := func(v Val, po int) { mu.Lock() defer mu.Unlock() d++ - // log.Trace(fmt.Sprintf("DEL %v", v)) - delete(m, v.String()) + delete(m, Label(v)) } n.EachNeighbourAsync(val, count, maxPos, forget, true) @@ -567,7 +549,7 @@ func TestPotEachNeighbourAsync(t *testing.T) { func benchmarkEachNeighbourSync(t *testing.B, max, count int, d time.Duration) { t.ReportAllocs() pin := randomTestAddr(keylen, 0) - n := NewPot(pin, 0) + n := NewPot(pin, 0, nil) for j := 1; j <= max; { v := randomTestAddr(keylen, j) _, found := n.Add(v) @@ -579,7 +561,7 @@ func benchmarkEachNeighbourSync(t *testing.B, max, count int, d time.Duration) { for i := 0; i < t.N; i++ { val := randomTestAddr(keylen, max+1) m := 0 - n.EachNeighbour(val, func(v PotVal, po int) bool { + n.EachNeighbour(val, func(v Val, po int) bool { time.Sleep(d) m++ if m == count { @@ -597,7 +579,7 @@ func benchmarkEachNeighbourSync(t *testing.B, max, count int, d time.Duration) { func benchmarkEachNeighbourAsync(t *testing.B, max, count int, d time.Duration) { t.ReportAllocs() pin := randomTestAddr(keylen, 0) - n := NewPot(pin, 0) + n := NewPot(pin, 0, nil) for j := 1; j <= max; { v := randomTestAddr(keylen, j) _, found := n.Add(v) @@ -608,7 +590,7 @@ func benchmarkEachNeighbourAsync(t *testing.B, max, count int, d time.Duration) t.ResetTimer() for i := 0; i < t.N; i++ { val := randomTestAddr(keylen, max+1) - n.EachNeighbourAsync(val, count, keylen, func(v PotVal, po int) { + n.EachNeighbourAsync(val, count, keylen, func(v Val, po int) { time.Sleep(d) }, true) } diff --git a/swarm/network/hive.go b/swarm/network/hive.go index e402930569..ab492201c5 100644 --- a/swarm/network/hive.go +++ b/swarm/network/hive.go @@ -89,9 +89,10 @@ type Hive struct { tick <-chan time.Time } -// Hive constructor embeds both arguments +// NewHive constructs a new hive // HiveParams: config parameters // Overlay: Topology Driver Interface +// StateStore: to save peers across sessions func NewHive(params *HiveParams, overlay Overlay, store StateStore) *Hive { return &Hive{ HiveParams: params, @@ -105,105 +106,98 @@ func NewHive(params *HiveParams, overlay Overlay, store StateStore) *Hive { // these are called on the p2p.Server which runs on the node // af() returns an arbitrary ticker channel // rw is a read writer for json configs -func (self *Hive) Start(server *p2p.Server) error { - if self.store != nil { - if err := self.loadPeers(); err != nil { +func (h *Hive) Start(server *p2p.Server) error { + if h.store != nil { + if err := h.loadPeers(); err != nil { return err } } - self.more = make(chan bool, 1) - self.quit = make(chan bool) - log.Debug("hive started") + h.more = make(chan bool, 1) + h.quit = make(chan bool) // this loop is doing bootstrapping and maintains a healthy table - go self.keepAlive() + go h.keepAlive() go func() { // each iteration, ask kademlia about most preferred peer - for more := range self.more { + for more := range h.more { if !more { // receiving false closes the loop while allowing parallel routines // to attempt to write to more (remove Peer when shutting down) return } - log.Debug("hive delegate to overlay driver: suggest addr to connect to") + log.Trace(fmt.Sprintf("%x: hive delegate to overlay driver: suggest addr to connect to", h.BaseAddr()[:4])) // log.Trace("hive delegate to overlay driver: suggest addr to connect to") - addr, order, want := self.SuggestPeer() - if self.Discovery { + addr, order, want := h.SuggestPeer() + if h.Discovery { if addr != nil { - log.Info(fmt.Sprintf("========> connect to bee %v", addr)) + log.Trace(fmt.Sprintf("%x ========> connect to bee %x", h.BaseAddr()[:4], addr.Address()[:4])) under, err := discover.ParseNode(string(addr.(Addr).Under())) if err == nil { server.AddPeer(under) } else { - log.Error(fmt.Sprintf("===X====> connect to bee %v failed: invalid node URL: %v", addr, err)) + log.Error(fmt.Sprintf("%x ===X====> connect to bee %x failed: invalid node URL: %v", h.BaseAddr()[:4], addr.Address()[:4], err)) } } else { - log.Trace("cannot suggest peers") + log.Trace(fmt.Sprintf("%x cannot suggest peers", h.BaseAddr()[:4])) } if want { - log.Debug(fmt.Sprintf("========> request peers nearest %v", addr)) - RequestOrder(self.Overlay, uint8(order), self.PeersBroadcastSetSize, self.MaxPeersPerRequest) + log.Trace(fmt.Sprintf("%x ========> request peers for PO%0d", h.BaseAddr()[:4], order)) + RequestOrder(h.Overlay, uint8(order), h.PeersBroadcastSetSize, h.MaxPeersPerRequest) } } - log.Info(fmt.Sprintf("%v", self)) - select { - case <-self.quit: - return - default: - } + log.Trace(fmt.Sprintf("%v", h)) } }() return nil } // Stop terminates the updateloop and saves the peers -func (self *Hive) Stop() { - if self.store != nil { - self.savePeers() +func (h *Hive) Stop() { + if h.store != nil { + h.savePeers() } // closing toggle channel quits the updateloop - close(self.quit) + close(h.quit) } -func (self *Hive) Run(p *bzzPeer) error { - dp := NewDiscovery(p, self) +// Run protocol run function +func (h *Hive) Run(p *bzzPeer) error { + dp := NewDiscovery(p, h) log.Debug(fmt.Sprintf("to add new bee %v", p)) - self.On(dp) - self.wake() - defer self.wake() - defer self.Off(dp) + h.On(dp) + h.wake() + defer h.wake() + defer h.Off(dp) return p.Run(dp.HandleMsg) } // NodeInfo function is used by the p2p.server RPC interface to display // protocol specific node information -func (self *Hive) NodeInfo() interface{} { - return self.String() +func (h *Hive) NodeInfo() interface{} { + return h.String() } // PeerInfo function is used by the p2p.server RPC interface to display // protocol specific information any connected peer referred to by their NodeID -func (self *Hive) PeerInfo(id discover.NodeID) interface{} { - self.lock.Lock() - defer self.lock.Unlock() +func (h *Hive) PeerInfo(id discover.NodeID) interface{} { + h.lock.Lock() + defer h.lock.Unlock() addr := NewAddrFromNodeID(id) return interface{}(addr) } -func (self *Hive) Register(peers chan OverlayAddr) error { - defer self.wake() - return self.Overlay.Register(peers) +func (h *Hive) Register(peers chan OverlayAddr) error { + defer h.wake() + return h.Overlay.Register(peers) } // wake triggers -func (self *Hive) wake() { +func (h *Hive) wake() { select { - case self.more <- true: - log.Trace("hive woken up") - case <-self.quit: + case h.more <- true: + case <-h.quit: default: - log.Trace("hive already awake") } } @@ -233,30 +227,30 @@ func (t *timeTicker) Ch() <-chan time.Time { // keepAlive is a forever loop // in its awake state it periodically triggers connection attempts -// by writing to self.more until Kademlia Table is saturated -// wake state is toggled by writing to self.toggle +// by writing to h.more until Kademlia Table is saturated +// wake state is toggled by writing to h.toggle // it goes to sleep mode if table is saturated // it restarts if the table becomes non-full again due to disconnections -func (self *Hive) keepAlive() { - if self.tick == nil { - ticker := time.NewTicker(self.KeepAliveInterval) +func (h *Hive) keepAlive() { + if h.tick == nil { + ticker := time.NewTicker(h.KeepAliveInterval) defer ticker.Stop() - self.tick = ticker.C + h.tick = ticker.C } for { select { - case <-self.tick: - log.Debug("wake up: make hive alive") - self.wake() - case <-self.quit: + case <-h.tick: + h.wake() + case <-h.quit: + h.more <- false return } } } // loadPeers, savePeer implement persistence callback/ -func (self *Hive) loadPeers() error { - data, err := self.store.Load("peers") +func (h *Hive) loadPeers() error { + data, err := h.store.Load("peers") if err != nil { return err } @@ -275,13 +269,13 @@ func (self *Hive) loadPeers() error { c <- a } }() - return self.Overlay.Register(c) + return h.Overlay.Register(c) } // savePeers, savePeer implement persistence callback/ -func (self *Hive) savePeers() error { +func (h *Hive) savePeers() error { var peers []*bzzAddr - self.Overlay.EachAddr(nil, 256, func(pa OverlayAddr, i int) bool { + h.Overlay.EachAddr(nil, 256, func(pa OverlayAddr, i int) bool { if pa == nil { log.Warn(fmt.Sprintf("empty addr: %v", i)) return true @@ -293,7 +287,7 @@ func (self *Hive) savePeers() error { if err != nil { return fmt.Errorf("could not encode peers: %v", err) } - if err := self.store.Save("peers", data); err != nil { + if err := h.store.Save("peers", data); err != nil { return fmt.Errorf("could not save peers: %v", err) } return nil diff --git a/swarm/network/kademlia.go b/swarm/network/kademlia.go index 69e2ef6f8e..1d7663c124 100644 --- a/swarm/network/kademlia.go +++ b/swarm/network/kademlia.go @@ -93,11 +93,12 @@ func NewKademlia(addr []byte, params *KadParams) *Kademlia { return &Kademlia{ base: addr, KadParams: params, - addrs: pot.NewPot(nil, 0), - conns: pot.NewPot(nil, 0), + addrs: pot.NewPot(nil, 0, nil), + conns: pot.NewPot(nil, 0, nil), } } +// Notifier interface type for peer allowing / requesting peer and depth notifications type Notifier interface { NotifyPeer(OverlayAddr, uint8) error NotifyDepth(uint8) error @@ -116,16 +117,14 @@ type OverlayConn interface { Off() OverlayAddr // call to return a persitent OverlayAddr } +// OverlayAddr represents a kademlia peer record type OverlayAddr interface { OverlayPeer Update(OverlayAddr) OverlayAddr // returns the updated version of the original } // entry represents a Kademlia table entry (an extension of OverlayPeer) -// implements the pot.PotVal interface via BytesAddress, so entry can be -// used directly as a pot element type entry struct { - pot.PotVal OverlayPeer seenAt time.Time retries int @@ -134,51 +133,56 @@ type entry struct { // newEntry creates a kademlia peer from an OverlayPeer interface func newEntry(p OverlayPeer) *entry { return &entry{ - PotVal: pot.NewBytesVal(p, nil), OverlayPeer: p, seenAt: time.Now(), } } -func (self *entry) addr() OverlayAddr { - a, _ := self.OverlayPeer.(OverlayAddr) +// Bin is the binary (bitvector) serialisation of the entry address +func (e *entry) Bin() string { + return pot.ToBin(e.addr().Address()) +} + +// Label is a short tag for the entry for debug +func Label(e *entry) string { + return fmt.Sprintf("%s (%d)", e.Bin()[:8], e.retries) +} + +// Hex is the hexadecimal serialisation of the entry address +func (e *entry) Hex() string { + return fmt.Sprintf("%x", e.addr().Address()) +} + +// String is the short tag for the entry +func (e *entry) String() string { + return fmt.Sprintf("%s (%d)", e.Hex()[:4], e.retries) +} + +// addr returns the kad peer record (OverlayAddr) corresponding to the entry +func (e *entry) addr() OverlayAddr { + a, _ := e.OverlayPeer.(OverlayAddr) return a } -func (self *entry) conn() OverlayConn { - c, _ := self.OverlayPeer.(OverlayConn) +// conn returns the connected peer (OverlayPeer) corresponding to the entry +func (e *entry) conn() OverlayConn { + c, _ := e.OverlayPeer.(OverlayConn) return c } -func (self *entry) String() string { - return fmt.Sprintf("%x", self.OverlayPeer.Address()) -} - // Register enters each OverlayAddr as kademlia peer record into the // database of known peer addresses -func (self *Kademlia) Register(peers chan OverlayAddr) error { - np := pot.NewPot(nil, 0) +func (k *Kademlia) Register(peers chan OverlayAddr) error { + np := pot.NewPot(nil, 0, nil) for p := range peers { - // error if self received, peer should know better - if bytes.Equal(p.Address(), self.base) { - return fmt.Errorf("add peers: %x is self", self.base) + // error if k received, peer should know better + if bytes.Equal(p.Address(), k.base) { + return fmt.Errorf("add peers: %x is k", k.base) } np, _, _ = pot.Add(np, newEntry(p)) } - com := self.addrs.Merge(np) - log.Debug(fmt.Sprintf("merged %v peers, %v known, total: %v", np.Size(), com, self.addrs.Size())) - // log.Trace(fmt.Sprintf("merged %v peers, %v known", np.Size(), com)) - - // TODO: remove this check - m := make(map[string]bool) - self.addrs.Each(func(val pot.PotVal, i int) bool { - _, found := m[val.String()] - if found { - panic("duplicate found") - } - m[val.String()] = true - return true - }) + com := k.addrs.Merge(np) + log.Trace(fmt.Sprintf("%x merged %v peers, %v known, total: %v", k.BaseAddr()[:4], np.Size(), com, k.addrs.Size())) return nil } @@ -186,32 +190,31 @@ func (self *Kademlia) Register(peers chan OverlayAddr) error { // lowest bincount below depth // naturally if there is an empty row it returns a peer for that // -func (self *Kademlia) SuggestPeer() (a OverlayAddr, o int, want bool) { - minsize := self.MinBinSize - depth := self.Depth() - empty := self.FirstEmptyBin() +func (k *Kademlia) SuggestPeer() (a OverlayAddr, o int, want bool) { + minsize := k.MinBinSize + depth := k.Depth() + // empty := k.FirstEmptyBin() // if there is a callable neighbour within the current proxBin, connect // this makes sure nearest neighbour set is fully connected - log.Debug(fmt.Sprintf("candidate nearest neighbour checking above PO %v", depth)) - // log.Trace(fmt.Sprintf("candidate nearest neighbour checking above PO %v", depth)) var ppo int - ba := pot.NewBytesVal(self.base, nil) - self.addrs.EachNeighbour(ba, func(val pot.PotVal, po int) bool { - a = self.callable(val) - log.Trace(fmt.Sprintf("candidate nearest neighbour at %x: %v (%v). a == nil is %v", val.(*entry).Address(), a, po, a == nil)) + k.addrs.EachNeighbour(k.base, func(val pot.Val, po int) bool { + if po < depth { + return false + } + a = k.callable(val) + log.Trace(fmt.Sprintf("%x candidate nearest neighbour at %v: %v (%v)", k.BaseAddr()[:4], val.(*entry), a, po)) ppo = po - return a == nil && po >= depth + return a == nil }) if a != nil { - log.Debug(fmt.Sprintf("candidate nearest neighbour found: %v (%v)", a, ppo)) + log.Trace(fmt.Sprintf("%x candidate nearest neighbour found: %v (%v)", k.BaseAddr()[:4], a, ppo)) return a, 0, false } - log.Debug(fmt.Sprintf("no candidate nearest neighbours to connect to (Depth: %v, minProxSize: %v) %#v", depth, self.MinProxBinSize, a)) + log.Trace(fmt.Sprintf("%x no candidate nearest neighbours to connect to (Depth: %v, minProxSize: %v) %#v", k.BaseAddr()[:4], depth, k.MinProxBinSize, a)) var bpo []int prev := -1 - self.conns.EachBin(pot.NewBytesVal(self.base, nil), 0, func(po, size int, f func(func(val pot.PotVal, i int) bool) bool) bool { - log.Trace(fmt.Sprintf("check PO%02d: ", po)) + k.conns.EachBin(k.base, 0, func(po, size int, f func(func(val pot.Val, i int) bool) bool) bool { prev++ for ; prev < po; prev++ { bpo = append(bpo, prev) @@ -224,34 +227,31 @@ func (self *Kademlia) SuggestPeer() (a OverlayAddr, o int, want bool) { return size > 0 && po < depth }) // all buckets are full - // minsize == self.MinBinSize + // minsize == k.MinBinSize if len(bpo) == 0 { - log.Trace(fmt.Sprintf("all bins saturated")) + log.Debug(fmt.Sprintf("%x: all bins saturated", k.BaseAddr()[:4])) return nil, 0, false } // as long as we got candidate peers to connect to // dont ask for new peers (want = false) // try to select a candidate peer - // for i := len(bpo) - 1; i >= 0; i-- { // find the first callable peer i := 0 nxt := bpo[0] - self.addrs.EachBin(ba, nxt, func(po, size int, f func(func(pot.PotVal, int) bool) bool) bool { + k.addrs.EachBin(k.base, nxt, func(po, size int, f func(func(pot.Val, int) bool) bool) bool { // for each bin we find callable candidate peers - if po == nxt { - if i == len(bpo)-1 { - return false - } - i++ - nxt = bpo[i] + if i >= depth { + return false } - log.Trace(fmt.Sprintf("check PO%02d: ", po)) - f(func(val pot.PotVal, j int) bool { - a = self.callable(val) - if po == empty { - log.Debug(fmt.Sprintf("candidate nearest neighbour found: %v (%v)", a, ppo)) + if po == nxt { + i++ + if i < len(bpo) { + nxt = bpo[i] } - return a == nil && po <= depth + } + f(func(val pot.Val, j int) bool { + a = k.callable(val) + return a == nil && i < len(bpo) && po <= depth }) return false }) @@ -259,22 +259,17 @@ func (self *Kademlia) SuggestPeer() (a OverlayAddr, o int, want bool) { if a != nil { return a, 0, false } - return a, bpo[i], true - // cannot find a candidate, ask for more for this proximity bin specifically - // o = bpo[i] - // want = true - // // } - // return a, o, want + return a, nxt, true } // On inserts the peer as a kademlia peer into the live peers -func (self *Kademlia) On(p OverlayConn) { +func (k *Kademlia) On(p OverlayConn) { e := newEntry(p) - self.conns.Swap(p, func(v pot.PotVal) pot.PotVal { + k.conns.Swap(p, func(v pot.Val) pot.Val { // if not found live if v == nil { // insert new online peer into addrs - self.addrs.Swap(p, func(v pot.PotVal) pot.PotVal { + k.addrs.Swap(p, func(v pot.Val) pot.Val { return e }) // insert new online peer into conns @@ -289,36 +284,34 @@ func (self *Kademlia) On(p OverlayConn) { return } - depth := uint8(self.Depth()) - if depth != self.depth { - self.depth = depth + depth := uint8(k.Depth()) + if depth != k.depth { + k.depth = depth } else { depth = 0 } go np.NotifyDepth(depth) - f := func(val pot.PotVal, po int) { + f := func(val pot.Val, po int) { dp := val.(*entry).OverlayPeer.(Notifier) dp.NotifyPeer(p.Off(), uint8(po)) - // log.Trace(fmt.Sprintf("peer %v notified of %v (%v)", dp, p, po)) - log.Debug(fmt.Sprintf("peer %v notified of %v (%v)", dp, p, po)) + log.Trace(fmt.Sprintf("peer %v notified of %v (%v)", dp, p, po)) if depth > 0 { dp.NotifyDepth(depth) - log.Debug(fmt.Sprintf("peer %v notified of new depth %v", dp, depth)) - // log.Trace(fmt.Sprintf("peer %v notified of new depth %v", dp, depth)) + log.Trace(fmt.Sprintf("peer %v notified of new depth %v", dp, depth)) } } - self.conns.EachNeighbourAsync(e, 1024, 255, f, false) + k.conns.EachNeighbourAsync(e, 1024, 255, f, false) } // Off removes a peer from among live peers -func (self *Kademlia) Off(p OverlayConn) { - self.addrs.Swap(p, func(v pot.PotVal) pot.PotVal { +func (k *Kademlia) Off(p OverlayConn) { + k.addrs.Swap(p, func(v pot.Val) pot.Val { // v cannot be nil, must check otherwise we overwrite entry if v == nil { panic(fmt.Sprintf("connected peer not found %v", p)) } - self.conns.Swap(p, func(v pot.PotVal) pot.PotVal { + k.conns.Swap(p, func(_ pot.Val) pot.Val { // v cannot be nil, but no need to check return nil }) @@ -329,13 +322,12 @@ func (self *Kademlia) Off(p OverlayConn) { // EachConn is an iterator with args (base, po, f) applies f to each live peer // that has proximity order po or less as measured from the base // if base is nil, kademlia base address is used -func (self *Kademlia) EachConn(base []byte, o int, f func(OverlayConn, int, bool) bool) { +func (k *Kademlia) EachConn(base []byte, o int, f func(OverlayConn, int, bool) bool) { if len(base) == 0 { - base = self.base + base = k.base } - p := pot.NewBytesVal(base, nil) - depth := self.Depth() - self.conns.EachNeighbour(p, func(val pot.PotVal, po int) bool { + depth := k.Depth() + k.conns.EachNeighbour(base, func(val pot.Val, po int) bool { if po > o { return true } @@ -343,15 +335,14 @@ func (self *Kademlia) EachConn(base []byte, o int, f func(OverlayConn, int, bool }) } -// EachAddr(base, po, f) is an iterator applying f to each known peer +// EachAddr called with (base, po, f) is an iterator applying f to each known peer // that has proximity order po or less as measured from the base // if base is nil, kademlia base address is used -func (self *Kademlia) EachAddr(base []byte, o int, f func(OverlayAddr, int) bool) { +func (k *Kademlia) EachAddr(base []byte, o int, f func(OverlayAddr, int) bool) { if len(base) == 0 { - base = self.base + base = k.base } - p := pot.NewBytesVal(base, nil) - self.addrs.EachNeighbour(p, func(val pot.PotVal, po int) bool { + k.addrs.EachNeighbour(base, func(val pot.Val, po int) bool { if po > o { return true } @@ -362,39 +353,39 @@ func (self *Kademlia) EachAddr(base []byte, o int, f func(OverlayAddr, int) bool // Depth returns the proximity order that defines the distance of // the nearest neighbour set with cardinality >= MinProxBinSize // if there is altogether less than MinProxBinSize peers it returns 0 -func (self *Kademlia) Depth() (depth int) { - if self.conns.Size() < self.MinProxBinSize { +func (k *Kademlia) Depth() (depth int) { + if k.conns.Size() < k.MinProxBinSize { return 0 } var size int - f := func(v pot.PotVal, i int) bool { + f := func(v pot.Val, i int) bool { size++ depth = i - return size < self.MinProxBinSize + return size < k.MinProxBinSize } - self.conns.EachNeighbour(pot.NewBytesVal(self.base, nil), f) + k.conns.EachNeighbour(k.base, f) return depth } -func (self *Kademlia) callable(val pot.PotVal) OverlayAddr { +// calleble when called with val, +func (k *Kademlia) callable(val pot.Val) OverlayAddr { e := val.(*entry) // not callable if peer is live or exceeded maxRetries - if e.conn() != nil || e.retries > self.MaxRetries { + if e.conn() != nil || e.retries > k.MaxRetries { log.Trace(fmt.Sprintf("peer %v (%T) not callable", e, e.OverlayPeer)) return nil } // calculate the allowed number of retries based on time lapsed since last seen timeAgo := time.Since(e.seenAt) var retries int - for delta := int(timeAgo) / self.RetryInterval; delta > 0; delta /= self.RetryExponent { - log.Trace(fmt.Sprintf("delta: %v", delta)) + for delta := int(timeAgo) / k.RetryInterval; delta > 0; delta /= k.RetryExponent { retries++ } // this is never called concurrently, so safe to increment // peer can be retried again if retries < e.retries { - log.Trace(fmt.Sprintf("long time since last try (at %v) needed before retry %v, wait only warrants %v", timeAgo, e.retries, retries)) + log.Trace(fmt.Sprintf("%v long time since last try (at %v) needed before retry %v, wait only warrants %v", e, timeAgo, e.retries, retries)) return nil } e.retries++ @@ -404,64 +395,61 @@ func (self *Kademlia) callable(val pot.PotVal) OverlayAddr { } // BaseAddr return the kademlia base addres -func (self *Kademlia) BaseAddr() []byte { - return self.base +func (k *Kademlia) BaseAddr() []byte { + return k.base } // String returns kademlia table + kaddb table displayed with ascii -func (self *Kademlia) String() string { - +func (k *Kademlia) String() string { + wsrow := " " var rows []string rows = append(rows, "=========================================================================") - rows = append(rows, fmt.Sprintf("%v KΛÐΞMLIΛ hive: queen's address: %x", time.Now().UTC().Format(time.UnixDate), self.BaseAddr()[:3])) - rows = append(rows, fmt.Sprintf("population: %d (%d), MinProxBinSize: %d, MinBinSize: %d, MaxBinSize: %d", self.conns.Size(), self.addrs.Size(), self.MinProxBinSize, self.MinBinSize, self.MaxBinSize)) + rows = append(rows, fmt.Sprintf("%v KΛÐΞMLIΛ hive: queen's address: %x", time.Now().UTC().Format(time.UnixDate), k.BaseAddr()[:3])) + rows = append(rows, fmt.Sprintf("population: %d (%d), MinProxBinSize: %d, MinBinSize: %d, MaxBinSize: %d", k.conns.Size(), k.addrs.Size(), k.MinProxBinSize, k.MinBinSize, k.MaxBinSize)) - liverows := make([]string, self.MaxProxDisplay) - peersrows := make([]string, self.MaxProxDisplay) + liverows := make([]string, k.MaxProxDisplay) + peersrows := make([]string, k.MaxProxDisplay) var depth int prev := -1 var depthSet bool - rest := self.conns.Size() - self.conns.EachBin(pot.NewBytesVal(self.base, nil), 0, func(po, size int, f func(func(val pot.PotVal, i int) bool) bool) bool { + rest := k.conns.Size() + k.conns.EachBin(k.base, 0, func(po, size int, f func(func(val pot.Val, i int) bool) bool) bool { var rowlen int - if po >= self.MaxProxDisplay { - po = self.MaxProxDisplay - 1 + if po >= k.MaxProxDisplay { + po = k.MaxProxDisplay - 1 } row := []string{fmt.Sprintf("%2d", size)} rest -= size - f(func(val pot.PotVal, vpo int) bool { + f(func(val pot.Val, vpo int) bool { e := val.(*entry) - label := e.String()[:6] - row = append(row, fmt.Sprintf("%s (%d)", label, e.retries)) - row = append(row, val.(*entry).String()[:6]) + row = append(row, e.String()) rowlen++ return rowlen < 4 }) - if !depthSet && (po > prev+1 || rest < self.MinProxBinSize) { + if !depthSet && (po > prev+1 || rest < k.MinProxBinSize) { depthSet = true depth = prev + 1 } - for ; rowlen <= 5; rowlen++ { - row = append(row, " ") - } - liverows[po] = strings.Join(row, " ") + row = append(row, wsrow) + r := strings.Join(row, " ") + liverows[po] = r[:35] prev = po return true }) - self.addrs.EachBin(pot.NewBytesVal(self.base, nil), 0, func(po, size int, f func(func(val pot.PotVal, i int) bool) bool) bool { + k.addrs.EachBin(k.base, 0, func(po, size int, f func(func(val pot.Val, i int) bool) bool) bool { var rowlen int - if po >= self.MaxProxDisplay { - po = self.MaxProxDisplay - 1 + if po >= k.MaxProxDisplay { + po = k.MaxProxDisplay - 1 } if size < 0 { panic("wtf") } row := []string{fmt.Sprintf("%2d", size)} // we are displaying live peers too - f(func(val pot.PotVal, vpo int) bool { - row = append(row, val.(*entry).String()[:6]) + f(func(val pot.Val, vpo int) bool { + row = append(row, val.(*entry).String()) rowlen++ return rowlen < 4 }) @@ -469,7 +457,7 @@ func (self *Kademlia) String() string { return true }) - for i := 0; i < self.MaxProxDisplay; i++ { + for i := 0; i < k.MaxProxDisplay; i++ { if i == depth { rows = append(rows, fmt.Sprintf("============ DEPTH: %d ==========================================", i)) } @@ -494,15 +482,15 @@ func (self *Kademlia) String() string { // the MaxBinSize parameter it drops the oldest n peers such that // the bin is reduced to MinBinSize peers thus leaving slots to newly // connecting peers -func (self *Kademlia) Prune(c <-chan time.Time) { +func (k *Kademlia) Prune(c <-chan time.Time) { go func() { for range c { total := 0 - self.conns.EachBin(pot.NewBytesVal(self.base, nil), 0, func(po, size int, f func(func(pot.PotVal, int) bool) bool) bool { - extra := size - self.MinBinSize - if size > self.MaxBinSize { + k.conns.EachBin(k.base, 0, func(po, size int, f func(func(pot.Val, int) bool) bool) bool { + extra := size - k.MinBinSize + if size > k.MaxBinSize { n := 0 - f(func(v pot.PotVal, po int) bool { + f(func(v pot.Val, po int) bool { v.(*entry).conn().Drop(fmt.Errorf("bucket full")) n++ return n < extra @@ -511,25 +499,28 @@ func (self *Kademlia) Prune(c <-chan time.Time) { } return true }) - log.Debug(fmt.Sprintf("pruned %v peers", total)) + log.Trace(fmt.Sprintf("pruned %v peers", total)) } }() } +// NewPeerPot just creates a new pot record OverlayAddr func NewPeerPot(kadMinProxSize int, ids ...discover.NodeID) map[discover.NodeID][][]byte { // create a table of all nodes for health check - np := pot.NewPot(nil, 0) + np := pot.NewPot(nil, 0, nil) for _, id := range ids { o := ToOverlayAddr(id.Bytes()) - np, _, _ = pot.Add(np, pot.NewBytesVal(o, nil)) + np, _, _ = pot.Add(np, o) } nnmap := make(map[discover.NodeID][][]byte) for _, id := range ids { pl := 0 var nns [][]byte - np.EachNeighbour(pot.NewBytesVal(id.Bytes(), nil), func(val pot.PotVal, po int) bool { - a := val.(pot.BytesAddress).Address() + np.EachNeighbour(id.Bytes(), func(val pot.Val, po int) bool { + // a := val.(pot.BytesAddress).Address() + // nns = append(nns, a) + a := val.([]byte) nns = append(nns, a) if len(nns) >= kadMinProxSize { pl = po @@ -541,9 +532,10 @@ func NewPeerPot(kadMinProxSize int, ids ...discover.NodeID) map[discover.NodeID] return nnmap } -func (self *Kademlia) FirstEmptyBin() (i int) { +// FirstEmptyBin returns the farthest proximity order (int) that has no peer records +func (k *Kademlia) FirstEmptyBin() (i int) { i = -1 - self.conns.EachBin(pot.NewBytesVal(self.base, nil), 0, func(po, size int, f func(func(val pot.PotVal, i int) bool) bool) bool { + k.conns.EachBin(k.base, 0, func(po, size int, f func(func(val pot.Val, i int) bool) bool) bool { if po > i+1 { i = po return false @@ -554,22 +546,22 @@ func (self *Kademlia) FirstEmptyBin() (i int) { return i } -func (self *Kademlia) Full() bool { - return self.FirstEmptyBin() >= self.Depth() +// Full returns a bool if the kademlia table is healthy and complete +func (k *Kademlia) Full() bool { + return k.FirstEmptyBin() >= k.Depth() } // Healthy reports the health state of the kademlia connectivity -// -func (self *Kademlia) Healthy(peers [][]byte) bool { - return self.gotNearestNeighbours(peers) && self.Full() +func (k *Kademlia) Healthy(peers [][]byte) bool { + return k.gotNearestNeighbours(peers) && k.Full() } -func (self *Kademlia) gotNearestNeighbours(peers [][]byte) (got bool) { +func (k *Kademlia) gotNearestNeighbours(peers [][]byte) (got bool) { pm := make(map[string]bool) for _, p := range peers { pm[string(p)] = true } - self.EachConn(nil, 255, func(p OverlayConn, po int, nn bool) bool { + k.EachConn(nil, 255, func(p OverlayConn, po int, nn bool) bool { if !nn { return false } diff --git a/swarm/network/kademlia_test.go b/swarm/network/kademlia_test.go index 1d1ebfba17..a59c6370ab 100644 --- a/swarm/network/kademlia_test.go +++ b/swarm/network/kademlia_test.go @@ -28,12 +28,13 @@ import ( func init() { h := log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))) + // h := log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stderr, log.TerminalFormat(true))) // h := log.CallerFileHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(true))) log.Root().SetHandler(h) } func testKadPeerAddr(s string) *bzzAddr { - a := pot.NewHashAddress(s).Bytes() + a := pot.NewAddressFromString(s) return &bzzAddr{OAddr: a, UAddr: a} } @@ -54,7 +55,7 @@ type testPeerNotification struct { po uint8 } -type testProxNotification struct { +type testDepthNotification struct { rec string po uint8 } @@ -99,7 +100,7 @@ func newTestKademlia(b string) *testKademlia { params := NewKadParams() params.MinBinSize = 1 params.MinProxBinSize = 2 - base := pot.NewHashAddress(b).Bytes() + base := pot.NewAddressFromString(b) return &testKademlia{ NewKademlia(base, params), false, @@ -119,8 +120,7 @@ func (k *testKademlia) newTestKadPeer(s string) Peer { func (k *testKademlia) On(ons ...string) *testKademlia { for _, s := range ons { - p := k.newTestKadPeer(s) - k.Kademlia.On(p) + k.Kademlia.On(k.newTestKadPeer(s).(OverlayConn)) } return k } @@ -158,6 +158,7 @@ func testSuggestPeer(t *testing.T, k *testKademlia, expAddr string, expPo int, e if want != expWant { return fmt.Errorf("expected SuggestPeer to want peers: %v", expWant) } + // t.Logf("%v", k) return nil } @@ -165,50 +166,50 @@ func binStr(a OverlayPeer) string { if a == nil { return "" } - return pot.ToBin(a.Address())[:6] + return pot.ToBin(a.Address())[:8] } func TestSuggestPeerFindPeers(t *testing.T) { // 2 row gap, unsaturated proxbin, no callables -> want PO 0 - k := newTestKademlia("000000").On("001000") - // k.MinProxBinSize = 2 - // k.MinBinSize = 2 + k := newTestKademlia("00000000").On("00100000") err := testSuggestPeer(t, k, "", 0, true) if err != nil { t.Fatal(err.Error()) } // 2 row gap, saturated proxbin, no callables -> want PO 0 - k.On("000100") + k.On("00010000") err = testSuggestPeer(t, k, "", 0, true) if err != nil { t.Fatal(err.Error()) } // 1 row gap (1 less), saturated proxbin, no callables -> want PO 1 - k.On("100000") + k.On("10000000") err = testSuggestPeer(t, k, "", 1, true) if err != nil { t.Fatal(err.Error()) } // no gap (1 less), saturated proxbin, no callables -> do not want more - k.On("010000", "001001") + k.On("01000000", "00100001") err = testSuggestPeer(t, k, "", 0, false) if err != nil { t.Fatal(err.Error()) } // oversaturated proxbin, > do not want more - k.On("001001") + k.On("00100001") err = testSuggestPeer(t, k, "", 0, false) if err != nil { t.Fatal(err.Error()) } // reintroduce gap, disconnected peer callable - k.Off("010000") - err = testSuggestPeer(t, k, "010000", 0, false) + log.Info(k.String()) + k.Off("01000000") + log.Info(k.String()) + err = testSuggestPeer(t, k, "01000000", 0, false) if err != nil { t.Fatal(err.Error()) } @@ -221,31 +222,33 @@ func TestSuggestPeerFindPeers(t *testing.T) { } // on and off again, peer callable again - k.On("010000") - k.Off("010000") - err = testSuggestPeer(t, k, "010000", 0, false) + k.On("01000000") + k.Off("01000000") + err = testSuggestPeer(t, k, "01000000", 0, false) if err != nil { t.Fatal(err.Error()) } - k.On("010000") + k.On("01000000") // new closer peer appears, it is immediately wanted - k.Register("000101") - err = testSuggestPeer(t, k, "000101", 0, false) + k.Register("00010001") + err = testSuggestPeer(t, k, "00010001", 0, false) if err != nil { t.Fatal(err.Error()) } // PO1 disconnects - k.On("000101") - k.Off("010000") + k.On("00010001") + log.Info(k.String()) + k.Off("01000000") + log.Info(k.String()) // second time, gap filling - err = testSuggestPeer(t, k, "010000", 0, false) + err = testSuggestPeer(t, k, "01000000", 0, false) if err != nil { t.Fatal(err.Error()) } - k.On("010000") + k.On("01000000") err = testSuggestPeer(t, k, "", 0, false) if err != nil { t.Fatal(err.Error()) @@ -257,53 +260,53 @@ func TestSuggestPeerFindPeers(t *testing.T) { t.Fatal(err.Error()) } - k.Register("010001") + k.Register("01000001") err = testSuggestPeer(t, k, "", 0, true) if err != nil { t.Fatal(err.Error()) } - k.On("100001") + k.On("10000001") log.Trace("Kad:\n%v", k.String()) - err = testSuggestPeer(t, k, "010001", 0, false) + err = testSuggestPeer(t, k, "01000001", 0, false) if err != nil { t.Fatal(err.Error()) } - k.On("100001") - k.On("010001") + k.On("10000001") + k.On("01000001") err = testSuggestPeer(t, k, "", 0, false) if err != nil { t.Fatal(err.Error()) } k.MinBinSize = 3 - k.Register("100010") - err = testSuggestPeer(t, k, "100010", 0, false) + k.Register("10000010") + err = testSuggestPeer(t, k, "10000010", 0, false) if err != nil { t.Fatal(err.Error()) } - k.On("100010") + k.On("10000010") err = testSuggestPeer(t, k, "", 1, true) if err != nil { t.Fatal(err.Error()) } - k.On("010010") + k.On("01000010") err = testSuggestPeer(t, k, "", 2, true) if err != nil { t.Fatal(err.Error()) } - k.On("001010") + k.On("00100010") err = testSuggestPeer(t, k, "", 3, true) if err != nil { log.Trace("Kad:\n%v", k.String()) t.Fatal(err.Error()) } - k.On("000110") + k.On("00010010") err = testSuggestPeer(t, k, "", 0, false) if err != nil { log.Trace("Kad:\n%v", k.String()) @@ -314,14 +317,14 @@ func TestSuggestPeerFindPeers(t *testing.T) { func TestSuggestPeerRetries(t *testing.T) { // 2 row gap, unsaturated proxbin, no callables -> want PO 0 - k := newTestKademlia("000000") + k := newTestKademlia("00000000") cycle := 50 * time.Millisecond k.RetryInterval = int(cycle) k.MaxRetries = 3 k.RetryExponent = 3 - k.Register("010000") - k.On("000001", "000010") - err := testSuggestPeer(t, k, "010000", 0, false) + k.Register("01000000") + k.On("00000001", "00000010") + err := testSuggestPeer(t, k, "01000000", 0, false) if err != nil { t.Fatal(err.Error()) } @@ -333,7 +336,7 @@ func TestSuggestPeerRetries(t *testing.T) { // cycle *= time.Duration(k.RetryExponent) time.Sleep(cycle) - err = testSuggestPeer(t, k, "010000", 0, false) + err = testSuggestPeer(t, k, "01000000", 0, false) if err != nil { t.Fatal(err.Error()) } @@ -345,7 +348,7 @@ func TestSuggestPeerRetries(t *testing.T) { cycle *= time.Duration(k.RetryExponent) time.Sleep(cycle) - err = testSuggestPeer(t, k, "010000", 0, false) + err = testSuggestPeer(t, k, "01000000", 0, false) if err != nil { t.Fatal(err.Error()) } @@ -357,7 +360,7 @@ func TestSuggestPeerRetries(t *testing.T) { cycle *= time.Duration(k.RetryExponent) time.Sleep(cycle) - err = testSuggestPeer(t, k, "010000", 0, false) + err = testSuggestPeer(t, k, "01000000", 0, false) if err != nil { t.Fatal(err.Error()) } @@ -378,10 +381,10 @@ func TestSuggestPeerRetries(t *testing.T) { } func TestPruning(t *testing.T) { - k := newTestKademlia("000000") - k.On("100000", "110000", "101000", "100100", "100010") - k.On("010000", "011000", "010100", "010010", "010001") - k.On("001000", "001100", "001010", "001001") + k := newTestKademlia("00000000") + k.On("10000000", "11000000", "10100000", "10010000", "10000010") + k.On("01000000", "01100000", "01000100", "01000010", "01000001") + k.On("00100000", "00110000", "00100010", "00100001") k.MaxBinSize = 4 k.MinBinSize = 3 prune := make(chan time.Time) @@ -411,10 +414,10 @@ func TestPruning(t *testing.T) { // TODO: this is now based on just taking the first 2 peers // in order of connecting expDropped := []string{ - "101000", - "110000", - "010100", - "011000", + "10100000", + "11000000", + "01000100", + "01100000", } for _, addr := range expDropped { err := dropped[addr] @@ -428,7 +431,7 @@ func TestPruning(t *testing.T) { } func TestKademliaHiveString(t *testing.T) { - k := newTestKademlia("000000").On("010000", "001000").Register("100000", "100001") + k := newTestKademlia("00000000").On("01000000", "00100000").Register("10000000", "10000001") h := k.String() expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), MinProxBinSize: 2, MinBinSize: 1, MaxBinSize: 4\n============ PROX LIMIT: 0 ==========================================\n000 0 | 2 840000 800000\n001 1 400000 | 1 400000\n002 1 200000 | 1 200000\n003 0 | 0\n004 0 | 0\n005 0 | 0\n006 0 | 0\n007 0 | 0\n=========================================================================" if expH[100:] != h[100:] { @@ -436,7 +439,7 @@ func TestKademliaHiveString(t *testing.T) { } } -func (self *testKademlia) checkNotifications(npeers []*testPeerNotification, nprox []*testProxNotification) error { +func (self *testKademlia) checkNotifications(npeers []*testPeerNotification, nprox []*testDepthNotification) error { for _, pn := range npeers { key := pn.rec + pn.addr po, found := self.notifications[key] @@ -460,50 +463,50 @@ func (self *testKademlia) checkNotifications(npeers []*testPeerNotification, npr } func TestNotifications(t *testing.T) { - k := newTestKademlia("000000") + k := newTestKademlia("00000000") k.Discovery = true k.MinProxBinSize = 3 - k.On("010000", "001000") + k.On("01000000", "00100000") time.Sleep(1000 * time.Millisecond) err := k.checkNotifications( []*testPeerNotification{ - &testPeerNotification{"010000", "001000", 1}, + &testPeerNotification{"01000000", "00100000", 1}, }, - []*testProxNotification{ - &testProxNotification{"001000", 0}, - &testProxNotification{"010000", 0}, + []*testDepthNotification{ + &testDepthNotification{"00100000", 0}, + &testDepthNotification{"01000000", 0}, }, ) if err != nil { t.Fatal(err.Error()) } - k = k.On("100000") + k = k.On("10000000") time.Sleep(100 * time.Millisecond) k.checkNotifications( []*testPeerNotification{ - &testPeerNotification{"010000", "100000", 0}, - &testPeerNotification{"001000", "100000", 0}, + &testPeerNotification{"01000000", "10000000", 0}, + &testPeerNotification{"00100000", "10000000", 0}, }, - []*testProxNotification{ - &testProxNotification{"100000", 0}, + []*testDepthNotification{ + &testDepthNotification{"10000000", 0}, }, ) - k = k.On("010001") + k = k.On("01000001") time.Sleep(100 * time.Millisecond) k.checkNotifications( []*testPeerNotification{ - &testPeerNotification{"010000", "010001", 5}, - &testPeerNotification{"001000", "010001", 1}, - &testPeerNotification{"100000", "010001", 0}, + &testPeerNotification{"01000000", "01000001", 5}, + &testPeerNotification{"00100000", "01000001", 1}, + &testPeerNotification{"10000000", "01000001", 0}, }, - []*testProxNotification{ - &testProxNotification{"100000", 0}, - &testProxNotification{"010000", 0}, - &testProxNotification{"010001", 0}, - &testProxNotification{"001000", 0}, + []*testDepthNotification{ + &testDepthNotification{"10000000", 0}, + &testDepthNotification{"01000000", 0}, + &testDepthNotification{"01000001", 0}, + &testDepthNotification{"00100000", 0}, }, ) } diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index 0aea049426..8e9bb4f595 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -333,15 +333,15 @@ func RandomAddr() *bzzAddr { pubkey := crypto.FromECDSAPub(&key.PublicKey) var id discover.NodeID copy(id[:], pubkey[1:]) - return &bzzAddr{ - OAddr: crypto.Keccak256(pubkey[1:]), - UAddr: id[:], - } + return NewAddrFromNodeID(id) } // NewNodeIDFromAddr transforms the underlay address to an adapters.NodeID func NewNodeIDFromAddr(addr Addr) discover.NodeID { - return discover.MustBytesID(addr.Under()) + log.Info(fmt.Sprintf("uaddr=%s", string(addr.Under()))) + node := discover.MustParseNode(string(addr.Under())) + // return discover.MustBytesID(addr.Under()) + return node.ID } // NewAddrFromNodeID constucts a bzzAddr from a discover.NodeID diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go index fb038ed2a7..7bfb2aef72 100644 --- a/swarm/network/simulations/discovery/discovery_test.go +++ b/swarm/network/simulations/discovery/discovery_test.go @@ -57,7 +57,7 @@ func TestDiscoverySimulationSimAdapter(t *testing.T) { } func testDiscoverySimulation(t *testing.T, adapter adapters.NodeAdapter) { - // create 10 node network + // create network nodeCount := 10 net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{ ID: "0", diff --git a/swarm/network/simulations/overlay.go b/swarm/network/simulations/overlay.go index ede71057c6..6570d30ac0 100644 --- a/swarm/network/simulations/overlay.go +++ b/swarm/network/simulations/overlay.go @@ -209,8 +209,9 @@ func main() { config := &simulations.ServerConfig{ NewAdapter: func() adapters.NodeAdapter { return adapters.NewSimAdapter(services) }, - DefaultMockerID: "bootNet", - Mockers: mockers, + DefaultMockerID: "randomNodes", + // DefaultMockerID: "bootNet", + Mockers: mockers, } log.Info("starting simulation server on 0.0.0.0:8888...") diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index 58e6df735c..9ac35d9f47 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -133,8 +133,8 @@ func (self *Pss) Protocols() []p2p.Protocol { func (self *Pss) Run(p *p2p.Peer, rw p2p.MsgReadWriter) error { pp := protocols.NewPeer(p, rw, pssSpec) id := p.ID() - h := pot.NewHashAddressFromBytes(network.ToOverlayAddr(id[:])) - self.fwdPool[h.Address] = pp + a := pot.NewAddressFromBytes(network.ToOverlayAddr(id[:])) + self.fwdPool[a] = pp return pp.Run(self.handlePssMsg) } @@ -315,9 +315,8 @@ func (self *Pss) Forward(msg *PssMsg) error { // send with kademlia // find the closest peer to the recipient and attempt to send self.Overlay.EachConn(msg.To, 256, func(op network.OverlayConn, po int, isproxbin bool) bool { - //p, ok := op.(senderPeer) - h := pot.NewHashAddressFromBytes(op.Address()) - pp := self.fwdPool[h.Address] + a := pot.NewAddressFromBytes(op.Address()) + pp := self.fwdPool[a] addr := self.Overlay.BaseAddr() sendMsg := fmt.Sprintf("%x: msg to %x via %x", common.ByteLabel(addr), common.ByteLabel(msg.To), common.ByteLabel(op.Address())) if self.checkFwdCache(op.Address(), digest) { @@ -459,7 +458,7 @@ func RegisterPssProtocol(ps *Pss, topic *Topic, spec *protocols.Spec, targetprot } func (self *PssProtocol) handle(msg []byte, p *p2p.Peer, senderAddr []byte) error { - hashoaddr := pot.NewHashAddressFromBytes(senderAddr).Address + hashoaddr := pot.NewAddressFromBytes(senderAddr) if !self.isActive(hashoaddr, *self.topic) { rw := &PssReadWriter{ Pss: self.Pss, diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index 875771b463..3ee8af3116 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -199,8 +199,8 @@ func TestSimpleLinear(t *testing.T) { Peer: pp, addr: network.ToOverlayAddr(id[:]), } - h := pot.NewHashAddressFromBytes(bp.addr) - ps.fwdPool[h.Address] = pp + a := pot.NewAddressFromBytes(bp.addr) + ps.fwdPool[a] = pp ps.Overlay.On(bp) defer ps.Overlay.Off(bp) log.Debug(fmt.Sprintf("%v", ps.Overlay)) From 26d7ceee1f33f40bc869f53f7ae9110834d51e9b Mon Sep 17 00:00:00 2001 From: nolash Date: Fri, 9 Jun 2017 01:13:43 +0200 Subject: [PATCH 03/17] swarm, swarm/network, swarm/pss: debug addr types + pss over devp2p --- swarm/network/protocol.go | 37 ++- swarm/pss/client/client.go | 4 + swarm/pss/protocols/chat/protocol.go | 98 ++++++ swarm/pss/pss.go | 43 ++- swarm/pss/pss_test.go | 4 +- swarm/swarm.go | 18 +- swarm/swarm_psschat.go | 428 +++++++++++++++++++++++++++ 7 files changed, 606 insertions(+), 26 deletions(-) create mode 100644 swarm/pss/protocols/chat/protocol.go create mode 100644 swarm/swarm_psschat.go diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index 8e9bb4f595..8243729100 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -23,6 +23,7 @@ import ( "net" "sync" "time" + "strings" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" @@ -113,6 +114,14 @@ func NewBzz(config *BzzConfig, kad Overlay, store StateStore) *Bzz { } } +func (b *Bzz) UpdateLocalAddr(byteaddr []byte) *bzzAddr { + b.localAddr.Update(&bzzAddr{ + UAddr: byteaddr, + OAddr: b.localAddr.OAddr, + }) + return b.localAddr +} + // Bzz implements the node.Service interface, offers Protocols // * handshake/hive // * discovery @@ -221,12 +230,12 @@ type bzzPeer struct { lastActive time.Time // time is updated whenever mutexes are releasing } -func newBzzPeer(p *protocols.Peer, over, under []byte) *bzzPeer { - return &bzzPeer{ - Peer: p, - localAddr: &bzzAddr{over, under}, - } -} +//func newBzzPeer(p *protocols.Peer, over, under []byte) *bzzPeer { +// return &bzzPeer{ +// Peer: p, +// localAddr: &bzzAddr{over, under}, +// } +//} // Off returns the overlay peer record for offline persistance func (self *bzzPeer) Off() OverlayAddr { @@ -283,6 +292,22 @@ func (self *bzzHandshake) Perform(p *p2p.Peer, rw p2p.MsgReadWriter) (err error) return fmt.Errorf("version mismatch %d (!= %d)", rhs.Version, self.Version) } self.peerAddr = rhs.Addr + peerid := p.ID() + log.Warn( + strings.Join( + []string{ + "handshake ok:", + fmt.Sprintf("p2p.Peer.ID(): ...... %v", p.ID()), + fmt.Sprintf("peerAddr.UAddr: ...... %x", self.peerAddr.UAddr[:32]), + fmt.Sprintf("peerAddr.OAddr: ...... %x", self.peerAddr.OAddr[:32]), + fmt.Sprintf("selfAddr.UAddr: ...... %x", self.Addr.UAddr[:32]), + fmt.Sprintf("selfAddr.OAddr: ...... %x", self.Addr.OAddr[:32]), + fmt.Sprintf("ToOverlayAddr(p2p.Peer.ID): .... %x", ToOverlayAddr(peerid[:])), + fmt.Sprintf("ToOverlayAddr(peerAddr.UAddr): .. %x", ToOverlayAddr(self.peerAddr.UAddr)), + }, "\n", + ), + ) + return nil } diff --git a/swarm/pss/client/client.go b/swarm/pss/client/client.go index a9bef6d3db..335316d3b2 100644 --- a/swarm/pss/client/client.go +++ b/swarm/pss/client/client.go @@ -207,6 +207,10 @@ func (self *Client) Stop() error { func (self *Client) AddPssPeer(addr pot.Address, spec *protocols.Spec) { topic := pss.NewTopic(spec.Name, int(spec.Version)) + if self.peerPool[topic] == nil { + log.Error("addpeer on unset topic") + return + } if self.peerPool[topic][addr] == nil { self.peerPool[topic][addr] = self.newpssRPCRW(addr, &topic) nid, _ := discover.HexID("0x00") diff --git a/swarm/pss/protocols/chat/protocol.go b/swarm/pss/protocols/chat/protocol.go new file mode 100644 index 0000000000..a6badc66e8 --- /dev/null +++ b/swarm/pss/protocols/chat/protocol.go @@ -0,0 +1,98 @@ +package chat + +import ( + "time" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/protocols" + "github.com/ethereum/go-ethereum/swarm/pss" + "github.com/ethereum/go-ethereum/swarm/network" +) + +const ( + ESendFail = iota +) + +var ( + chatConnString = map[int]string{ + ESendFail: "Send error", + } +) + +type ChatMsg struct { + Serial uint64 + Content []byte + Source string +} + +type chatPing struct { + Created time.Time + Pong bool +} + +type chatAck struct { + Seen time.Time + Serial uint64 +} + +var ChatProtocol = &protocols.Spec{ + Name: "pssChat", + Version: 1, + MaxMsgSize: 1024, + Messages: []interface{}{ + ChatMsg{}, chatPing{}, chatAck{}, + }, +} + +var ChatTopic = pss.NewTopic(ChatProtocol.Name, int(ChatProtocol.Version)) + +type ChatConn struct { + Addr []byte + E int +} + +func (c* ChatConn) Error() string { + return chatConnString[c.E] +} + +type ChatCtrl struct { + Peer *protocols.Peer + OutC chan interface{} + ConnC chan ChatConn + inC chan *ChatMsg + oAddr []byte + pingTX int + pingRX int + pingLast int +} + +func (self *ChatCtrl) chatHandler(msg interface{}) error { + Chatmsg, ok := msg.(*ChatMsg) + if ok { + if self.inC != nil { + self.inC <- Chatmsg + } + } + return nil +} + +func New(inC chan *ChatMsg, connC chan ChatConn, injectfunc func(*ChatCtrl)) *p2p.Protocol { +//func New(inC chan *ChatMsg, outC chan interface{}, connC chan ChatConn) *p2p.Protocol { + chatctrl := &ChatCtrl{ + inC: inC, + ConnC: connC, + } + return &p2p.Protocol{ + Name: ChatProtocol.Name, + Version: ChatProtocol.Version, + Length: 3, + Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error { + peerid := p.ID() + pp := protocols.NewPeer(p, rw, ChatProtocol) + chatctrl.Peer = pp + chatctrl.oAddr = network.ToOverlayAddr(peerid[:]) + injectfunc(chatctrl) + pp.Run(chatctrl.chatHandler) + return nil + }, + } +} diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index 9ac35d9f47..2f82c9c42b 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -32,6 +32,7 @@ var ( ) type senderPeer interface { + ID() discover.NodeID Address() []byte Send(interface{}) error } @@ -69,7 +70,8 @@ type pssDigest [digestLength]byte type Pss struct { network.Overlay // we can get the overlayaddress from this peerPool map[pot.Address]map[Topic]p2p.MsgReadWriter // keep track of all virtual p2p.Peers we are currently speaking to - fwdPool map[pot.Address]*protocols.Peer // keep track of all peers sitting on the pssmsg routing layer + //fwdPool map[pot.Address]*protocols.Peer // keep track of all peers sitting on the pssmsg routing layer + fwdPool map[discover.NodeID]*protocols.Peer // keep track of all peers sitting on the pssmsg routing layer handlers map[Topic]map[*Handler]bool // topic and version based pss payload handlers fwdcache map[pssDigest]pssCacheEntry // checksum of unique fields from pssmsg mapped to expiry, cache to determine whether to drop msg cachettl time.Duration // how long to keep messages in fwdcache @@ -98,7 +100,8 @@ func NewPss(k network.Overlay, dpa *storage.DPA, params *PssParams) *Pss { return &Pss{ Overlay: k, peerPool: make(map[pot.Address]map[Topic]p2p.MsgReadWriter, PssPeerCapacity), - fwdPool: make(map[pot.Address]*protocols.Peer), + //fwdPool: make(map[pot.Address]*protocols.Peer), + fwdPool: make(map[discover.NodeID]*protocols.Peer), handlers: make(map[Topic]map[*Handler]bool), fwdcache: make(map[pssDigest]pssCacheEntry), cachettl: params.Cachettl, @@ -132,9 +135,11 @@ func (self *Pss) Protocols() []p2p.Protocol { func (self *Pss) Run(p *p2p.Peer, rw p2p.MsgReadWriter) error { pp := protocols.NewPeer(p, rw, pssSpec) - id := p.ID() - a := pot.NewAddressFromBytes(network.ToOverlayAddr(id[:])) - self.fwdPool[a] = pp + //addr := network.NewAddrFromNodeID(id) + //potaddr := pot.NewHashAddressFromBytes(addr.OAddr) + //self.fwdPool[potaddr.Address] = pp + self.fwdPool[p.ID()] = pp + return pp.Run(self.handlePssMsg) } @@ -315,15 +320,21 @@ func (self *Pss) Forward(msg *PssMsg) error { // send with kademlia // find the closest peer to the recipient and attempt to send self.Overlay.EachConn(msg.To, 256, func(op network.OverlayConn, po int, isproxbin bool) bool { - a := pot.NewAddressFromBytes(op.Address()) - pp := self.fwdPool[a] - addr := self.Overlay.BaseAddr() - sendMsg := fmt.Sprintf("%x: msg to %x via %x", common.ByteLabel(addr), common.ByteLabel(msg.To), common.ByteLabel(op.Address())) + sp, ok := op.(senderPeer) + if !ok { + log.Crit("Pss cannot use kademlia peer type") + return false + } + sendMsg := fmt.Sprintf("TO %x FROM %x VIA %x", common.ByteLabel(msg.To), common.ByteLabel(self.BaseAddr()), common.ByteLabel(op.Address())) + //h := pot.NewHashAddressFromBytes(op.Address()) + //pp := self.fwdPool[h.Address] + pp := self.fwdPool[sp.ID()] if self.checkFwdCache(op.Address(), digest) { - log.Info(fmt.Sprintf("%v: peer already forwarded to", sendMsg)) + log.Info("%v: peer already forwarded to", sendMsg) return true } err := pp.Send(msg) + //err := sp.Send(msg) if err != nil { log.Warn(fmt.Sprintf("%v: failed forwarding: %v", sendMsg, err)) return true @@ -416,7 +427,7 @@ func (prw PssReadWriter) ReadMsg() (p2p.Msg, error) { // Implements p2p.MsgWriter func (prw PssReadWriter) WriteMsg(msg p2p.Msg) error { - log.Warn("got writemsg pssclient", "msg", msg) + log.Trace("pssrw writemsg", "msg", msg) rlpdata := make([]byte, msg.Size) msg.Payload.Read(rlpdata) pmsg, err := rlp.EncodeToBytes(ProtocolMsg{ @@ -446,19 +457,18 @@ type PssProtocol struct { } // Constructor -func RegisterPssProtocol(ps *Pss, topic *Topic, spec *protocols.Spec, targetprotocol *p2p.Protocol) error { +func RegisterPssProtocol(ps *Pss, topic *Topic, spec *protocols.Spec, targetprotocol *p2p.Protocol) *PssProtocol { pp := &PssProtocol{ Pss: ps, proto: targetprotocol, topic: topic, spec: spec, } - ps.Register(topic, pp.handle) - return nil + return pp } -func (self *PssProtocol) handle(msg []byte, p *p2p.Peer, senderAddr []byte) error { - hashoaddr := pot.NewAddressFromBytes(senderAddr) +func (self *PssProtocol) Handle(msg []byte, p *p2p.Peer, senderAddr []byte) error { + hashoaddr := pot.NewHashAddressFromBytes(senderAddr).Address if !self.isActive(hashoaddr, *self.topic) { rw := &PssReadWriter{ Pss: self.Pss, @@ -480,3 +490,4 @@ func (self *PssProtocol) handle(msg []byte, p *p2p.Peer, senderAddr []byte) erro return nil } + diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index 3ee8af3116..3702868ebc 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -187,7 +187,7 @@ func TestSimpleLinear(t *testing.T) { C: make(chan struct{}), } - err = RegisterPssProtocol(ps, &PingTopic, PingProtocol, NewPingProtocol(ping.PingHandler)) + ps.Register(&PingTopic, RegisterPssProtocol(ps, &PingTopic, PingProtocol, NewPingProtocol(ping.PingHandler)).Handle) if err != nil { t.Fatalf("Failed to register virtual protocol in pss: %v", err) @@ -543,7 +543,7 @@ func newServices() adapters.Services { ping := &Ping{ C: make(chan struct{}), } - err = RegisterPssProtocol(ps, &PingTopic, PingProtocol, NewPingProtocol(ping.PingHandler)) + ps.Register(&PingTopic, RegisterPssProtocol(ps, &PingTopic, PingProtocol, NewPingProtocol(ping.PingHandler)).Handle) if err != nil { log.Error("Couldnt register pss protocol", "err", err) os.Exit(1) diff --git a/swarm/swarm.go b/swarm/swarm.go index 09260c634f..71334ba661 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -1,3 +1,5 @@ +// +build !psschat + // Copyright 2016 The go-ethereum Authors // This file is part of the go-ethereum library. // @@ -21,6 +23,7 @@ import ( "context" "crypto/ecdsa" "fmt" + "io/ioutil" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" @@ -106,6 +109,7 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api. kp, ) + config.HiveParams.Discovery = true bzzconfig := &network.BzzConfig{ UnderlayAddr: common.FromHex(config.PublicKey), OverlayAddr: common.FromHex(config.BzzKey), @@ -129,10 +133,20 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api. dpaChunkStore := storage.NewDpaChunkStore(self.lstore, self.storage) log.Debug(fmt.Sprintf("-> Local Access to Swarm")) - // Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage + // Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage self.dpa = storage.NewDPA(dpaChunkStore, self.config.ChunkerParams) log.Debug(fmt.Sprintf("-> Content Store API")) + // netstore is broken, temp workaround to fiddle with pss + cachedir, err := ioutil.TempDir("", "bzz-tmp") + if err != nil { + return nil, err + } + self.dpa, err = storage.NewLocalDPA(cachedir) + if err != nil { + return nil, err + } + // Pss = postal service over swarm (devp2p over bzz) if pssEnabled { pssparams := pss.NewPssParams(false) @@ -195,7 +209,7 @@ func (self *Swarm) Start(net *p2p.Server) error { log.Error("bzz failed", "err", err) return err } - log.Info(fmt.Sprintf("Swarm network started on bzz address: %v", self.bzz.Hive.Overlay.BaseAddr())) + log.Info(fmt.Sprintf("Swarm network started on bzz address: %x", self.bzz.Hive.Overlay.BaseAddr())) if self.pss != nil { self.pss.Start(net) diff --git a/swarm/swarm_psschat.go b/swarm/swarm_psschat.go new file mode 100644 index 0000000000..18d31b21c8 --- /dev/null +++ b/swarm/swarm_psschat.go @@ -0,0 +1,428 @@ +// +build psschat + +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package swarm + +import ( + "bytes" + "context" + "crypto/ecdsa" + "fmt" + "io/ioutil" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/contracts/chequebook" + "github.com/ethereum/go-ethereum/contracts/ens" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/swarm/api" + httpapi "github.com/ethereum/go-ethereum/swarm/api/http" + "github.com/ethereum/go-ethereum/swarm/fuse" + "github.com/ethereum/go-ethereum/swarm/network" + "github.com/ethereum/go-ethereum/swarm/pss" + "github.com/ethereum/go-ethereum/swarm/storage" + psschat "github.com/ethereum/go-ethereum/swarm/pss/protocols/chat" +) + +// the swarm stack +type Swarm struct { + config *api.Config // swarm configuration + api *api.Api // high level api layer (fs/manifest) + dns api.Resolver // DNS registrar + storage storage.ChunkStore // internal access to storage, common interface to cloud storage backends + dpa *storage.DPA // distributed preimage archive, the local API to the storage with document level storage/retrieval support + cloud storage.CloudStore // procurement, cloud storage backend (can multi-cloud) + //hive *network.Hive // the logistic manager + bzz *network.Bzz // hive and bzz protocol + backend chequebook.Backend // simple blockchain Backend + privateKey *ecdsa.PrivateKey + corsString string + swapEnabled bool + lstore *storage.LocalStore // local store, needs to store for releasing resources after node stopped + sfs *fuse.SwarmFS // need this to cleanup all the active mounts on node exit + pss *pss.Pss +} + +type SwarmAPI struct { + Api *api.Api + Backend chequebook.Backend + PrvKey *ecdsa.PrivateKey +} + +func (self *Swarm) API() *SwarmAPI { + return &SwarmAPI{ + Api: self.api, + Backend: self.backend, + PrvKey: self.privateKey, + } +} + +// creates a new swarm service instance +// implements node.Service +func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api.Config, swapEnabled, syncEnabled bool, cors string, pssEnabled bool) (self *Swarm, err error) { + if bytes.Equal(common.FromHex(config.PublicKey), storage.ZeroKey) { + return nil, fmt.Errorf("empty public key") + } + if bytes.Equal(common.FromHex(config.BzzKey), storage.ZeroKey) { + return nil, fmt.Errorf("empty bzz key") + } + + self = &Swarm{ + config: config, + swapEnabled: swapEnabled, + backend: backend, + privateKey: config.Swap.PrivateKey(), + corsString: cors, + } + log.Debug(fmt.Sprintf("Setting up Swarm service components")) + + hash := storage.MakeHashFunc(config.ChunkerParams.Hash) + self.lstore, err = storage.NewLocalStore(hash, config.StoreParams) + if err != nil { + return + } + + log.Debug("Set up local db access (iterator/counter)") + + kp := network.NewKadParams() + to := network.NewKademlia( + common.FromHex(config.BzzKey), + kp, + ) + + config.HiveParams.Discovery = true + + nodeid := discover.PubkeyID(crypto.ToECDSAPub(common.FromHex(config.PublicKey))) + addr := network.NewAddrFromNodeID(nodeid) + bzzconfig := &network.BzzConfig{ + OverlayAddr: common.FromHex(config.BzzKey), + UnderlayAddr: addr.UAddr, + HiveParams: config.HiveParams, + } + self.bzz = network.NewBzz(bzzconfig, to, nil) + + // set up the kademlia hive + // self.hive = network.NewHive( + // config.HiveParams, // configuration parameters + // self.Kademlia, + // stateStore, + // ) + // log.Debug(fmt.Sprintf("Set up swarm network with Kademlia hive")) + // + // setup cloud storage internal access layer + self.storage = storage.NewNetStore(hash, self.lstore, nil, config.StoreParams) + log.Debug("-> swarm net store shared access layer to Swarm Chunk Store") + + // set up DPA, the cloud storage local access layer + dpaChunkStore := storage.NewDpaChunkStore(self.lstore, self.storage) + log.Debug(fmt.Sprintf("-> Local Access to Swarm")) + + // Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage + self.dpa = storage.NewDPA(dpaChunkStore, self.config.ChunkerParams) + log.Debug(fmt.Sprintf("-> Content Store API")) + + // netstore is broken, temp workaround to fiddle with pss + cachedir, err := ioutil.TempDir("", "bzz-tmp") + if err != nil { + return nil, err + } + self.dpa, err = storage.NewLocalDPA(cachedir) + if err != nil { + return nil, err + } + + // Pss = postal service over swarm (devp2p over bzz) + if pssEnabled { + pssparams := pss.NewPssParams(false) + self.pss = pss.NewPss(to, self.dpa, pssparams) + } + + // set up high level api + transactOpts := bind.NewKeyedTransactor(self.privateKey) + + if backend == (*ethclient.Client)(nil) { + log.Warn("No ENS, please specify non-empty --ethapi to use domain name resolution") + } else { + self.dns, err = ens.NewENS(transactOpts, config.EnsRoot, self.backend) + if err != nil { + return nil, err + } + } + log.Debug(fmt.Sprintf("-> Swarm Domain Name Registrar @ address %v", config.EnsRoot.Hex())) + + self.api = api.NewApi(self.dpa, self.dns) + + // Manifests for Smart Hosting + log.Debug(fmt.Sprintf("-> Web3 virtual server API")) + + self.sfs = fuse.NewSwarmFS(self.api) + log.Debug("-> Initializing Fuse file system") + + return self, nil +} + +/* +Start is called when the stack is started +* starts the network kademlia hive peer management +* (starts netStore level 0 api) +* starts DPA level 1 api (chunking -> store/retrieve requests) +* (starts level 2 api) +* starts http proxy server +* registers url scheme handlers for bzz, etc +* TODO: start subservices like sword, swear, swarmdns +*/ +// implements the node.Service interface +func (self *Swarm) Start(net *p2p.Server) error { + + // update uaddr to correct enode + newaddr := self.bzz.UpdateLocalAddr([]byte(net.Self().String())) + log.Warn("Updated bzz local addr", "oaddr", fmt.Sprintf("%s", newaddr.OAddr), "uaddr", fmt.Sprintf("%x", newaddr.UAddr)) + + // set chequebook + if self.swapEnabled { + ctx := context.Background() // The initial setup has no deadline. + err := self.SetChequebook(ctx) + if err != nil { + return fmt.Errorf("Unable to set chequebook for SWAP: %v", err) + } + log.Debug(fmt.Sprintf("-> cheque book for SWAP: %v", self.config.Swap.Chequebook())) + } else { + log.Debug(fmt.Sprintf("SWAP disabled: no cheque book set")) + } + + log.Warn(fmt.Sprintf("Starting Swarm service")) + + // self.hive.Start(net) + err := self.bzz.Start(net) + if err != nil { + log.Error("bzz failed", "err", err) + return err + } + log.Info(fmt.Sprintf("Swarm network started on bzz address: %x", self.bzz.Hive.Overlay.BaseAddr())) + + if self.pss != nil { + self.pss.Start(net) + log.Info("Pss started") + pss.RegisterPssProtocol(self.pss, &psschat.ChatTopic, psschat.ChatProtocol, psschat.New(nil, nil, func(ctrl *psschat.ChatCtrl) { + if ctrl.OutC != nil { + go func() { + for { + select { + case msg := <-ctrl.OutC: + err := ctrl.Peer.Send(msg) + if err != nil { + // do something with ctrl.ConnC here + //pp.Drop(err) + //return + } + } + } + }() + } + + })) + } + + self.dpa.Start() + log.Debug(fmt.Sprintf("Swarm DPA started")) + + // start swarm http proxy server + if self.config.Port != "" { + addr := ":" + self.config.Port + go httpapi.StartHttpServer(self.api, &httpapi.ServerConfig{ + Addr: addr, + CorsString: self.corsString, + }) + } + + log.Debug(fmt.Sprintf("Swarm http proxy started on port: %v", self.config.Port)) + + if self.corsString != "" { + log.Debug(fmt.Sprintf("Swarm http proxy started with corsdomain: %v", self.corsString)) + } + + return nil +} + +// implements the node.Service interface +// stops all component services. +func (self *Swarm) Stop() error { + self.dpa.Stop() + //self.hive.Stop() + self.bzz.Stop() + //if self.pss != nil { + // self.pss.Stop() + //} + if ch := self.config.Swap.Chequebook(); ch != nil { + ch.Stop() + ch.Save() + } + + if self.lstore != nil { + self.lstore.DbStore.Close() + } + self.sfs.Stop() + return self.config.Save() +} + +// implements the node.Service interface +func (self *Swarm) Protocols() (protos []p2p.Protocol) { + + for _, p := range self.bzz.Protocols() { + protos = append(protos, p) + } + + if self.pss != nil { + log.Warn("adding pss protos") + for _, p := range self.pss.Protocols() { + protos = append(protos, p) + } + } + // ct := network.BzzCodeMap() + // ct.Register(2, network.DiscoveryMsgs...) + + // proto := network.Bzz( + // self.hive.Overlay.GetAddr().Over(), + // self.hive.Overlay.GetAddr().Under(), + // ct, + // nil, + // nil, + // nil, + // ) + // + return +} + +// implements node.Service +// Apis returns the RPC Api descriptors the Swarm implementation offers +func (self *Swarm) APIs() []rpc.API { + + apis := []rpc.API{ + // public APIs + { + Namespace: "bzz", + Version: "0.1", + Service: &Info{self.config, chequebook.ContractParams}, + Public: true, + }, + // admin APIs + { + Namespace: "bzz", + Version: "0.1", + Service: api.NewControl(self.api, self.bzz.Hive), + Public: false, + }, + { + Namespace: "chequebook", + Version: chequebook.Version, + Service: chequebook.NewApi(self.config.Swap.Chequebook), + Public: false, + }, + { + Namespace: "swarmfs", + Version: fuse.Swarmfs_Version, + Service: self.sfs, + Public: false, + }, + // storage APIs + // DEPRECATED: Use the HTTP API instead + { + Namespace: "bzz", + Version: "0.1", + Service: api.NewStorage(self.api), + Public: true, + }, + { + Namespace: "bzz", + Version: "0.1", + Service: api.NewFileSystem(self.api), + Public: false, + }, + // {Namespace, Version, api.NewAdmin(self), false}, + } + + for _, api := range self.bzz.APIs() { + apis = append(apis, api) + } + + if self.pss != nil { + for _, api := range self.pss.APIs() { + apis = append(apis, api) + } + } + + return apis +} + +func (self *Swarm) Api() *api.Api { + return self.api +} + +// SetChequebook ensures that the local checquebook is set up on chain. +func (self *Swarm) SetChequebook(ctx context.Context) error { + err := self.config.Swap.SetChequebook(ctx, self.backend, self.config.Path) + if err != nil { + return err + } + log.Info(fmt.Sprintf("new chequebook set (%v): saving config file, resetting all connections in the hive", self.config.Swap.Contract.Hex())) + self.config.Save() + return nil +} + +// Local swarm without netStore +func NewLocalSwarm(datadir, port string) (self *Swarm, err error) { + + prvKey, err := crypto.GenerateKey() + if err != nil { + return + } + + config, err := api.NewConfig(datadir, common.Address{}, prvKey, network.NetworkId) + if err != nil { + return + } + config.Port = port + + dpa, err := storage.NewLocalDPA(datadir) + if err != nil { + return + } + + self = &Swarm{ + api: api.NewApi(dpa, nil), + config: config, + } + + return +} + +// serialisable info about swarm +type Info struct { + *api.Config + *chequebook.Params +} + +func (self *Info) Info() *Info { + return self +} From dca2fd0f0ffaef2760ff1800bc08b97cfc058d12 Mon Sep 17 00:00:00 2001 From: nolash Date: Fri, 9 Jun 2017 20:54:58 +0200 Subject: [PATCH 04/17] swarm/pss: newhashaddressfrombytes change --- swarm/pss/pss.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index 2f82c9c42b..fe03f7a0ed 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -468,7 +468,8 @@ func RegisterPssProtocol(ps *Pss, topic *Topic, spec *protocols.Spec, targetprot } func (self *PssProtocol) Handle(msg []byte, p *p2p.Peer, senderAddr []byte) error { - hashoaddr := pot.NewHashAddressFromBytes(senderAddr).Address + //hashoaddr := pot.NewHashAddressFromBytes(senderAddr).Address + hashoaddr := pot.NewAddressFromBytes(senderAddr) if !self.isActive(hashoaddr, *self.topic) { rw := &PssReadWriter{ Pss: self.Pss, From d3f689a01e0addfbcd7b5af4a306147d70dd06dc Mon Sep 17 00:00:00 2001 From: zelig Date: Mon, 12 Jun 2017 13:58:41 +0200 Subject: [PATCH 05/17] potfix --- pot/pot.go | 412 ++++++++++++++++----------------- pot/pot_test.go | 151 ++++++------ swarm/network/hive_test.go | 1 + swarm/network/kademlia.go | 134 +++++++---- swarm/network/kademlia_test.go | 7 +- 5 files changed, 370 insertions(+), 335 deletions(-) diff --git a/pot/pot.go b/pot/pot.go index 2ebce82cec..155b26ef91 100644 --- a/pot/pot.go +++ b/pot/pot.go @@ -28,21 +28,20 @@ const ( ) // Pot is the root node type, allows locked non-applicative manipulation -type Pot struct { - lock sync.RWMutex - *pot -} +// type Pot struct { +// lock sync.RWMutex +// *Pot +// } -// pot is the node type (same for root, branching node and leaf) -type pot struct { +// Pot is the node type (same for root, branching node and leaf) +type Pot struct { pin Val - bins []*pot + bins []*Pot size int po int - pof Pof } -// Val is the element type for pots +// Val is the element type for Pots type Val interface{} // Pof is the proximity order function @@ -51,21 +50,15 @@ type Pof func(Val, Val, int) (int, bool) // NewPot constructor. Requires value of type Val to pin // and po to point to a span in the Val key // The pinned item counts towards the size -func NewPot(v Val, po int, pof Pof) *Pot { +func NewPot(v Val, po int) *Pot { var size int if v != nil { size++ } - if pof == nil { - pof = DefaultPof(keylen) - } return &Pot{ - pot: &pot{ - pin: v, - po: po, - size: size, - pof: pof, - }, + pin: v, + po: po, + size: size, } } @@ -76,67 +69,71 @@ func (t *Pot) Pin() Val { // Size returns the number of values in the Pot func (t *Pot) Size() int { - t.lock.RLock() - defer t.lock.RUnlock() + // t.lock.RLock() + // defer t.lock.RUnlock() + if t == nil { + return 0 + } return t.size } // Add inserts v into the Pot and // returns the proximity order of v and a boolean // indicating if the item was found -// Add locks the Pot while using applicative add on its pot -func (t *Pot) Add(val Val) (po int, found bool) { - t.lock.Lock() - defer t.lock.Unlock() - t.pot, po, found = add(t.pot, val) - return po, found -} +// Add locks the Pot while using applicative add on its Pot +// func (t *Pot) Add(val Val) (po int, found bool) { +// // t.lock.Lock() +// // defer t.lock.Unlock() +// t.Pot, po, found = add(t.Pot, val) +// return po, found +// } // Add called on (t, v) returns a new Pot that contains all the elements of t // plus the value v, using the applicative add // the second return value is the proximity order of the inserted element // the third is boolean indicating if the item was found -// it only readlocks the Pot while reading its pot -func Add(t *Pot, val Val) (*Pot, int, bool) { - t.lock.RLock() - n := t.pot - t.lock.RUnlock() - r, po, found := add(n, val) - return &Pot{pot: r}, po, found +// it only readlocks the Pot while reading its Pot +func Add(t *Pot, val Val, pof Pof) (*Pot, int, bool) { + // // t.lock.RLock() + // n := t.Pot + // // t.lock.RUnlock() + // r, po, found := add(n, val) + // return &Pot{Pot: r}, po, found + // // return &Pot{Pot: r}, po, found + return add(t, val, pof) } -func (t *pot) clone() *pot { - return &pot{ +func (t *Pot) clone() *Pot { + return &Pot{ pin: t.pin, size: t.size, po: t.po, bins: t.bins, - pof: t.pof, } } -func add(t *pot, val Val) (*pot, int, bool) { - var r *pot +func add(t *Pot, val Val, pof Pof) (*Pot, int, bool) { + var r *Pot if t == nil || t.pin == nil { r = t.clone() r.pin = val r.size++ return r, 0, false } - po, found := t.pof(t.pin, val, t.po) + po, found := pof(t.pin, val, t.po) if found { r = t.clone() r.pin = val return r, po, true } - var p *pot + var p *Pot var i, j int size := t.size for i < len(t.bins) { n := t.bins[i] if n.po == po { - p, _, found = add(n, val) + p, _, found = add(n, val, pof) if !found { size++ } @@ -151,23 +148,21 @@ func add(t *pot, val Val) (*pot, int, bool) { } if p == nil { size++ - p = &pot{ + p = &Pot{ pin: val, size: 1, po: po, - pof: t.pof, } } - bins := append([]*pot{}, t.bins[:i]...) + bins := append([]*Pot{}, t.bins[:i]...) bins = append(bins, p) bins = append(bins, t.bins[j:]...) - r = &pot{ + r = &Pot{ pin: t.pin, size: size, po: t.po, bins: bins, - pof: t.pof, } return r, po, found @@ -176,57 +171,56 @@ func add(t *pot, val Val) (*pot, int, bool) { // Remove called on (v) deletes v from the Pot and returns // the proximity order of v and a boolean value indicating // if the value was found -// Remove locks Pot while using applicative remove on its pot -func (t *Pot) Remove(val Val) (po int, found bool) { - t.lock.Lock() - defer t.lock.Unlock() - t.pot, po, found = remove(t.pot, val) - return po, found -} +// Remove locks Pot while using applicative remove on its Pot +// func (t *Pot) Remove(val Val) (po int, found bool) { +// // t.lock.Lock() +// // defer t.lock.Unlock() +// t.Pot, po, found = remove(t.Pot, val) +// return po, found +// } // Remove called on (t, v) returns a new Pot that contains all the elements of t // minus the value v, using the applicative remove // the second return value is the proximity order of the inserted element // the third is boolean indicating if the item was found -// it only readlocks the Pot while reading its pot -func Remove(t *Pot, v Val) (*Pot, int, bool) { - t.lock.RLock() - n := t.pot - t.lock.RUnlock() - r, po, found := remove(n, v) - return &Pot{pot: r}, po, found +// it only readlocks the Pot while reading its Pot +func Remove(t *Pot, v Val, pof Pof) (*Pot, int, bool) { + // // t.lock.RLock() + // n := t.Pot + // // t.lock.RUnlock() + // r, po, found := remove(n, v) + // return &Pot{Pot: r}, po, found + return remove(t, v, pof) } -func remove(t *pot, val Val) (r *pot, po int, found bool) { +func remove(t *Pot, val Val, pof Pof) (r *Pot, po int, found bool) { size := t.size - po, found = t.pof(t.pin, val, t.po) + po, found = pof(t.pin, val, t.po) if found { size-- if size == 0 { - r = &pot{ - po: t.po, - pof: t.pof, + r = &Pot{ + po: t.po, } return r, po, true } i := len(t.bins) - 1 last := t.bins[i] - r = &pot{ + r = &Pot{ pin: last.pin, bins: append(t.bins[:i], last.bins...), size: size, po: t.po, - pof: t.pof, } return r, t.po, true } - var p *pot + var p *Pot var i, j int for i < len(t.bins) { n := t.bins[i] if n.po == po { - p, po, found = remove(n, val) + p, po, found = remove(n, val, pof) if found { size-- } @@ -244,12 +238,11 @@ func remove(t *pot, val Val) (r *pot, po int, found bool) { bins = append(bins, p) } bins = append(bins, t.bins[j:]...) - r = &pot{ + r = &Pot{ pin: val, size: size, po: t.po, bins: bins, - pof: t.pof, } return r, po, found } @@ -258,149 +251,139 @@ func remove(t *pot, val Val) (r *pot, po int, found bool) { // and applies the function f to the value v at k or nil if the item is not found // if f returns nil, the element is removed // if f returns v' <> v then v' is inserted into the Pot -// if v' == v the pot is not changed +// if v' == v the Pot is not changed // it panics if v'.PO(k, 0) says v and k are not equal -func (t *Pot) Swap(val Val, f func(v Val) Val) (po int, found bool, change bool) { - t.lock.Lock() - defer t.lock.Unlock() - var t0 *pot - t0, po, found, change = swap(t.pot, val, f) - if change { - t.pot = t0 - } - return po, found, change -} +// func (t *Pot) Swap(val Val, f func(v Val) Val) (po int, found bool, change bool) { +// t.lock.Lock() +// defer t.lock.Unlock() +// var t0 *Pot +// t0, po, found, change = swap(t.Pot, val, f) +// if change { +// t.Pot = t0 +// } +// return po, found, change +// } -func swap(t *pot, k Val, f func(v Val) Val) (r *pot, po int, found bool, change bool) { +// Swap is applicative add, change, remove on a Pot +// func Swap(t *Pot, val Val, f func(v Val) Val) (_ *Pot, po int, found bool, change bool) { +// var t0 *Pot +// t0, po, found, change = swap(t.Pot, val, f) +// return &Pot{Pot: t0}, po, found, change +// } + +// func swap(t *Pot, k Val, f func(v Val) Val) (r *Pot, po int, found bool, change bool) { +func Swap(t *Pot, k Val, pof Pof, f func(v Val) Val) (r *Pot, po int, found bool, change bool) { var val Val - if t == nil || t.pin == nil { + if t.pin == nil { val = f(nil) if val == nil { return t, t.po, false, false } - // if _, eq := t.pof(k, t.pin, t.po); !eq { - // panic("value key mismatch") - // } - r = t.clone() - r.pin = val - r.size++ - return r, t.po, false, true + return NewPot(val, t.po), t.po, false, true } size := t.size - if k == nil { - panic("k is nil") - } - po, found = t.pof(k, t.pin, t.po) + po, found = pof(k, t.pin, t.po) if found { val = f(t.pin) if val == nil { size-- if size == 0 { - r = &pot{ - po: t.po, - pof: t.pof, + r = &Pot{ + po: t.po, } return r, po, true, true } i := len(t.bins) - 1 last := t.bins[i] - r = &pot{ + r = &Pot{ pin: last.pin, bins: append(t.bins[:i], last.bins...), size: size, po: t.po, - pof: t.pof, } return r, t.po, true, true // remove element } else if val == t.pin { - return nil, po, true, false + return t, po, true, false } else { // add element r = t.clone() r.pin = val return r, po, true, true } } - - var p *pot - var i, j int - for i < len(t.bins) { - n := t.bins[i] - if n.po == po { - p, po, found, change = swap(n, k, f) - if !change { - return nil, po, found, false - } - size += p.size - n.size - j++ - break + // recursive step + var p *Pot + n, i := t.getPos(po) + if n != nil { + p, po, found, change = Swap(n, k, pof, f) + if !change { + return t, po, found, false } - if n.po > po { - break - } - i++ - j++ - } - if p == nil { - val := f(nil) + size += p.size - n.size + } else { + val = f(nil) if val == nil { - return nil, po, false, false + return t, po, false, false + } + if _, eq := pof(val, k, po); !eq { + panic("invalid value") } size++ - p = &pot{ + p = &Pot{ pin: val, size: 1, po: po, - pof: t.pof, } } - bins := append([]*pot{}, t.bins[:i]...) + bins := append([]*Pot{}, t.bins[:i]...) if p.pin != nil { bins = append(bins, p) } - bins = append(bins, t.bins[j:]...) - r = &pot{ - pin: t.pin, + if i < len(t.bins) { + bins = append(bins, t.bins[i+1:]...) + } + r = &Pot{ + pin: f(t.pin), size: size, po: t.po, bins: bins, - pof: t.pof, } return r, po, found, true } // Merge called on (t1) changes t0 to contain all the elements of t1 -// it locks t0, but only readlocks t1 while taking its pot +// it locks t0, but only readlocks t1 while taking its Pot // uses applicative union -func (t *Pot) Merge(t1 *Pot) (c int) { - t.lock.Lock() - defer t.lock.Unlock() - t1.lock.RLock() - n1 := t1.pot - t1.lock.RUnlock() - t.pot, c = union(t.pot, n1) - return c -} +// func (t *Pot) Merge(t1 *Pot) (c int) { +// t.lock.Lock() +// defer t.lock.Unlock() +// t1.lock.RLock() +// n1 := t1.Pot +// t1.lock.RUnlock() +// t.Pot, c = union(t.Pot, n1) +// return c +// } // Union returns the union of t0 and t1 -// it only readlocks the Pot-s to read their pots and +// it only readlocks the Pot-s to read their Pots and // calculates the union using the applicative union // the second return value is the number of common elements -func Union(t0, t1 *Pot) (*Pot, int) { - t0.lock.RLock() - n0 := t0.pot - t0.lock.RUnlock() - t1.lock.RLock() - n1 := t1.pot - t1.lock.RUnlock() - - p, c := union(n0, n1) - return &Pot{pot: p}, c +func Union(t0, t1 *Pot, pof Pof) (*Pot, int) { + // t0.lock.RLock() + // n0 := t0.Pot + // t0.lock.RUnlock() + // t1.lock.RLock() + // n1 := t1.Pot + // t1.lock.RUnlock() + // + // p, c := union(n0, n1) + // return &Pot{Pot: p}, c + return union(t0, t1, pof) } -func union(t0, t1 *pot) (*pot, int) { +func union(t0, t1 *Pot, pof Pof) (*Pot, int) { if t0 == nil || t0.size == 0 { return t1, 0 } @@ -408,7 +391,7 @@ func union(t0, t1 *pot) (*pot, int) { return t0, 0 } var pin Val - var bins []*pot + var bins []*Pot var mis []int wg := &sync.WaitGroup{} pin0 := t0.pin @@ -418,12 +401,12 @@ func union(t0, t1 *pot) (*pot, int) { var i0, i1 int var common int - po, eq := t0.pof(pin0, pin1, 0) + po, eq := pof(pin0, pin1, 0) for { l0 := len(bins0) l1 := len(bins1) - var n0, n1 *pot + var n0, n1 *Pot var p0, p1 int var a0, a1 bool @@ -463,9 +446,9 @@ func union(t0, t1 *pot) (*pot, int) { ml := len(mis) mis = append(mis, 0) wg.Add(1) - go func(b, m int, m0, m1 *pot) { + go func(b, m int, m0, m1 *Pot) { defer wg.Done() - bins[b], mis[m] = union(m0, m1) + bins[b], mis[m] = union(m0, m1, pof) }(bl, ml, n0, n1) i0++ i1++ @@ -488,15 +471,14 @@ func union(t0, t1 *pot) (*pot, int) { for _, n := range bins0[i:] { size0 += n.size } - np := &pot{ + np := &Pot{ pin: pin0, bins: bins0[i:], size: size0 + 1, po: po, - pof: t0.pof, } - bins2 := []*pot{np} + bins2 := []*Pot{np} if n0 == nil { pin0 = pin1 po = maxkeylen + 1 @@ -507,7 +489,7 @@ func union(t0, t1 *pot) (*pot, int) { bins2 = append(bins2, n0.bins...) pin0 = pin1 pin1 = n0.pin - po, eq = t0.pof(pin0, pin1, n0.po) + po, eq = pof(pin0, pin1, n0.po) } bins0 = bins1 @@ -521,12 +503,11 @@ func union(t0, t1 *pot) (*pot, int) { for _, c := range mis { common += c } - n := &pot{ + n := &Pot{ pin: pin, bins: bins, size: t0.size + t1.size - common, po: t0.po, - pof: t0.pof, } return n, common } @@ -535,13 +516,14 @@ func union(t0, t1 *pot) (*pot, int) { // respecting an ordering // proximity > pinnedness func (t *Pot) Each(f func(Val, int) bool) bool { - t.lock.RLock() - n := t.pot - t.lock.RUnlock() - return n.each(f) + // t.lock.RLock() + // n := t.Pot + // t.lock.RUnlock() + // return n.each(f) + return t.each(f) } -func (t *pot) each(f func(Val, int) bool) bool { +func (t *Pot) each(f func(Val, int) bool) bool { var next bool for _, n := range t.bins { if n == nil { @@ -555,7 +537,7 @@ func (t *pot) each(f func(Val, int) bool) bool { return f(t.pin, t.po) } -// EachFrom called with (f, start) is a synchronous iterator over the elements of a pot +// EachFrom called with (f, start) is a synchronous iterator over the elements of a Pot // within the inclusive range starting from proximity order start // the function argument is passed the value and the proximity order wrt the root pin // it does NOT include the pinned item of the root @@ -564,13 +546,14 @@ func (t *pot) each(f func(Val, int) bool) bool { // the iteration ends if the function return false or there are no more elements // end of a po range can be implemented since po is passed to the function func (t *Pot) EachFrom(f func(Val, int) bool, po int) bool { - t.lock.RLock() - n := t.pot - t.lock.RUnlock() - return n.eachFrom(f, po) + // t.lock.RLock() + // n := t.Pot + // t.lock.RUnlock() + // return n.eachFrom(f, po) + return t.eachFrom(f, po) } -func (t *pot) eachFrom(f func(Val, int) bool, po int) bool { +func (t *Pot) eachFrom(f func(Val, int) bool, po int) bool { var next bool _, lim := t.getPos(po) for i := lim; i < len(t.bins); i++ { @@ -587,21 +570,22 @@ func (t *pot) eachFrom(f func(Val, int) bool, po int) bool { // subtree passing the proximity order and the size // the iteration continues until the function's return value is false // or there are no more subtries -func (t *Pot) EachBin(val Val, po int, f func(int, int, func(func(val Val, i int) bool) bool) bool) { - t.lock.RLock() - n := t.pot - t.lock.RUnlock() - n.eachBin(val, po, f) +func (t *Pot) EachBin(val Val, pof Pof, po int, f func(int, int, func(func(val Val, i int) bool) bool) bool) { + // t.lock.RLock() + // n := t.Pot + // t.lock.RUnlock() + // n.eachBin(val, po, f) + t.eachBin(val, pof, po, f) } -func (t *pot) eachBin(val Val, po int, f func(int, int, func(func(val Val, i int) bool) bool) bool) { +func (t *Pot) eachBin(val Val, pof Pof, po int, f func(int, int, func(func(val Val, i int) bool) bool) bool) { if t == nil || t.size == 0 { return } - spr, _ := t.pof(t.pin, val, t.po) + spr, _ := pof(t.pin, val, t.po) _, lim := t.getPos(spr) var size int - var n *pot + var n *Pot for i := 0; i < lim; i++ { n = t.bins[i] size += n.size @@ -633,34 +617,35 @@ func (t *pot) eachBin(val Val, po int, f func(int, int, func(func(val Val, i int return } if spo > spr { - n.eachBin(val, spo, f) + n.eachBin(val, pof, spo, f) } } // EachNeighbour is a syncronous iterator over neighbours of any target val // the order of elements retrieved reflect proximity order to the target // TODO: add maximum proxbin to start range of iteration -func (t *Pot) EachNeighbour(val Val, f func(Val, int) bool) bool { - t.lock.RLock() - n := t.pot - t.lock.RUnlock() - return n.eachNeighbour(val, f) +func (t *Pot) EachNeighbour(val Val, pof Pof, f func(Val, int) bool) bool { + // t.lock.RLock() + // n := t.Pot + // t.lock.RUnlock() + // return n.eachNeighbour(val, f) + return t.eachNeighbour(val, pof, f) } -func (t *pot) eachNeighbour(val Val, f func(Val, int) bool) bool { +func (t *Pot) eachNeighbour(val Val, pof Pof, f func(Val, int) bool) bool { if t == nil || t.size == 0 { return false } var next bool l := len(t.bins) - var n *pot + var n *Pot ir := l il := l - po, eq := t.pof(t.pin, val, t.po) + po, eq := pof(t.pin, val, t.po) if !eq { n, il = t.getPos(po) if n != nil { - next = n.eachNeighbour(val, f) + next = n.eachNeighbour(val, pof, f) if !next { return false } @@ -698,19 +683,19 @@ func (t *pot) eachNeighbour(val Val, f func(Val, int) bool) bool { // EachNeighbourAsync called on (val, max, maxPos, f, wait) is an asyncronous iterator // over elements not closer than maxPos wrt val. -// val does not need to be match an element of the pot, but if it does, and +// val does not need to be match an element of the Pot, but if it does, and // maxPos is keylength than it is included in the iteration // Calls to f are parallelised, the order of calls is undefined. -// proximity order is respected in that there is no element in the pot that +// proximity order is respected in that there is no element in the Pot that // is not visited if a closer node is visited. // The iteration is finished when max number of nearest nodes is visited // or if the entire there are no nodes not closer than maxPos that is not visited // if wait is true, the iterator returns only if all calls to f are finished // TODO: implement minPos for proper prox range iteration -func (t *Pot) EachNeighbourAsync(val Val, max int, maxPos int, f func(Val, int), wait bool) { - t.lock.RLock() - n := t.pot - t.lock.RUnlock() +func (t *Pot) EachNeighbourAsync(val Val, pof Pof, max int, maxPos int, f func(Val, int), wait bool) { + // t.lock.RLock() + // n := t.Pot + // t.lock.RUnlock() if max > t.size { max = t.size } @@ -718,28 +703,25 @@ func (t *Pot) EachNeighbourAsync(val Val, max int, maxPos int, f func(Val, int), if wait { wg = &sync.WaitGroup{} } - _ = n.eachNeighbourAsync(val, max, maxPos, f, wg) + // _ = n.eachNeighbourAsync(val, max, maxPos, f, wg) + _ = t.eachNeighbourAsync(val, pof, max, maxPos, f, wg) if wait { wg.Wait() } } -func (t *pot) eachNeighbourAsync(val Val, max int, maxPos int, f func(Val, int), wg *sync.WaitGroup) (extra int) { +func (t *Pot) eachNeighbourAsync(val Val, pof Pof, max int, maxPos int, f func(Val, int), wg *sync.WaitGroup) (extra int) { l := len(t.bins) - var n *pot - il := l - ir := l - // ic := l - po, eq := t.pof(t.pin, val, t.po) + po, eq := pof(t.pin, val, t.po) // if po is too close, set the pivot branch (pom) to maxPos pom := po if pom > maxPos { pom = maxPos } - n, il = t.getPos(pom) - ir = il + n, il := t.getPos(pom) + ir := il // if pivot branch exists and po is not too close, iterate on the pivot branch if pom == po { if n != nil { @@ -750,7 +732,7 @@ func (t *pot) eachNeighbourAsync(val Val, max int, maxPos int, f func(Val, int), } max -= m - extra = n.eachNeighbourAsync(val, m, maxPos, f, wg) + extra = n.eachNeighbourAsync(val, pof, m, maxPos, f, wg) } else { if !eq { @@ -806,7 +788,7 @@ func (t *pot) eachNeighbourAsync(val Val, max int, maxPos int, f func(Val, int), if wg != nil { wg.Add(m) } - go func(pn *pot, pm int) { + go func(pn *Pot, pm int) { pn.each(func(v Val, _ int) bool { if wg != nil { defer wg.Done() @@ -833,7 +815,7 @@ func (t *pot) eachNeighbourAsync(val Val, max int, maxPos int, f func(Val, int), if wg != nil { wg.Add(m) } - go func(pn *pot, pm int) { + go func(pn *Pot, pm int) { pn.each(func(v Val, _ int) bool { if wg != nil { defer wg.Done() @@ -851,7 +833,7 @@ func (t *pot) eachNeighbourAsync(val Val, max int, maxPos int, f func(Val, int), // getPos called on (n) returns the forking node at PO n and its index if it exists // otherwise nil // caller is suppoed to hold the lock -func (t *pot) getPos(po int) (n *pot, i int) { +func (t *Pot) getPos(po int) (n *Pot, i int) { for i, n = range t.bins { if po > n.po { continue @@ -877,11 +859,11 @@ func need(m, max, extra int) (int, int, int) { return m, max, 0 } -func (t *pot) String() string { +func (t *Pot) String() string { return t.sstring("") } -func (t *pot) sstring(indent string) string { +func (t *Pot) sstring(indent string) string { if t == nil { return "" } diff --git a/pot/pot_test.go b/pot/pot_test.go index 1d4dd76966..bab923a0ea 100644 --- a/pot/pot_test.go +++ b/pot/pot_test.go @@ -19,7 +19,6 @@ import ( "errors" "fmt" "math/rand" - "os" "runtime" "sync" "testing" @@ -34,7 +33,7 @@ const ( ) func init() { - log.Root().SetHandler(log.LvlFilterHandler(log.LvlError, log.StreamHandler(os.Stderr, log.TerminalFormat(false)))) + // log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(false)))) } type testAddr struct { @@ -84,15 +83,18 @@ func indexes(t *Pot) (i []int, po []int) { return i, po } -func testAdd(t *Pot, n int, values ...string) { +func testAdd(t *Pot, pof Pof, j int, values ...string) (_ *Pot, n int, f bool) { for i, val := range values { - t.Add(newTestAddr(val, i+n)) + t, n, f = Add(t, newTestAddr(val, i+j), pof) + // t.Add(newTestAddr(val, i+n)) } + return t, n, f } // func RandomBoolAddress() func TestPotAdd(t *testing.T) { - n := NewPot(newTestAddr("00111100", 0), 0, DefaultPof(8)) + pof := DefaultPof(8) + n := NewPot(newTestAddr("00111100", 0), 0) // Pin set correctly exp := "00111100" got := Label(n.Pin())[:8] @@ -106,7 +108,7 @@ func TestPotAdd(t *testing.T) { t.Fatalf("incorrect number of elements in Pot. Expected %v, got %v", expi, goti) } - testAdd(n, 1, "01111100", "00111100", "01111100", "00011100") + n, _, _ = testAdd(n, pof, 1, "01111100", "00111100", "01111100", "00011100") // check size goti = n.Size() expi = 3 @@ -128,15 +130,16 @@ func TestPotAdd(t *testing.T) { // func RandomBoolAddress() func TestPotRemove(t *testing.T) { - n := NewPot(newTestAddr("00111100", 0), 0, DefaultPof(8)) - n.Remove(newTestAddr("00111100", 0)) + pof := DefaultPof(8) + n := NewPot(newTestAddr("00111100", 0), 0) + n, _, _ = Remove(n, newTestAddr("00111100", 0), pof) exp := "" got := Label(n.Pin()) if got != exp { t.Fatalf("incorrect pinned value. Expected %v, got %v", exp, got) } - testAdd(n, 1, "00000000", "01111100", "00111100", "00011100") - n.Remove(newTestAddr("00111100", 0)) + n, _, _ = testAdd(n, pof, 1, "00000000", "01111100", "00111100", "00011100") + n, _, _ = Remove(n, newTestAddr("00111100", 0), pof) goti := n.Size() expi := 3 if goti != expi { @@ -154,8 +157,8 @@ func TestPotRemove(t *testing.T) { t.Fatalf("incorrect po-s in iteration over Pot. Expected %v, got %v", exp, got) } // remove again - n.Remove(newTestAddr("00111100", 0)) - inds, po = indexes(n) + n, _, _ = Remove(n, newTestAddr("00111100", 0), pof) + inds, _ = indexes(n) got = fmt.Sprintf("%v", inds) exp = "[2 4]" if got != exp { @@ -165,13 +168,14 @@ func TestPotRemove(t *testing.T) { } func TestPotSwap(t *testing.T) { - // t.Skip("") + pof := DefaultPof(8) max := maxEachNeighbour - n := NewPot(nil, 0, nil) + n := NewPot(nil, 0) var m []*testAddr + var found bool for j := 0; j < 2*max; { v := randomtestAddr(keylen, j) - _, found := n.Add(v) + n, _, found = Add(n, v, pof) if !found { m = append(m, v) j++ @@ -198,7 +202,7 @@ func TestPotSwap(t *testing.T) { return v } for _, val := range m { - n.Swap(val, func(v Val) Val { + n, _, _, _ = Swap(n, val, pof, func(v Val) Val { if v == nil { return val } @@ -234,7 +238,7 @@ func checkPo(val Val, pof Pof) func(Val, int) error { } func checkOrder(val Val) func(Val, int) error { - var po int = keylen + po := keylen return func(v Val, p int) error { if po < p { return fmt.Errorf("incorrect order for item %v in neighbour iteration for %v. PO %v > %v (previous max)", v, val, p, po) @@ -260,10 +264,10 @@ func checkValues(m map[string]bool, val Val) func(Val, int) error { var errNoCount = errors.New("not count") -func testPotEachNeighbour(n *Pot, val Val, expCount int, fs ...func(Val, int) error) error { +func testPotEachNeighbour(n *Pot, pof Pof, val Val, expCount int, fs ...func(Val, int) error) error { var err error var count int - n.EachNeighbour(val, func(v Val, po int) bool { + n.EachNeighbour(val, pof, func(v Val, po int) bool { for _, f := range fs { err = f(v, po) if err != nil { @@ -287,41 +291,43 @@ const ( mergeTestChoose = 5 ) -func TestPotMergeOne(t *testing.T) { - pot1 := NewPot(nil, 0, DefaultPof(2)) - pot1.Add(newTestAddr("10", 0)) - pot1.Add(newTestAddr("00", 0)) - pot2 := NewPot(nil, 0, DefaultPof(2)) - pot2.Add(newTestAddr("01", 0)) - pot1.Merge(pot2) - count := 0 - pot1.Each(func(val Val, i int) bool { - count++ - return true - }) - if count != 3 { - t.Fatalf("expected count to be 3, got %d\n%v\n%v", count, pot2, pot1) - } -} +// +// func TestPotMergeOne(t *testing.T) { +// pot1 := NewPot(nil, 0, DefaultPof(2)) +// pot1.Add(newTestAddr("10", 0)) +// pot1.Add(newTestAddr("00", 0)) +// pot2 := NewPot(nil, 0, DefaultPof(2)) +// pot2.Add(newTestAddr("01", 0)) +// pot1.Merge(pot2) +// count := 0 +// pot1.Each(func(val Val, i int) bool { +// count++ +// return true +// }) +// if count != 3 { +// t.Fatalf("expected count to be 3, got %d\n%v\n%v", count, pot2, pot1) +// } +// } func TestPotMergeCommon(t *testing.T) { vs := make([]*testAddr, mergeTestCount) - // vs := make([]*testAddr, mergeTestCount) + pof := DefaultPof(maxkeylen) for i := 0; i < maxEachNeighbourTests; i++ { - for i := 0; i < len(vs); i++ { - vs[i] = randomtestAddr(keylen, i) + for j := 0; j < len(vs); j++ { + vs[j] = randomtestAddr(keylen, j) } max0 := rand.Intn(mergeTestChoose) + 1 max1 := rand.Intn(mergeTestChoose) + 1 - n0 := NewPot(nil, 0, nil) - n1 := NewPot(nil, 0, nil) + n0 := NewPot(nil, 0) + n1 := NewPot(nil, 0) log.Trace(fmt.Sprintf("round %v: %v - %v", i, max0, max1)) m := make(map[string]bool) + var found bool for j := 0; j < max0; { r := rand.Intn(max0) v := vs[r] - _, found := n0.Add(v) + n0, _, found = Add(n0, v, pof) if !found { m[Label(v)] = false j++ @@ -332,7 +338,7 @@ func TestPotMergeCommon(t *testing.T) { for j := 0; j < max1; { r := rand.Intn(max1) v := vs[r] - _, found := n1.Add(v) + n1, _, found = Add(n1, v, pof) if !found { j++ } @@ -349,7 +355,7 @@ func TestPotMergeCommon(t *testing.T) { log.Trace(fmt.Sprintf("%v-0: pin: %v, size: %v", i, n0.Pin(), max0)) log.Trace(fmt.Sprintf("%v-1: pin: %v, size: %v", i, n1.Pin(), max1)) log.Trace(fmt.Sprintf("%v: merged tree size: %v, newly added: %v", i, expSize, expAdded)) - n, common := Union(n0, n1) + n, common := Union(n0, n1, pof) added := n1.Size() - common size := n.Size() @@ -359,11 +365,11 @@ func TestPotMergeCommon(t *testing.T) { if expAdded != added { t.Fatalf("%v: incorrect number of added elements in merged pot, expected %v, got %v", i, expAdded, added) } - if !checkDuplicates(n.pot) { + if !checkDuplicates(n) { t.Fatalf("%v: merged pot contains duplicates: \n%v", i, n) } for k := range m { - _, found := n.Add(newTestAddr(k, 0)) + n, _, found := Add(n, newTestAddr(k, 0), pof) if !found { t.Fatalf("%v: merged pot (size:%v, added: %v) missing element %v\n%v", i, size, added, k, n) } @@ -372,17 +378,20 @@ func TestPotMergeCommon(t *testing.T) { } func TestPotMergeScale(t *testing.T) { + pof := DefaultPof(maxkeylen) for i := 0; i < maxEachNeighbourTests; i++ { max0 := rand.Intn(maxEachNeighbour) + 1 max1 := rand.Intn(maxEachNeighbour) + 1 - n0 := NewPot(nil, 0, nil) - n1 := NewPot(nil, 0, nil) + n0 := NewPot(nil, 0) + n1 := NewPot(nil, 0) log.Trace(fmt.Sprintf("round %v: %v - %v", i, max0, max1)) m := make(map[string]bool) + var found bool for j := 0; j < max0; { v := randomtestAddr(keylen, j) // v := randomtestAddr(keylen, j) - _, found := n0.Add(v) + n0, _, found = Add(n0, v, pof) + // _, found := n0.Add(v) if !found { m[Label(v)] = false j++ @@ -393,7 +402,8 @@ func TestPotMergeScale(t *testing.T) { for j := 0; j < max1; { v := randomtestAddr(keylen, j) // v := randomtestAddr(keylen, j) - _, found := n1.Add(v) + n1, _, found = Add(n1, v, pof) + // _, found := n1.Add(v) if !found { j++ } @@ -410,7 +420,7 @@ func TestPotMergeScale(t *testing.T) { log.Trace(fmt.Sprintf("%v-0: pin: %v, size: %v", i, n0.Pin(), max0)) log.Trace(fmt.Sprintf("%v-1: pin: %v, size: %v", i, n1.Pin(), max1)) log.Trace(fmt.Sprintf("%v: merged tree size: %v, newly added: %v", i, expSize, expAdded)) - n, common := Union(n0, n1) + n, common := Union(n0, n1, pof) added := n1.Size() - common size := n.Size() @@ -420,11 +430,11 @@ func TestPotMergeScale(t *testing.T) { if expAdded != added { t.Fatalf("%v: incorrect number of added elements in merged pot, expected %v, got %v", i, expAdded, added) } - if !checkDuplicates(n.pot) { + if !checkDuplicates(n) { t.Fatalf("%v: merged pot contains duplicates: \n%v", i, n) } for k := range m { - _, found := n.Add(newTestAddr(k, 0)) + n, _, found := Add(n, newTestAddr(k, 0), pof) if !found { t.Fatalf("%v: merged pot (size:%v, added: %v) missing element %v\n%v", i, size, added, k, n) } @@ -432,7 +442,7 @@ func TestPotMergeScale(t *testing.T) { } } -func checkDuplicates(t *pot) bool { +func checkDuplicates(t *Pot) bool { po := -1 for _, c := range t.bins { if c == nil { @@ -447,15 +457,16 @@ func checkDuplicates(t *pot) bool { } func TestPotEachNeighbourSync(t *testing.T) { + pof := DefaultPof(maxkeylen) for i := 0; i < maxEachNeighbourTests; i++ { max := rand.Intn(maxEachNeighbour/2) + maxEachNeighbour/2 pin := randomTestAddr(keylen, 0) - n := NewPot(pin, 0, nil) + n := NewPot(pin, 0) m := make(map[string]bool) m[Label(pin)] = false for j := 1; j <= max; j++ { v := randomTestAddr(keylen, j) - n.Add(v) + n, _, _ = Add(n, v, pof) m[Label(v)] = false } @@ -466,14 +477,14 @@ func TestPotEachNeighbourSync(t *testing.T) { count := rand.Intn(size/2) + size/2 val := randomTestAddr(keylen, max+1) log.Trace(fmt.Sprintf("%v: pin: %v, size: %v, val: %v, count: %v", i, n.Pin(), size, val, count)) - err := testPotEachNeighbour(n, val, count, checkPo(val, n.pof), checkOrder(val), checkValues(m, val)) + err := testPotEachNeighbour(n, pof, val, count, checkPo(val, pof), checkOrder(val), checkValues(m, val)) if err != nil { t.Fatal(err) } minPoFound := keylen maxPoNotFound := 0 for k, found := range m { - po, _ := n.pof(val, newTestAddr(k, 0), 0) + po, _ := pof(val, newTestAddr(k, 0), 0) if found { if po < minPoFound { minPoFound = po @@ -491,13 +502,15 @@ func TestPotEachNeighbourSync(t *testing.T) { } func TestPotEachNeighbourAsync(t *testing.T) { + pof := DefaultPof(maxkeylen) for i := 0; i < maxEachNeighbourTests; i++ { max := rand.Intn(maxEachNeighbour/2) + maxEachNeighbour/2 - n := NewPot(randomTestAddr(keylen, 0), 0, nil) + n := NewPot(randomTestAddr(keylen, 0), 0) size := 1 + var found bool for j := 1; j <= max; j++ { v := randomTestAddr(keylen, j) - _, found := n.Add(v) + n, _, found = Add(n, v, pof) if !found { size++ } @@ -527,7 +540,7 @@ func TestPotEachNeighbourAsync(t *testing.T) { if i == 0 { continue } - testPotEachNeighbour(n, val, count, remember) + testPotEachNeighbour(n, pof, val, count, remember) d := 0 forget := func(v Val, po int) { mu.Lock() @@ -536,7 +549,7 @@ func TestPotEachNeighbourAsync(t *testing.T) { delete(m, Label(v)) } - n.EachNeighbourAsync(val, count, maxPos, forget, true) + n.EachNeighbourAsync(val, pof, count, maxPos, forget, true) if d != msize { t.Fatalf("incorrect number of neighbour calls in async iterator. expected %v, got %v", msize, d) } @@ -548,11 +561,13 @@ func TestPotEachNeighbourAsync(t *testing.T) { func benchmarkEachNeighbourSync(t *testing.B, max, count int, d time.Duration) { t.ReportAllocs() + pof := DefaultPof(maxkeylen) pin := randomTestAddr(keylen, 0) - n := NewPot(pin, 0, nil) + n := NewPot(pin, 0) + var found bool for j := 1; j <= max; { v := randomTestAddr(keylen, j) - _, found := n.Add(v) + n, _, found = Add(n, v, pof) if !found { j++ } @@ -561,7 +576,7 @@ func benchmarkEachNeighbourSync(t *testing.B, max, count int, d time.Duration) { for i := 0; i < t.N; i++ { val := randomTestAddr(keylen, max+1) m := 0 - n.EachNeighbour(val, func(v Val, po int) bool { + n.EachNeighbour(val, pof, func(v Val, po int) bool { time.Sleep(d) m++ if m == count { @@ -578,11 +593,13 @@ func benchmarkEachNeighbourSync(t *testing.B, max, count int, d time.Duration) { func benchmarkEachNeighbourAsync(t *testing.B, max, count int, d time.Duration) { t.ReportAllocs() + pof := DefaultPof(maxkeylen) pin := randomTestAddr(keylen, 0) - n := NewPot(pin, 0, nil) + n := NewPot(pin, 0) + var found bool for j := 1; j <= max; { v := randomTestAddr(keylen, j) - _, found := n.Add(v) + n, _, found = Add(n, v, pof) if !found { j++ } @@ -590,7 +607,7 @@ func benchmarkEachNeighbourAsync(t *testing.B, max, count int, d time.Duration) t.ResetTimer() for i := 0; i < t.N; i++ { val := randomTestAddr(keylen, max+1) - n.EachNeighbourAsync(val, count, keylen, func(v Val, po int) { + n.EachNeighbourAsync(val, pof, count, keylen, func(v Val, po int) { time.Sleep(d) }, true) } diff --git a/swarm/network/hive_test.go b/swarm/network/hive_test.go index 3fa969df4c..f833919ed4 100644 --- a/swarm/network/hive_test.go +++ b/swarm/network/hive_test.go @@ -17,6 +17,7 @@ func newHiveTester(t *testing.T, params *HiveParams) (*bzzTester, *Hive) { } func TestRegisterAndConnect(t *testing.T) { + t.Skip("deadlocked") params := NewHiveParams() s, pp := newHiveTester(t, params) defer s.Stop() diff --git a/swarm/network/kademlia.go b/swarm/network/kademlia.go index 1d7663c124..e2cfb20443 100644 --- a/swarm/network/kademlia.go +++ b/swarm/network/kademlia.go @@ -20,6 +20,7 @@ import ( "bytes" "fmt" "strings" + "sync" "time" "github.com/ethereum/go-ethereum/log" @@ -47,6 +48,8 @@ a guaranteed constant maximum limit on the number of hops needed to reach one node from the other. */ +var pof = pot.DefaultPof(256) + // KadParams holds the config params for Kademlia type KadParams struct { // adjustable parameters @@ -76,11 +79,12 @@ func NewKadParams() *KadParams { // Kademlia is a table of live peers and a db of known peers type Kademlia struct { - *KadParams // Kademlia configuration parameters - base []byte // immutable baseaddress of the table - addrs *pot.Pot // pots container for known peer addresses - conns *pot.Pot // pots container for live peer connections - depth uint8 // stores the last calculated depth + lock sync.RWMutex + *KadParams // Kademlia configuration parameters + base []byte // immutable baseaddress of the table + addrs *pot.Pot // pots container for known peer addresses + conns *pot.Pot // pots container for live peer connections + currentDepth uint8 // stores the last calculated depth } // NewKademlia creates a Kademlia table for base address addr @@ -93,8 +97,8 @@ func NewKademlia(addr []byte, params *KadParams) *Kademlia { return &Kademlia{ base: addr, KadParams: params, - addrs: pot.NewPot(nil, 0, nil), - conns: pot.NewPot(nil, 0, nil), + addrs: pot.NewPot(nil, 0), + conns: pot.NewPot(nil, 0), } } @@ -173,15 +177,19 @@ func (e *entry) conn() OverlayConn { // Register enters each OverlayAddr as kademlia peer record into the // database of known peer addresses func (k *Kademlia) Register(peers chan OverlayAddr) error { - np := pot.NewPot(nil, 0, nil) + + np := pot.NewPot(nil, 0) for p := range peers { // error if k received, peer should know better if bytes.Equal(p.Address(), k.base) { return fmt.Errorf("add peers: %x is k", k.base) } - np, _, _ = pot.Add(np, newEntry(p)) + np, _, _ = pot.Add(np, newEntry(p), pof) } - com := k.addrs.Merge(np) + var com int + k.lock.Lock() + defer k.lock.Unlock() + k.addrs, com = pot.Union(k.addrs, np, pof) log.Trace(fmt.Sprintf("%x merged %v peers, %v known, total: %v", k.BaseAddr()[:4], np.Size(), com, k.addrs.Size())) return nil } @@ -191,13 +199,14 @@ func (k *Kademlia) Register(peers chan OverlayAddr) error { // naturally if there is an empty row it returns a peer for that // func (k *Kademlia) SuggestPeer() (a OverlayAddr, o int, want bool) { + k.lock.RLock() + defer k.lock.RUnlock() minsize := k.MinBinSize - depth := k.Depth() - // empty := k.FirstEmptyBin() + depth := k.depth() // if there is a callable neighbour within the current proxBin, connect // this makes sure nearest neighbour set is fully connected var ppo int - k.addrs.EachNeighbour(k.base, func(val pot.Val, po int) bool { + k.addrs.EachNeighbour(k.base, pof, func(val pot.Val, po int) bool { if po < depth { return false } @@ -214,7 +223,7 @@ func (k *Kademlia) SuggestPeer() (a OverlayAddr, o int, want bool) { var bpo []int prev := -1 - k.conns.EachBin(k.base, 0, func(po, size int, f func(func(val pot.Val, i int) bool) bool) bool { + k.conns.EachBin(k.base, pof, 0, func(po, size int, f func(func(val pot.Val, i int) bool) bool) bool { prev++ for ; prev < po; prev++ { bpo = append(bpo, prev) @@ -238,7 +247,7 @@ func (k *Kademlia) SuggestPeer() (a OverlayAddr, o int, want bool) { // find the first callable peer i := 0 nxt := bpo[0] - k.addrs.EachBin(k.base, nxt, func(po, size int, f func(func(pot.Val, int) bool) bool) bool { + k.addrs.EachBin(k.base, pof, nxt, func(po, size int, f func(func(pot.Val, int) bool) bool) bool { // for each bin we find callable candidate peers if i >= depth { return false @@ -264,29 +273,34 @@ func (k *Kademlia) SuggestPeer() (a OverlayAddr, o int, want bool) { // On inserts the peer as a kademlia peer into the live peers func (k *Kademlia) On(p OverlayConn) { + k.lock.Lock() + defer k.lock.Unlock() e := newEntry(p) - k.conns.Swap(p, func(v pot.Val) pot.Val { + var ins bool + k.conns, _, _, _ = pot.Swap(k.conns, p, pof, func(v pot.Val) pot.Val { // if not found live if v == nil { - // insert new online peer into addrs - k.addrs.Swap(p, func(v pot.Val) pot.Val { - return e - }) + ins = true // insert new online peer into conns return e } // found among live peers, do nothing return v }) - + if ins { + // insert new online peer into addrs + k.addrs, _, _, _ = pot.Swap(k.addrs, p, pof, func(v pot.Val) pot.Val { + return e + }) + } np, ok := p.(Notifier) if !ok { return } - depth := uint8(k.Depth()) - if depth != k.depth { - k.depth = depth + depth := uint8(k.depth()) + if depth != k.currentDepth { + k.currentDepth = depth } else { depth = 0 } @@ -301,33 +315,41 @@ func (k *Kademlia) On(p OverlayConn) { log.Trace(fmt.Sprintf("peer %v notified of new depth %v", dp, depth)) } } - k.conns.EachNeighbourAsync(e, 1024, 255, f, false) + k.conns.EachNeighbourAsync(e, pof, 1024, 255, f, false) } // Off removes a peer from among live peers func (k *Kademlia) Off(p OverlayConn) { - k.addrs.Swap(p, func(v pot.Val) pot.Val { + k.lock.Lock() + defer k.lock.Unlock() + var del bool + k.addrs, _, _, _ = pot.Swap(k.addrs, p, pof, func(v pot.Val) pot.Val { // v cannot be nil, must check otherwise we overwrite entry if v == nil { panic(fmt.Sprintf("connected peer not found %v", p)) } - k.conns.Swap(p, func(_ pot.Val) pot.Val { + del = true + return newEntry(p.Off()) + }) + if del { + k.conns, _, _, _ = pot.Swap(k.conns, p, pof, func(_ pot.Val) pot.Val { // v cannot be nil, but no need to check return nil }) - return newEntry(p.Off()) - }) + } } // EachConn is an iterator with args (base, po, f) applies f to each live peer // that has proximity order po or less as measured from the base // if base is nil, kademlia base address is used func (k *Kademlia) EachConn(base []byte, o int, f func(OverlayConn, int, bool) bool) { + k.lock.RLock() + defer k.lock.RUnlock() if len(base) == 0 { base = k.base } - depth := k.Depth() - k.conns.EachNeighbour(base, func(val pot.Val, po int) bool { + depth := k.depth() + k.conns.EachNeighbour(base, pof, func(val pot.Val, po int) bool { if po > o { return true } @@ -342,7 +364,9 @@ func (k *Kademlia) EachAddr(base []byte, o int, f func(OverlayAddr, int) bool) { if len(base) == 0 { base = k.base } - k.addrs.EachNeighbour(base, func(val pot.Val, po int) bool { + k.lock.RLock() + defer k.lock.RUnlock() + k.addrs.EachNeighbour(base, pof, func(val pot.Val, po int) bool { if po > o { return true } @@ -354,6 +378,12 @@ func (k *Kademlia) EachAddr(base []byte, o int, f func(OverlayAddr, int) bool) { // the nearest neighbour set with cardinality >= MinProxBinSize // if there is altogether less than MinProxBinSize peers it returns 0 func (k *Kademlia) Depth() (depth int) { + k.lock.RLock() + defer k.lock.RUnlock() + return k.depth() +} + +func (k *Kademlia) depth() (depth int) { if k.conns.Size() < k.MinProxBinSize { return 0 } @@ -363,7 +393,7 @@ func (k *Kademlia) Depth() (depth int) { depth = i return size < k.MinProxBinSize } - k.conns.EachNeighbour(k.base, f) + k.conns.EachNeighbour(k.base, pof, f) return depth } @@ -401,7 +431,9 @@ func (k *Kademlia) BaseAddr() []byte { // String returns kademlia table + kaddb table displayed with ascii func (k *Kademlia) String() string { - wsrow := " " + k.lock.RLock() + defer k.lock.RUnlock() + wsrow := " " var rows []string rows = append(rows, "=========================================================================") @@ -414,7 +446,7 @@ func (k *Kademlia) String() string { prev := -1 var depthSet bool rest := k.conns.Size() - k.conns.EachBin(k.base, 0, func(po, size int, f func(func(val pot.Val, i int) bool) bool) bool { + k.conns.EachBin(k.base, pof, 0, func(po, size int, f func(func(val pot.Val, i int) bool) bool) bool { var rowlen int if po >= k.MaxProxDisplay { po = k.MaxProxDisplay - 1 @@ -423,7 +455,7 @@ func (k *Kademlia) String() string { rest -= size f(func(val pot.Val, vpo int) bool { e := val.(*entry) - row = append(row, e.String()) + row = append(row, fmt.Sprintf("%x", e.Address()[:2])) rowlen++ return rowlen < 4 }) @@ -431,14 +463,14 @@ func (k *Kademlia) String() string { depthSet = true depth = prev + 1 } - row = append(row, wsrow) r := strings.Join(row, " ") - liverows[po] = r[:35] + r = r + wsrow + liverows[po] = r[:31] prev = po return true }) - k.addrs.EachBin(k.base, 0, func(po, size int, f func(func(val pot.Val, i int) bool) bool) bool { + k.addrs.EachBin(k.base, pof, 0, func(po, size int, f func(func(val pot.Val, i int) bool) bool) bool { var rowlen int if po >= k.MaxProxDisplay { po = k.MaxProxDisplay - 1 @@ -464,7 +496,7 @@ func (k *Kademlia) String() string { left := liverows[i] right := peersrows[i] if len(left) == 0 { - left = " 0 " + left = " 0 " } if len(right) == 0 { right = " 0" @@ -483,10 +515,12 @@ func (k *Kademlia) String() string { // the bin is reduced to MinBinSize peers thus leaving slots to newly // connecting peers func (k *Kademlia) Prune(c <-chan time.Time) { + k.lock.RLock() + defer k.lock.RUnlock() go func() { for range c { total := 0 - k.conns.EachBin(k.base, 0, func(po, size int, f func(func(pot.Val, int) bool) bool) bool { + k.conns.EachBin(k.base, pof, 0, func(po, size int, f func(func(pot.Val, int) bool) bool) bool { extra := size - k.MinBinSize if size > k.MaxBinSize { n := 0 @@ -507,17 +541,17 @@ func (k *Kademlia) Prune(c <-chan time.Time) { // NewPeerPot just creates a new pot record OverlayAddr func NewPeerPot(kadMinProxSize int, ids ...discover.NodeID) map[discover.NodeID][][]byte { // create a table of all nodes for health check - np := pot.NewPot(nil, 0, nil) + np := pot.NewPot(nil, 0) for _, id := range ids { o := ToOverlayAddr(id.Bytes()) - np, _, _ = pot.Add(np, o) + np, _, _ = pot.Add(np, o, pof) } nnmap := make(map[discover.NodeID][][]byte) for _, id := range ids { pl := 0 var nns [][]byte - np.EachNeighbour(id.Bytes(), func(val pot.Val, po int) bool { + np.EachNeighbour(id.Bytes(), pof, func(val pot.Val, po int) bool { // a := val.(pot.BytesAddress).Address() // nns = append(nns, a) a := val.([]byte) @@ -533,9 +567,9 @@ func NewPeerPot(kadMinProxSize int, ids ...discover.NodeID) map[discover.NodeID] } // FirstEmptyBin returns the farthest proximity order (int) that has no peer records -func (k *Kademlia) FirstEmptyBin() (i int) { +func (k *Kademlia) firstEmptyBin() (i int) { i = -1 - k.conns.EachBin(k.base, 0, func(po, size int, f func(func(val pot.Val, i int) bool) bool) bool { + k.conns.EachBin(k.base, pof, 0, func(po, size int, f func(func(val pot.Val, i int) bool) bool) bool { if po > i+1 { i = po return false @@ -547,13 +581,15 @@ func (k *Kademlia) FirstEmptyBin() (i int) { } // Full returns a bool if the kademlia table is healthy and complete -func (k *Kademlia) Full() bool { - return k.FirstEmptyBin() >= k.Depth() +func (k *Kademlia) full() bool { + return k.firstEmptyBin() >= k.depth() } // Healthy reports the health state of the kademlia connectivity func (k *Kademlia) Healthy(peers [][]byte) bool { - return k.gotNearestNeighbours(peers) && k.Full() + k.lock.RLock() + defer k.lock.RUnlock() + return k.gotNearestNeighbours(peers) && k.full() } func (k *Kademlia) gotNearestNeighbours(peers [][]byte) (got bool) { diff --git a/swarm/network/kademlia_test.go b/swarm/network/kademlia_test.go index a59c6370ab..ddf3613923 100644 --- a/swarm/network/kademlia_test.go +++ b/swarm/network/kademlia_test.go @@ -158,7 +158,7 @@ func testSuggestPeer(t *testing.T, k *testKademlia, expAddr string, expPo int, e if want != expWant { return fmt.Errorf("expected SuggestPeer to want peers: %v", expWant) } - // t.Logf("%v", k) + t.Logf("%v", k) return nil } @@ -206,9 +206,8 @@ func TestSuggestPeerFindPeers(t *testing.T) { } // reintroduce gap, disconnected peer callable - log.Info(k.String()) + // log.Info(k.String()) k.Off("01000000") - log.Info(k.String()) err = testSuggestPeer(t, k, "01000000", 0, false) if err != nil { t.Fatal(err.Error()) @@ -433,7 +432,7 @@ func TestPruning(t *testing.T) { func TestKademliaHiveString(t *testing.T) { k := newTestKademlia("00000000").On("01000000", "00100000").Register("10000000", "10000001") h := k.String() - expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), MinProxBinSize: 2, MinBinSize: 1, MaxBinSize: 4\n============ PROX LIMIT: 0 ==========================================\n000 0 | 2 840000 800000\n001 1 400000 | 1 400000\n002 1 200000 | 1 200000\n003 0 | 0\n004 0 | 0\n005 0 | 0\n006 0 | 0\n007 0 | 0\n=========================================================================" + expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), MinProxBinSize: 2, MinBinSize: 1, MaxBinSize: 4\n============ DEPTH: 0 ==========================================\n000 0 | 2 8100 (0) 8000 (0)\n001 1 4000 | 1 4000 (0)\n002 1 2000 | 1 2000 (0)\n003 0 | 0\n004 0 | 0\n005 0 | 0\n006 0 | 0\n007 0 | 0\n=========================================================================" if expH[100:] != h[100:] { t.Fatalf("incorrect hive output. expected %v, got %v", expH, h) } From a63fec8096a2b83bd359111571230080620865d0 Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 15 Jun 2017 12:22:38 +0200 Subject: [PATCH 06/17] pot, swarm/network: merge bug, kad deadlock --- pot/pot.go | 102 ++++++++++------- pot/pot_test.go | 228 +++++++++++++++++-------------------- swarm/network/hive_test.go | 7 +- 3 files changed, 173 insertions(+), 164 deletions(-) diff --git a/pot/pot.go b/pot/pot.go index 155b26ef91..d7bc459151 100644 --- a/pot/pot.go +++ b/pot/pot.go @@ -23,7 +23,6 @@ import ( ) const ( - keylen = 256 maxkeylen = 256 ) @@ -277,22 +276,25 @@ func Swap(t *Pot, k Val, pof Pof, f func(v Val) Val) (r *Pot, po int, found bool if t.pin == nil { val = f(nil) if val == nil { - return t, t.po, false, false + return nil, 0, false, false } - return NewPot(val, t.po), t.po, false, true + return NewPot(val, t.po), 0, false, true } size := t.size po, found = pof(k, t.pin, t.po) if found { val = f(t.pin) + // remove element if val == nil { size-- if size == 0 { r = &Pot{ po: t.po, } + // return empty pot return r, po, true, true } + // actually remove pin, by merging last bin i := len(t.bins) - 1 last := t.bins[i] r = &Pot{ @@ -301,55 +303,71 @@ func Swap(t *Pot, k Val, pof Pof, f func(v Val) Val) (r *Pot, po int, found bool size: size, po: t.po, } - return r, t.po, true, true - // remove element - } else if val == t.pin { - return t, po, true, false - } else { // add element - r = t.clone() - r.pin = val return r, po, true, true } + // element found but no change + if val == t.pin { + return t, po, true, false + } + // actually modify the pinned element, but no change in structure + r = t.clone() + r.pin = val + return r, po, true, true } + // recursive step var p *Pot n, i := t.getPos(po) if n != nil { p, po, found, change = Swap(n, k, pof, f) + // recursive no change if !change { return t, po, found, false } - size += p.size - n.size - } else { - val = f(nil) - if val == nil { - return t, po, false, false + // recursive change + //size += p.size - n.size + bins := append([]*Pot{}, t.bins[:i]...) + if p.size == 0 { + size-- + } else { + size += p.size - n.size + bins = append(bins, p) } - if _, eq := pof(val, k, po); !eq { - panic("invalid value") - } - size++ - p = &Pot{ - pin: val, - size: 1, - po: po, + i++ + if i < len(t.bins) { + bins = append(bins, t.bins[i:]...) } + r = t.clone() + r.bins = bins + r.size = size + return r, po, found, true + } + // key does not exist + val = f(nil) + if val == nil { + // and it should not be created + return t, po, false, false + } + // otherwise check val if equal to k + if _, eq := pof(val, k, po); !eq { + panic("invalid value") + } + /// + size++ + p = &Pot{ + pin: val, + size: 1, + po: po, } bins := append([]*Pot{}, t.bins[:i]...) - if p.pin != nil { - bins = append(bins, p) - } + bins = append(bins, p) if i < len(t.bins) { - bins = append(bins, t.bins[i+1:]...) + bins = append(bins, t.bins[i:]...) } - r = &Pot{ - pin: f(t.pin), - size: size, - po: t.po, - bins: bins, - } - + r = t.clone() + r.bins = bins + r.size = size return r, po, found, true } @@ -394,6 +412,7 @@ func union(t0, t1 *Pot, pof Pof) (*Pot, int) { var bins []*Pot var mis []int wg := &sync.WaitGroup{} + wg.Add(1) pin0 := t0.pin pin1 := t1.pin bins0 := t0.bins @@ -445,11 +464,12 @@ func union(t0, t1 *Pot, pof Pof) (*Pot, int) { bins = append(bins, nil) ml := len(mis) mis = append(mis, 0) - wg.Add(1) - go func(b, m int, m0, m1 *Pot) { - defer wg.Done() - bins[b], mis[m] = union(m0, m1, pof) - }(bl, ml, n0, n1) + // wg.Add(1) + // go func(b, m int, m0, m1 *Pot) { + // defer wg.Done() + // bins[b], mis[m] = union(m0, m1, pof) + // }(bl, ml, n0, n1) + bins[bl], mis[ml] = union(n0, n1, pof) i0++ i1++ n0 = nil @@ -499,6 +519,7 @@ func union(t0, t1 *Pot, pof Pof) (*Pot, int) { } + wg.Done() wg.Wait() for _, c := range mis { common += c @@ -534,6 +555,9 @@ func (t *Pot) each(f func(Val, int) bool) bool { return false } } + if t.size == 0 { + return false + } return f(t.pin, t.po) } diff --git a/pot/pot_test.go b/pot/pot_test.go index bab923a0ea..b41c64b076 100644 --- a/pot/pot_test.go +++ b/pot/pot_test.go @@ -30,26 +30,19 @@ import ( const ( maxEachNeighbourTests = 420 maxEachNeighbour = 420 + maxSwap = 420 + maxSwapTests = 420 ) -func init() { - // log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(false)))) -} +// func init() { +// log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(false)))) +// } type testAddr struct { a []byte i int } -// -// func newTestAddr(s string, i int) *testAddr { -// b := NewAddressFromString(s) -// h := &common.Hash{} -// copy((*h)[:], b) -// a := Address(*h) -// return &testAddr{&a, i} -// } - func newTestAddr(s string, i int) *testAddr { return &testAddr{NewAddressFromString(s), i} } @@ -60,7 +53,6 @@ func (a *testAddr) Address() []byte { func (a *testAddr) String() string { return Label(a.a) - // return a.Address.String()[:keylen] } func randomTestAddr(n int, i int) *testAddr { @@ -86,12 +78,10 @@ func indexes(t *Pot) (i []int, po []int) { func testAdd(t *Pot, pof Pof, j int, values ...string) (_ *Pot, n int, f bool) { for i, val := range values { t, n, f = Add(t, newTestAddr(val, i+j), pof) - // t.Add(newTestAddr(val, i+n)) } return t, n, f } -// func RandomBoolAddress() func TestPotAdd(t *testing.T) { pof := DefaultPof(8) n := NewPot(newTestAddr("00111100", 0), 0) @@ -128,7 +118,6 @@ func TestPotAdd(t *testing.T) { } } -// func RandomBoolAddress() func TestPotRemove(t *testing.T) { pof := DefaultPof(8) n := NewPot(newTestAddr("00111100", 0), 0) @@ -168,61 +157,69 @@ func TestPotRemove(t *testing.T) { } func TestPotSwap(t *testing.T) { - pof := DefaultPof(8) - max := maxEachNeighbour - n := NewPot(nil, 0) - var m []*testAddr - var found bool - for j := 0; j < 2*max; { - v := randomtestAddr(keylen, j) - n, _, found = Add(n, v, pof) - if !found { - m = append(m, v) - j++ - } - } - k := make(map[string]*testAddr) - for j := 0; j < max; { - v := randomtestAddr(keylen, 1) - _, found := k[Label(v)] - if !found { - k[Label(v)] = v - j++ - } - } - for _, v := range k { - m = append(m, v) - } - f := func(v Val) Val { - tv := v.(*testAddr) - if tv.i < max { - return nil - } - tv.i = 0 - return v - } - for _, val := range m { - n, _, _, _ = Swap(n, val, pof, func(v Val) Val { - if v == nil { - return val + for i := 0; i < maxSwapTests; i++ { + // alen := maxkeylen + alen := maxkeylen + pof := DefaultPof(alen) + max := rand.Intn(maxSwap) + + n := NewPot(nil, 0) + var m []*testAddr + var found bool + for j := 0; j < 2*max; { + v := randomtestAddr(alen, j) + n, _, found = Add(n, v, pof) + if !found { + m = append(m, v) + j++ } - return f(v) - }) - } - sum := 0 - n.Each(func(v Val, i int) bool { - sum++ - tv := v.(*testAddr) - if tv.i > 1 { - t.Fatalf("item value incorrect, expected 0, got %v", tv.i) } - return true - }) - if sum != 2*max { - t.Fatalf("incorrect number of elements. expected %v, got %v", 2*max, sum) - } - if sum != n.Size() { - t.Fatalf("incorrect size. expected %v, got %v", sum, n.Size()) + k := make(map[string]*testAddr) + for j := 0; j < max; { + v := randomtestAddr(alen, 1) + _, found := k[Label(v)] + if !found { + k[Label(v)] = v + j++ + } + } + for _, v := range k { + m = append(m, v) + } + f := func(v Val) Val { + tv := v.(*testAddr) + if tv.i < max { + return nil + } + tv.i = 0 + return v + } + for _, val := range m { + n, _, _, _ = Swap(n, val, pof, func(v Val) Val { + if v == nil { + return val + } + return f(v) + }) + } + sum := 0 + n.Each(func(v Val, i int) bool { + if v == nil { + return true + } + sum++ + tv := v.(*testAddr) + if tv.i > 1 { + t.Fatalf("item value incorrect, expected 0, got %v", tv.i) + } + return true + }) + if sum != 2*max { + t.Fatalf("incorrect number of elements. expected %v, got %v", 2*max, sum) + } + if sum != n.Size() { + t.Fatalf("incorrect size. expected %v, got %v", sum, n.Size()) + } } } @@ -238,7 +235,7 @@ func checkPo(val Val, pof Pof) func(Val, int) error { } func checkOrder(val Val) func(Val, int) error { - po := keylen + po := maxkeylen return func(v Val, p int) error { if po < p { return fmt.Errorf("incorrect order for item %v in neighbour iteration for %v. PO %v > %v (previous max)", v, val, p, po) @@ -291,31 +288,15 @@ const ( mergeTestChoose = 5 ) -// -// func TestPotMergeOne(t *testing.T) { -// pot1 := NewPot(nil, 0, DefaultPof(2)) -// pot1.Add(newTestAddr("10", 0)) -// pot1.Add(newTestAddr("00", 0)) -// pot2 := NewPot(nil, 0, DefaultPof(2)) -// pot2.Add(newTestAddr("01", 0)) -// pot1.Merge(pot2) -// count := 0 -// pot1.Each(func(val Val, i int) bool { -// count++ -// return true -// }) -// if count != 3 { -// t.Fatalf("expected count to be 3, got %d\n%v\n%v", count, pot2, pot1) -// } -// } - func TestPotMergeCommon(t *testing.T) { vs := make([]*testAddr, mergeTestCount) - pof := DefaultPof(maxkeylen) for i := 0; i < maxEachNeighbourTests; i++ { + alen := maxkeylen + // alen := maxkeylen + pof := DefaultPof(alen) for j := 0; j < len(vs); j++ { - vs[j] = randomtestAddr(keylen, j) + vs[j] = randomtestAddr(alen, j) } max0 := rand.Intn(mergeTestChoose) + 1 max1 := rand.Intn(mergeTestChoose) + 1 @@ -369,17 +350,18 @@ func TestPotMergeCommon(t *testing.T) { t.Fatalf("%v: merged pot contains duplicates: \n%v", i, n) } for k := range m { - n, _, found := Add(n, newTestAddr(k, 0), pof) + _, _, found = Add(n, newTestAddr(k, 0), pof) if !found { - t.Fatalf("%v: merged pot (size:%v, added: %v) missing element %v\n%v", i, size, added, k, n) + t.Fatalf("%v: merged pot (size:%v, added: %v) missing element %v", i, size, added, k) } } } } func TestPotMergeScale(t *testing.T) { - pof := DefaultPof(maxkeylen) for i := 0; i < maxEachNeighbourTests; i++ { + alen := maxkeylen + pof := DefaultPof(alen) max0 := rand.Intn(maxEachNeighbour) + 1 max1 := rand.Intn(maxEachNeighbour) + 1 n0 := NewPot(nil, 0) @@ -388,10 +370,8 @@ func TestPotMergeScale(t *testing.T) { m := make(map[string]bool) var found bool for j := 0; j < max0; { - v := randomtestAddr(keylen, j) - // v := randomtestAddr(keylen, j) + v := randomtestAddr(alen, j) n0, _, found = Add(n0, v, pof) - // _, found := n0.Add(v) if !found { m[Label(v)] = false j++ @@ -400,10 +380,8 @@ func TestPotMergeScale(t *testing.T) { expAdded := 0 for j := 0; j < max1; { - v := randomtestAddr(keylen, j) - // v := randomtestAddr(keylen, j) + v := randomtestAddr(alen, j) n1, _, found = Add(n1, v, pof) - // _, found := n1.Add(v) if !found { j++ } @@ -425,18 +403,18 @@ func TestPotMergeScale(t *testing.T) { size := n.Size() if expSize != size { - t.Fatalf("%v: incorrect number of elements in merged pot, expected %v, got %v\n%v", i, expSize, size, n) + t.Fatalf("%v: incorrect number of elements in merged pot, expected %v, got %v", i, expSize, size) } if expAdded != added { t.Fatalf("%v: incorrect number of added elements in merged pot, expected %v, got %v", i, expAdded, added) } if !checkDuplicates(n) { - t.Fatalf("%v: merged pot contains duplicates: \n%v", i, n) + t.Fatalf("%v: merged pot contains duplicates", i) } for k := range m { - n, _, found := Add(n, newTestAddr(k, 0), pof) + _, _, found = Add(n, newTestAddr(k, 0), pof) if !found { - t.Fatalf("%v: merged pot (size:%v, added: %v) missing element %v\n%v", i, size, added, k, n) + t.Fatalf("%v: merged pot (size:%v, added: %v) missing element %v", i, size, added, k) } } } @@ -457,15 +435,16 @@ func checkDuplicates(t *Pot) bool { } func TestPotEachNeighbourSync(t *testing.T) { - pof := DefaultPof(maxkeylen) for i := 0; i < maxEachNeighbourTests; i++ { + alen := maxkeylen + pof := DefaultPof(maxkeylen) max := rand.Intn(maxEachNeighbour/2) + maxEachNeighbour/2 - pin := randomTestAddr(keylen, 0) + pin := randomTestAddr(alen, 0) n := NewPot(pin, 0) m := make(map[string]bool) m[Label(pin)] = false for j := 1; j <= max; j++ { - v := randomTestAddr(keylen, j) + v := randomTestAddr(alen, j) n, _, _ = Add(n, v, pof) m[Label(v)] = false } @@ -475,13 +454,13 @@ func TestPotEachNeighbourSync(t *testing.T) { continue } count := rand.Intn(size/2) + size/2 - val := randomTestAddr(keylen, max+1) + val := randomTestAddr(alen, max+1) log.Trace(fmt.Sprintf("%v: pin: %v, size: %v, val: %v, count: %v", i, n.Pin(), size, val, count)) err := testPotEachNeighbour(n, pof, val, count, checkPo(val, pof), checkOrder(val), checkValues(m, val)) if err != nil { t.Fatal(err) } - minPoFound := keylen + minPoFound := alen maxPoNotFound := 0 for k, found := range m { po, _ := pof(val, newTestAddr(k, 0), 0) @@ -502,14 +481,15 @@ func TestPotEachNeighbourSync(t *testing.T) { } func TestPotEachNeighbourAsync(t *testing.T) { - pof := DefaultPof(maxkeylen) for i := 0; i < maxEachNeighbourTests; i++ { max := rand.Intn(maxEachNeighbour/2) + maxEachNeighbour/2 - n := NewPot(randomTestAddr(keylen, 0), 0) + alen := maxkeylen + pof := DefaultPof(alen) + n := NewPot(randomTestAddr(alen, 0), 0) size := 1 var found bool for j := 1; j <= max; j++ { - v := randomTestAddr(keylen, j) + v := randomTestAddr(alen, j) n, _, found = Add(n, v, pof) if !found { size++ @@ -522,11 +502,11 @@ func TestPotEachNeighbourAsync(t *testing.T) { continue } count := rand.Intn(size/2) + size/2 - val := randomTestAddr(keylen, max+1) + val := randomTestAddr(alen, max+1) mu := sync.Mutex{} m := make(map[string]bool) - maxPos := rand.Intn(keylen) + maxPos := rand.Intn(alen) log.Trace(fmt.Sprintf("%v: pin: %v, size: %v, val: %v, count: %v, maxPos: %v", i, n.Pin(), size, val, count, maxPos)) msize := 0 remember := func(v Val, po int) error { @@ -561,12 +541,13 @@ func TestPotEachNeighbourAsync(t *testing.T) { func benchmarkEachNeighbourSync(t *testing.B, max, count int, d time.Duration) { t.ReportAllocs() - pof := DefaultPof(maxkeylen) - pin := randomTestAddr(keylen, 0) + alen := maxkeylen + pof := DefaultPof(alen) + pin := randomTestAddr(alen, 0) n := NewPot(pin, 0) var found bool for j := 1; j <= max; { - v := randomTestAddr(keylen, j) + v := randomTestAddr(alen, j) n, _, found = Add(n, v, pof) if !found { j++ @@ -574,7 +555,7 @@ func benchmarkEachNeighbourSync(t *testing.B, max, count int, d time.Duration) { } t.ResetTimer() for i := 0; i < t.N; i++ { - val := randomTestAddr(keylen, max+1) + val := randomTestAddr(alen, max+1) m := 0 n.EachNeighbour(val, pof, func(v Val, po int) bool { time.Sleep(d) @@ -588,17 +569,17 @@ func benchmarkEachNeighbourSync(t *testing.B, max, count int, d time.Duration) { t.StopTimer() stats := new(runtime.MemStats) runtime.ReadMemStats(stats) - // fmt.Println(stats.Sys) } func benchmarkEachNeighbourAsync(t *testing.B, max, count int, d time.Duration) { t.ReportAllocs() - pof := DefaultPof(maxkeylen) - pin := randomTestAddr(keylen, 0) + alen := maxkeylen + pof := DefaultPof(alen) + pin := randomTestAddr(alen, 0) n := NewPot(pin, 0) var found bool for j := 1; j <= max; { - v := randomTestAddr(keylen, j) + v := randomTestAddr(alen, j) n, _, found = Add(n, v, pof) if !found { j++ @@ -606,15 +587,14 @@ func benchmarkEachNeighbourAsync(t *testing.B, max, count int, d time.Duration) } t.ResetTimer() for i := 0; i < t.N; i++ { - val := randomTestAddr(keylen, max+1) - n.EachNeighbourAsync(val, pof, count, keylen, func(v Val, po int) { + val := randomTestAddr(alen, max+1) + n.EachNeighbourAsync(val, pof, count, alen, func(v Val, po int) { time.Sleep(d) }, true) } t.StopTimer() stats := new(runtime.MemStats) runtime.ReadMemStats(stats) - // fmt.Println(stats.Sys) } func BenchmarkEachNeighbourSync_3_1_0(t *testing.B) { diff --git a/swarm/network/hive_test.go b/swarm/network/hive_test.go index f833919ed4..c621e97241 100644 --- a/swarm/network/hive_test.go +++ b/swarm/network/hive_test.go @@ -17,7 +17,7 @@ func newHiveTester(t *testing.T, params *HiveParams) (*bzzTester, *Hive) { } func TestRegisterAndConnect(t *testing.T) { - t.Skip("deadlocked") + //t.Skip("deadlocked") params := NewHiveParams() s, pp := newHiveTester(t, params) defer s.Stop() @@ -47,6 +47,11 @@ func TestRegisterAndConnect(t *testing.T) { s.TestExchanges(p2ptest.Exchange{ Label: "getPeersMsg message", Expects: []p2ptest.Expect{ + p2ptest.Expect{ + Code: 2, + Msg: &subPeersMsg{0}, + Peer: id, + }, p2ptest.Expect{ Code: 1, Msg: &getPeersMsg{uint8(o), 5}, From 1b6f480eba74812e3cbad21efc542c87b5843b43 Mon Sep 17 00:00:00 2001 From: nolash Date: Sat, 10 Jun 2017 01:12:40 +0200 Subject: [PATCH 07/17] cmd/swarm, swarm/pss: pss snap, docs, pot dup, kad hang WIP introduce snapshots in pss cmd can now set websocket host for pss removed binary log garbage moved baseaddr get to main pss api changed pssclient constructor sig to return error documentation --- cmd/swarm/main.go | 10 +- .../simulations/discovery/discovery_test.go | 8 + swarm/pss/README.md | 185 ++++++++++ swarm/pss/client/README.md | 83 +++++ swarm/pss/client/client.go | 160 +++++++-- swarm/pss/client/client_test.go | 5 +- swarm/pss/{common.go => ping.go} | 1 + swarm/pss/protocols/chat/protocol.go | 98 ----- swarm/pss/pss.go | 285 ++++++++++++--- swarm/pss/pss_test.go | 340 +++++++++++------- swarm/pss/pssapi.go | 41 ++- swarm/pss/simulations/service_test.go | 6 +- swarm/pss/types.go | 62 ++-- swarm/swarm_psschat.go | 47 +-- 14 files changed, 955 insertions(+), 376 deletions(-) create mode 100644 swarm/pss/README.md create mode 100644 swarm/pss/client/README.md rename swarm/pss/{common.go => ping.go} (98%) delete mode 100644 swarm/pss/protocols/chat/protocol.go diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index e6da14c83e..fedb413d60 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -121,6 +121,11 @@ var ( Name: "pss", Usage: "Enable pss (message passing over swarm)", } + PssHostFlag = cli.StringFlag{ + Name: "psshost", + Usage: fmt.Sprintf("Websockets host or ip for pss (default '%v')", node.DefaultWSHost), + Value: node.DefaultWSHost, + } PssPortFlag = cli.IntFlag{ Name: "pssport", Usage: fmt.Sprintf("Websockets port for pss (default %d)", node.DefaultWSPort), @@ -269,6 +274,7 @@ Cleans database of corrupted entries. SwarmUpFromStdinFlag, SwarmUploadMimeType, // pss flags + PssHostFlag, PssEnabledFlag, PssPortFlag, } @@ -307,7 +313,7 @@ func version(ctx *cli.Context) error { func bzzd(ctx *cli.Context) error { cfg := defaultNodeConfig if ctx.GlobalIsSet(PssEnabledFlag.Name) { - cfg.WSHost = "127.0.0.1" + cfg.WSHost = ctx.GlobalString(PssHostFlag.Name) cfg.WSModules = []string{"eth","pss"} cfg.WSOrigins = []string{"*"} if ctx.GlobalIsSet(PssPortFlag.Name) { @@ -366,7 +372,7 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { swapEnabled := ctx.GlobalBool(SwarmSwapEnabledFlag.Name) syncEnabled := ctx.GlobalBoolT(SwarmSyncEnabledFlag.Name) pssEnabled := ctx.GlobalBool(PssEnabledFlag.Name) - + ethapi := ctx.GlobalString(EthAPIFlag.Name) cors := ctx.GlobalString(CorsStringFlag.Name) diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go index 7bfb2aef72..d91d8ba33e 100644 --- a/swarm/network/simulations/discovery/discovery_test.go +++ b/swarm/network/simulations/discovery/discovery_test.go @@ -2,6 +2,7 @@ package discovery_test import ( "context" + "encoding/json" "fmt" "io/ioutil" "os" @@ -135,6 +136,13 @@ func testDiscoverySimulation(t *testing.T, adapter adapters.NodeAdapter) { t.Fatalf("simulation failed: %s", result.Error) } + snap, err := net.Snapshot() + if err != nil { + t.Fatalf("no shapshot dude") + } + jsonsnapshot, err := json.Marshal(snap) + ioutil.WriteFile("jsonsnapshot.txt", jsonsnapshot, os.ModePerm) + t.Log("Simulation Passed:") t.Logf("Duration: %s", result.FinishedAt.Sub(result.StartedAt)) for _, id := range ids { diff --git a/swarm/pss/README.md b/swarm/pss/README.md new file mode 100644 index 0000000000..3577c905d0 --- /dev/null +++ b/swarm/pss/README.md @@ -0,0 +1,185 @@ +# Postal Service over Swarm + +pss provides devp2p functionality for swarm nodes without the need for a direct tcp connection between them. + +It uses swarm kademlia routing to send and receive messages. Routing is deterministic and will seek the shortest route available on the network. + +Messages are encapsulated in a devp2p message structure `PssMsg`. These capsules are forwarded from node to node using ordinary tcp devp2p until it reaches it's destination. The destination address is hinted in `PssMsg.To` + +The content of a PssMsg can be anything at all, down to a simple, non-descript byte-slices. But convenience methods are made available to implement devp2p protocol functionality on top of it. + +In its final implementation, pss is intended to become "shh over bzz," that is; "whisper over swarm." Specifically, this means that the emphemeral encryption envelopes of whisper will be used to obfuscate the correspondance. Ideally, the unencrypted content of the PssMsg will only contain a part of the address of the recipient, where the final recipient is the one who matches this partial address *and* successfully can encrypt the message. + +For the current state and roadmap of pss development please see https://github.com/ethersphere/swarm/wiki/swarm-dev-progress. + +Please report issues on https://github.com/ethersphere/go-ethereum + +Feel free to ask questions in https://gitter.im/ethersphere/pss + +## TL;DR IMPLEMENTATION + +Most developers will most probably want to use the protocol-wrapping convenience client in swarm/pss/client. Documentation and a minimal code example for the latter is found in the package documentation. The pss API can of course also be used directly. The client implementation provides a clear illustration of its intended usage. + +pss implements the node.Service interface. This means that the API methods will be auto-magically exposed to any RPC layer the node activates. In particular, pss provides subscription to incoming messages using the go-ethereum rpc websocket layer. + +The important API methods are: +- Receive() - start a subscription to receive new incoming messages matching specific "topics" +- Send() - send content over pss to a specified recipient + + +## LOWLEVEL IMPLEMENTATION + +code speaks louder than words: + + import ( + "io/ioutil" + "os" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/pss" + "github.com/ethereum/go-ethereum/swarm/network" + "github.com/ethereum/go-ethereum/swarm/storage" + ) + + var ( + righttopic = pss.NewTopic("foo", 4) + wrongtopic = pss.NewTopic("bar", 2) + ) + + // if you want to see what's going on + func init() { + hs := log.StreamHandler(os.Stderr, log.TerminalFormat(true)) + hf := log.LvlFilterHandler(log.LvlTrace, hs) + h := log.CallerFileHandler(hf) + log.Root().SetHandler(h) + } + + + // Pss.Handler type + func handler(msg []byte, p *p2p.Peer, from []byte) error { + log.Debug("received", "msg", msg, "from", from, "forwarder", p.ID()) + return nil + } + + func implementation() { + + // bogus addresses for illustration purposes + meaddr := network.RandomAddr() + toaddr := network.RandomAddr() + fwdaddr := network.RandomAddr() + + // new kademlia for routing + kp := network.NewKadParams() + to := network.NewKademlia(meaddr.Over(), kp) + + // new (local) storage for cache + cachedir, err := ioutil.TempDir("", "pss-cache") + if err != nil { + panic("overlay") + } + dpa, err := storage.NewLocalDPA(cachedir) + if err != nil { + panic("storage") + } + + // setup pss + psp := pss.NewPssParams(false) + ps := pss.NewPss(to, dpa, psp) + + // does nothing but please include it + ps.Start(nil) + + dereg := ps.Register(&righttopic, handler) + + // in its simplest form a message is just a byteslice + payload := []byte("foobar") + + // send a raw message + err = ps.SendRaw(toaddr.Over(), righttopic, payload) + log.Error("Fails. Not connect, so nothing in kademlia. But it illustrates the point.", "err", err) + + // forward a full message + envfwd := pss.NewEnvelope(fwdaddr.Over(), righttopic, payload) + msgfwd := &pss.PssMsg{ + To: toaddr.Over(), + Payload: envfwd, + } + err = ps.Forward(msgfwd) + log.Error("Also fails, same reason. I wish, I wish, I wish there was somebody out there.", "err", err) + + // process an incoming message + // (this is the first step after the devp2p PssMsg message handler) + envme := pss.NewEnvelope(toaddr.Over(), righttopic, payload) + msgme := &pss.PssMsg{ + To: meaddr.Over(), + Payload: envme, + } + err = ps.Process(msgme) + if err == nil { + log.Info("this works :)") + } + + // if we don't have a registered topic it fails + dereg() // remove the previously registered topic-handler link + ps.Process(msgme) + log.Error("It fails as we expected", "err", err) + + // does nothing but please include it + ps.Stop() + } + +## MESSAGE STRUCTURE + +NOTE! This part is subject to change. In particular the envelope structure will be re-implemented using whisper. + +A pss message has the following layers: + +- PssMsg + Contains (eventually only part of) recipient address, and (eventually) encrypted Envelope. + +- Envelope + Currently rlp-encoded. Contains the Payload, along with sender address, topic and expiry information. + +- Payload + Byte-slice of arbitrary data + +- ProtocolMsg + An optional convenience structure for implementation of devp2p protocols. Contains Code, Size and Payload analogous to the p2p.Msg structure, where the payload is a rlp-encoded byteslice. For transport, this struct is serialized and used as the "payload" above. + +## TOPICS AND PROTOCOLS + +Pure pss is protocol agnostic. Instead it uses the notion of Topic. This is NOT the "subject" of a message. Instead this type is used to internally register handlers for messages matching respective Topics. + +Topic in this context virtually mean anything; protocols, chatrooms, or social media groups. + +When implementing devp2p protocols, topics are direct mappings to protocols name and version. The pss package provides the PssProtocol convenience structure, and a generic Handler that can be passed to Pss.Register. This makes it possible to use the same message handler code for pss that are used for direct connected peers. + +## CONNECTIONS + +A "connection" in pss is a purely virtual construct. There is no mechanisms in place to ensure that the remote peer actually is there. In fact, "adding" a peer involves merely the node's opinion that the peer is there. It may issue messages to that remote peer to a directly connected peer, which in turn passes it on. But if it is not present on the network - or if there is no route to it - the message will never reach its destination through mere forwarding. + +When implementing the devp2p protocol stack, the "adding" of a remote peer is a prerequisite for the side actually initiating the protocol communication. Adding a peer in effect "runs" the protocol on that peer, and adds an internal mapping between a topic and that peer. It also enables sending and receiving messages using the main io-construct in devp2p - the p2p.MsgReadWriter. + +Under the hood, pss implements its own MsgReadWriter, which bridges MsgReadWriter.WriteMsg with Pss.SendRaw, and deftly adds an InjectMsg method which pipes incoming messages to appear on the MsgReadWriter.ReadMsg channel. + +An incoming connection is nothing more than an actual PssMsg appearing with a certain Topic. If a Handler har been registered to that Topic, the message will be passed to it. This constitutes a "new" connection if: + +- The pss node never called AddPeer with this combination of remote peer address and topic, and + +- The pss node never received a PssMsg from this remote peer with this specific Topic before. + +If it is a "new" connection, the protocol will be "run" on the remote peer, in the same manner as if it was pre-emptively added. + +## ROUTING AND CACHING + +(please refer to swarm kademlia routing for an explanation of the routing algorithm used for pss) + +pss implements a simple caching mechanism, using the swarm DPA for storage of the messages and generation of the digest keys used in the cache table. The caching is intended to alleviate the following: + +- save messages so that they can be delivered later if the recipient was not online at the time of sending. + +- drop an identical message to the same recipient if received within a given time interval + +- prevent backwards routing of messages + +the latter may occur if only one entry is in the receiving node's kademlia. In this case the forwarder will be provided as the "nearest node" to the final recipient. The cache keeps the address of who the message was forwarded from, and if the cache lookup matches, the message will be dropped. diff --git a/swarm/pss/client/README.md b/swarm/pss/client/README.md new file mode 100644 index 0000000000..7d2cad1728 --- /dev/null +++ b/swarm/pss/client/README.md @@ -0,0 +1,83 @@ +# pss client + +simple abstraction for implementing pss functionality + +the pss client library aims to simplify usage of the p2p.protocols package over pss + +IO is performed using the ordinary p2p.MsgReadWriter interface, which transparently communicates with a pss node via RPC using websockets as transport layer, using methods in the PssAPI class in the swarm/pss package + +## USAGE + +Minimal-ish usage example. It needs a running pss-enabled swarm node to work. Please refer to the test files for more details. + + + import ( + "context" + "fmt" + "os" + pss "github.com/ethereum/go-ethereum/swarm/pss/client" + "github.com/ethereum/go-ethereum/p2p/protocols" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/pot" + "github.com/ethereum/go-ethereum/log" + ) + + type FooMsg struct { + Bar int + } + + + func fooHandler (msg interface{}) error { + foomsg, ok := msg.(*FooMsg) + if ok { + log.Debug("Yay, just got a message", "msg", foomsg) + } + return fmt.Errorf("Unknown message") + } + + spec := &protocols.Spec{ + Name: "foo", + Version: 1, + MaxMsgSize: 1024, + Messages: []interface{}{ + FooMsg{}, + }, + } + + proto := &p2p.Protocol{ + Name: spec.Name, + Version: spec.Version, + Length: uint64(len(spec.Messages)), + Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error { + pp := protocols.NewPeer(p, rw, spec) + return pp.Run(fooHandler) + }, + } + + func implementation() { + cfg := pss.NewClientConfig() + psc := pss.NewClient(context.Background(), nil, cfg) + err := psc.Start() + if err != nil { + log.Crit("can't start pss client") + os.Exit(1) + } + + log.Debug("connected to pss node", "bzz addr", psc.BaseAddr) + + err = psc.RunProtocol(proto) + if err != nil { + log.Crit("can't start protocol on pss websocket") + os.Exit(1) + } + + addr := pot.RandomAddress() // should be a real address, of course + psc.AddPssPeer(addr, spec) + + // use the protocol for something + + psc.Stop() + } + +BUG(test): TestIncoming test times out due to deadlock issues in the swarm hive + diff --git a/swarm/pss/client/client.go b/swarm/pss/client/client.go index 335316d3b2..0b3f7918ab 100644 --- a/swarm/pss/client/client.go +++ b/swarm/pss/client/client.go @@ -1,3 +1,82 @@ +// simple abstraction for implementing pss functionality +// +// the pss client library aims to simplify usage of the p2p.protocols package over pss +// +// IO is performed using the ordinary p2p.MsgReadWriter interface, which transparently communicates with a pss node via RPC using websockets as transport layer, using methods in the PssAPI class in the swarm/pss package +// +// +// Minimal-ish usage example (requires a running pss node with websocket RPC): +// +// +// import ( +// "context" +// "fmt" +// "os" +// pss "github.com/ethereum/go-ethereum/swarm/pss/client" +// "github.com/ethereum/go-ethereum/p2p/protocols" +// "github.com/ethereum/go-ethereum/p2p" +// "github.com/ethereum/go-ethereum/pot" +// "github.com/ethereum/go-ethereum/log" +// ) +// +// type FooMsg struct { +// Bar int +// } +// +// +// func fooHandler (msg interface{}) error { +// foomsg, ok := msg.(*FooMsg) +// if ok { +// log.Debug("Yay, just got a message", "msg", foomsg) +// } +// return fmt.Errorf("Unknown message") +// } +// +// spec := &protocols.Spec{ +// Name: "foo", +// Version: 1, +// MaxMsgSize: 1024, +// Messages: []interface{}{ +// FooMsg{}, +// }, +// } +// +// proto := &p2p.Protocol{ +// Name: spec.Name, +// Version: spec.Version, +// Length: uint64(len(spec.Messages)), +// Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error { +// pp := protocols.NewPeer(p, rw, spec) +// return pp.Run(fooHandler) +// }, +// } +// +// func implementation() { +// cfg := pss.NewClientConfig() +// psc := pss.NewClient(context.Background(), nil, cfg) +// err := psc.Start() +// if err != nil { +// log.Crit("can't start pss client") +// os.Exit(1) +// } +// +// log.Debug("connected to pss node", "bzz addr", psc.BaseAddr) +// +// err = psc.RunProtocol(proto) +// if err != nil { +// log.Crit("can't start protocol on pss websocket") +// os.Exit(1) +// } +// +// addr := pot.RandomAddress() // should be a real address, of course +// psc.AddPssPeer(addr, spec) +// +// // use the protocol for something +// +// psc.Stop() +// } +// +// BUG(test): TestIncoming test times out due to deadlock issues in the swarm hive package client import ( @@ -6,12 +85,12 @@ import ( "sync" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/protocols" "github.com/ethereum/go-ethereum/pot" + "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm/pss" @@ -26,24 +105,27 @@ const ( // RemoteHost: hostname of node running websockets proxy to pss (default localhost) // RemotePort: port of node running websockets proxy to pss (0 = go-ethereum node default) -// Secure: whether or not to use secure connection // SelfHost: local if host to connect from +// Secure: whether or not to use secure connection (not currently in use) type ClientConfig struct { - SelfHost string RemoteHost string RemotePort int + SelfHost string Secure bool } +// Generates a pss client configuration with default values func NewClientConfig() *ClientConfig { return &ClientConfig{ - SelfHost: "localhost", - RemoteHost: "localhost", - RemotePort: 8546, + SelfHost: node.DefaultWSHost, + RemoteHost: node.DefaultWSHost, + RemotePort: node.DefaultWSPort, } } +// After a successful connection with Client.Start, BaseAddr contains the swarm overlay address of the pss node type Client struct { + BaseAddr []byte localuri string remoteuri string ctx context.Context @@ -58,6 +140,7 @@ type Client struct { protos map[pss.Topic]*p2p.Protocol } +// implements p2p.MsgReadWriter type pssRPCRW struct { *Client topic *pss.Topic @@ -97,13 +180,17 @@ func (rw *pssRPCRW) WriteMsg(msg p2p.Msg) error { if err != nil { return err } + return rw.Client.ws.CallContext(rw.Client.ctx, nil, "pss_send", rw.topic, pss.APIMsg{ Addr: rw.addr.Bytes(), Msg: pmsg, }) + } -func NewClient(ctx context.Context, cancel func(), config *ClientConfig) *Client { +// Constructor for production-environment clients +// Performs sanity checks on configuration paramters and gets everything ready to connect to pss node +func NewClient(ctx context.Context, cancel func(), config *ClientConfig) (*Client, error) { prefix := "ws" if ctx == nil { @@ -129,24 +216,34 @@ func NewClient(ctx context.Context, cancel func(), config *ClientConfig) *Client pssc.remoteuri = fmt.Sprintf("%s://%s:%d", prefix, config.RemoteHost, config.RemotePort) pssc.localuri = fmt.Sprintf("%s://%s", prefix, config.SelfHost) - return pssc + return pssc, nil } -func NewClientWithRPC(ctx context.Context, client *rpc.Client) *Client { +// Constructor for test implementations +// The 'rpcclient' parameter allows passing a in-memory rpc client to act as the remote websocket RPC. +func NewClientWithRPC(ctx context.Context, rpcclient *rpc.Client) (*Client, error) { + var oaddr []byte + err := rpcclient.CallContext(ctx, &oaddr, "pss_baseAddr") + if err != nil { + return nil, fmt.Errorf("cannot get pss node baseaddress: %v", err) + } return &Client{ msgC: make(chan pss.APIMsg), quitC: make(chan struct{}), peerPool: make(map[pss.Topic]map[pot.Address]*pssRPCRW), protos: make(map[pss.Topic]*p2p.Protocol), - ws: client, + ws: rpcclient, ctx: ctx, - } + BaseAddr: oaddr, + }, nil } func (self *Client) shutdown() { self.cancel() } +// Connects to the websockets RPC +// Retrieves the swarm overlay address from the pss node func (self *Client) Start() error { if self.ws != nil { return nil @@ -157,11 +254,23 @@ func (self *Client) Start() error { return fmt.Errorf("Couldnt dial pss websocket: %v", err) } + var oaddr []byte + err = ws.CallContext(self.ctx, &oaddr, "pss_baseAddr") + if err != nil { + return err + } + self.ws = ws + self.BaseAddr = oaddr return nil } +// Mounts a new devp2p protcool on the pss connection +// the protocol is aliased as a "pss topic" +// uses normal devp2p Send and incoming message handler routines from the p2p/protocols package +// +// when an incoming message is received from a peer that is not yet known to the client, this peer object is instantiated, and the protocol is run on it. func (self *Client) RunProtocol(proto *p2p.Protocol) error { topic := pss.NewTopic(proto.Name, int(proto.Version)) msgC := make(chan pss.APIMsg) @@ -200,11 +309,13 @@ func (self *Client) RunProtocol(proto *p2p.Protocol) error { return nil } +// Always call this to ensure that we exit cleanly func (self *Client) Stop() error { self.cancel() return nil } +// Preemptively add a remote pss peer func (self *Client) AddPssPeer(addr pot.Address, spec *protocols.Spec) { topic := pss.NewTopic(spec.Name, int(spec.Version)) if self.peerPool[topic] == nil { @@ -219,31 +330,10 @@ func (self *Client) AddPssPeer(addr pot.Address, spec *protocols.Spec) { } } +// Remove a remote pss peer +// +// Note this doesn't actually currently drop the peer, but only remmoves the reference from the client's peer lookup table func (self *Client) RemovePssPeer(addr pot.Address, spec *protocols.Spec) { topic := pss.NewTopic(spec.Name, int(spec.Version)) delete(self.peerPool[topic], addr) } - -func (self *Client) SubscribeEvents(ch chan *p2p.PeerEvent) event.Subscription { - log.Error("PSS client handles events internally, use the read functions instead") - return nil -} - -func (self *Client) PeerCount() int { - return len(self.peerPool) -} - -func (self *Client) NodeInfo() *p2p.NodeInfo { - return nil -} - -func (self *Client) PeersInfo() []*p2p.PeerInfo { - return nil -} -func (self *Client) AddPeer(node *discover.Node) { - log.Error("Cannot add peer in PSS with discover.Node, need swarm overlay address") -} - -func (self *Client) RemovePeer(node *discover.Node) { - log.Error("Cannot remove peer in PSS with discover.Node, need swarm overlay address") -} diff --git a/swarm/pss/client/client_test.go b/swarm/pss/client/client_test.go index 115316501a..9f2dea8ef9 100644 --- a/swarm/pss/client/client_test.go +++ b/swarm/pss/client/client_test.go @@ -152,7 +152,10 @@ func newClient(t *testing.T, ctx context.Context, cancel func(), quitC chan stru conf := NewClientConfig() - pssclient := NewClient(ctx, cancel, conf) + pssclient, err := NewClient(ctx, cancel, conf) + if err != nil { + t.Fatalf(err.Error()) + } ps := pss.NewTestPss(nil) srv := rpc.NewServer() diff --git a/swarm/pss/common.go b/swarm/pss/ping.go similarity index 98% rename from swarm/pss/common.go rename to swarm/pss/ping.go index d6274bcd63..479fe7eee1 100644 --- a/swarm/pss/common.go +++ b/swarm/pss/ping.go @@ -27,6 +27,7 @@ func (self *Ping) PingHandler(msg interface{}) error { return nil } +// Sample protocol used for tests var PingProtocol = &protocols.Spec{ Name: "psstest", Version: 1, diff --git a/swarm/pss/protocols/chat/protocol.go b/swarm/pss/protocols/chat/protocol.go deleted file mode 100644 index a6badc66e8..0000000000 --- a/swarm/pss/protocols/chat/protocol.go +++ /dev/null @@ -1,98 +0,0 @@ -package chat - -import ( - "time" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/protocols" - "github.com/ethereum/go-ethereum/swarm/pss" - "github.com/ethereum/go-ethereum/swarm/network" -) - -const ( - ESendFail = iota -) - -var ( - chatConnString = map[int]string{ - ESendFail: "Send error", - } -) - -type ChatMsg struct { - Serial uint64 - Content []byte - Source string -} - -type chatPing struct { - Created time.Time - Pong bool -} - -type chatAck struct { - Seen time.Time - Serial uint64 -} - -var ChatProtocol = &protocols.Spec{ - Name: "pssChat", - Version: 1, - MaxMsgSize: 1024, - Messages: []interface{}{ - ChatMsg{}, chatPing{}, chatAck{}, - }, -} - -var ChatTopic = pss.NewTopic(ChatProtocol.Name, int(ChatProtocol.Version)) - -type ChatConn struct { - Addr []byte - E int -} - -func (c* ChatConn) Error() string { - return chatConnString[c.E] -} - -type ChatCtrl struct { - Peer *protocols.Peer - OutC chan interface{} - ConnC chan ChatConn - inC chan *ChatMsg - oAddr []byte - pingTX int - pingRX int - pingLast int -} - -func (self *ChatCtrl) chatHandler(msg interface{}) error { - Chatmsg, ok := msg.(*ChatMsg) - if ok { - if self.inC != nil { - self.inC <- Chatmsg - } - } - return nil -} - -func New(inC chan *ChatMsg, connC chan ChatConn, injectfunc func(*ChatCtrl)) *p2p.Protocol { -//func New(inC chan *ChatMsg, outC chan interface{}, connC chan ChatConn) *p2p.Protocol { - chatctrl := &ChatCtrl{ - inC: inC, - ConnC: connC, - } - return &p2p.Protocol{ - Name: ChatProtocol.Name, - Version: ChatProtocol.Version, - Length: 3, - Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error { - peerid := p.ID() - pp := protocols.NewPeer(p, rw, ChatProtocol) - chatctrl.Peer = pp - chatctrl.oAddr = network.ToOverlayAddr(peerid[:]) - injectfunc(chatctrl) - pp.Run(chatctrl.chatHandler) - return nil - }, - } -} diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index fe03f7a0ed..ccf7ad03b6 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -1,3 +1,185 @@ +// pss provides devp2p functionality for swarm nodes without the need for a direct tcp connection between them. +// +// It uses swarm kademlia routing to send and receive messages. Routing is deterministic and will seek the shortest route available on the network. +// +// Messages are encapsulated in a devp2p message structure `PssMsg`. These capsules are forwarded from node to node using ordinary tcp devp2p until it reaches it's destination. The destination address is hinted in `PssMsg.To` +// +// The content of a PssMsg can be anything at all, down to a simple, non-descript byte-slices. But convenience methods are made available to implement devp2p protocol functionality on top of it. +// +// In its final implementation, pss is intended to become "shh over bzz," that is; "whisper over swarm." Specifically, this means that the emphemeral encryption envelopes of whisper will be used to obfuscate the correspondance. Ideally, the unencrypted content of the PssMsg will only contain a part of the address of the recipient, where the final recipient is the one who matches this partial address *and* successfully can encrypt the message. +// +// For the current state and roadmap of pss development please see https://github.com/ethersphere/swarm/wiki/swarm-dev-progress. +// +// Please report issues on https://github.com/ethersphere/go-ethereum +// +// Feel free to ask questions in https://gitter.im/ethersphere/pss +// +// TLDR IMPLEMENTATION +// +// Most developers will most probably want to use the protocol-wrapping convenience client in swarm/pss/client. Documentation and a minimal code example for the latter is found in the package documentation. The pss API can of course also be used directly. The client implementation provides a clear illustration of its intended usage. +// +// pss implements the node.Service interface. This means that the API methods will be auto-magically exposed to any RPC layer the node activates. In particular, pss provides subscription to incoming messages using the go-ethereum rpc websocket layer. +// +// The important API methods are: +// - Receive() - start a subscription to receive new incoming messages matching specific "topics" +// - Send() - send content over pss to a specified recipient +// +// +// LOWLEVEL IMPLEMENTATION +// +// code speaks louder than words: +// +// import ( +// "io/ioutil" +// "os" +// "github.com/ethereum/go-ethereum/p2p" +// "github.com/ethereum/go-ethereum/log" +// "github.com/ethereum/go-ethereum/swarm/pss" +// "github.com/ethereum/go-ethereum/swarm/network" +// "github.com/ethereum/go-ethereum/swarm/storage" +// ) +// +// var ( +// righttopic = pss.NewTopic("foo", 4) +// wrongtopic = pss.NewTopic("bar", 2) +// ) +// +// func init() { +// hs := log.StreamHandler(os.Stderr, log.TerminalFormat(true)) +// hf := log.LvlFilterHandler(log.LvlTrace, hs) +// h := log.CallerFileHandler(hf) +// log.Root().SetHandler(h) +// } +// +// +// // Pss.Handler type +// func handler(msg []byte, p *p2p.Peer, from []byte) error { +// log.Debug("received", "msg", msg, "from", from, "forwarder", p.ID()) +// return nil +// } +// +// func implementation() { +// +// // bogus addresses for illustration purposes +// meaddr := network.RandomAddr() +// toaddr := network.RandomAddr() +// fwdaddr := network.RandomAddr() +// +// // new kademlia for routing +// kp := network.NewKadParams() +// to := network.NewKademlia(meaddr.Over(), kp) +// +// // new (local) storage for cache +// cachedir, err := ioutil.TempDir("", "pss-cache") +// if err != nil { +// panic("overlay") +// } +// dpa, err := storage.NewLocalDPA(cachedir) +// if err != nil { +// panic("storage") +// } +// +// // setup pss +// psp := pss.NewPssParams(false) +// ps := pss.NewPss(to, dpa, psp) +// +// // does nothing but please include it +// ps.Start(nil) +// +// dereg := ps.Register(&righttopic, handler) +// +// // in its simplest form a message is just a byteslice +// payload := []byte("foobar") +// +// // send a raw message +// err = ps.SendRaw(toaddr.Over(), righttopic, payload) +// log.Error("Fails. Not connect, so nothing in kademlia. But it illustrates the point.", "err", err) +// +// // forward a full message +// envfwd := pss.NewEnvelope(fwdaddr.Over(), righttopic, payload) +// msgfwd := &pss.PssMsg{ +// To: toaddr.Over(), +// Payload: envfwd, +// } +// err = ps.Forward(msgfwd) +// log.Error("Also fails, same reason. I wish, I wish, I wish there was somebody out there.", "err", err) +// +// // process an incoming message +// // (this is the first step after the devp2p PssMsg message handler) +// envme := pss.NewEnvelope(toaddr.Over(), righttopic, payload) +// msgme := &pss.PssMsg{ +// To: meaddr.Over(), +// Payload: envme, +// } +// err = ps.Process(msgme) +// if err == nil { +// log.Info("this works :)") +// } +// +// // if we don't have a registered topic it fails +// dereg() // remove the previously registered topic-handler link +// ps.Process(msgme) +// log.Error("It fails as we expected", "err", err) +// +// // does nothing but please include it +// ps.Stop() +// } +// +// MESSAGE STRUCTURE +// +// NOTE! This part is subject to change. In particular the envelope structure will be re-implemented using whisper. +// +// A pss message has the following layers: +// +// PssMsg +// Contains (eventually only part of) recipient address, and (eventually) encrypted Envelope. +// +// Envelope +// Currently rlp-encoded. Contains the Payload, along with sender address, topic and expiry information. +// +// Payload +// Byte-slice of arbitrary data +// +// ProtocolMsg +// An optional convenience structure for implementation of devp2p protocols. Contains Code, Size and Payload analogous to the p2p.Msg structure, where the payload is a rlp-encoded byteslice. For transport, this struct is serialized and used as the "payload" above. +// +// TOPICS AND PROTOCOLS +// +// Pure pss is protocol agnostic. Instead it uses the notion of Topic. This is NOT the "subject" of a message. Instead this type is used to internally register handlers for messages matching respective Topics. +// +// Topic in this context virtually mean anything; protocols, chatrooms, or social media groups. +// +// When implementing devp2p protocols, topics are direct mappings to protocols name and version. The pss package provides the PssProtocol convenience structure, and a generic Handler that can be passed to Pss.Register. This makes it possible to use the same message handler code for pss that are used for direct connected peers. +// +// CONNECTIONS +// +// A "connection" in pss is a purely virtual construct. There is no mechanisms in place to ensure that the remote peer actually is there. In fact, "adding" a peer involves merely the node's opinion that the peer is there. It may issue messages to that remote peer to a directly connected peer, which in turn passes it on. But if it is not present on the network - or if there is no route to it - the message will never reach its destination through mere forwarding. +// +// When implementing the devp2p protocol stack, the "adding" of a remote peer is a prerequisite for the side actually initiating the protocol communication. Adding a peer in effect "runs" the protocol on that peer, and adds an internal mapping between a topic and that peer. It also enables sending and receiving messages using the main io-construct in devp2p - the p2p.MsgReadWriter. +// +// Under the hood, pss implements its own MsgReadWriter, which bridges MsgReadWriter.WriteMsg with Pss.SendRaw, and deftly adds an InjectMsg method which pipes incoming messages to appear on the MsgReadWriter.ReadMsg channel. +// +// An incoming connection is nothing more than an actual PssMsg appearing with a certain Topic. If a Handler har been registered to that Topic, the message will be passed to it. This constitutes a "new" connection if: +// +// - The pss node never called AddPeer with this combination of remote peer address and topic, and +// +// - The pss node never received a PssMsg from this remote peer with this specific Topic before. +// +// If it is a "new" connection, the protocol will be "run" on the remote peer, in the same manner as if it was pre-emptively added. +// +// ROUTING AND CACHING +// +// (please refer to swarm kademlia routing for an explanation of the routing algorithm used for pss) +// +// pss implements a simple caching mechanism, using the swarm DPA for storage of the messages and generation of the digest keys used in the cache table. The caching is intended to alleviate the following: +// +// - save messages so that they can be delivered later if the recipient was not online at the time of sending. +// +// - drop an identical message to the same recipient if received within a given time interval +// +// - prevent backwards routing of messages +// +// the latter may occur if only one entry is in the receiving node's kademlia. In this case the forwarder will be provided as the "nearest node" to the final recipient. The cache keeps the address of who the message was forwarded from, and if the cache lookup matches, the message will be dropped. package pss import ( @@ -20,23 +202,24 @@ import ( ) const ( - TopicResolverLength = 8 - PssPeerCapacity = 256 - PssPeerTopicDefaultCapacity = 8 - digestLength = 32 - digestCapacity = 256 + PssPeerCapacity = 256 // limit of peers kept in cache. (not implemented) + PssPeerTopicDefaultCapacity = 8 // limit of topics kept per peer. (not implemented) + digestLength = 32 // byte length of digest used for pss cache (currently same as swarm chunk hash) + digestCapacity = 256 // cache entry limit (not implement) ) var ( errorForwardToSelf = errors.New("forward to self") ) +// abstraction to enable access to p2p.protocols.Peer.Send type senderPeer interface { ID() discover.NodeID Address() []byte Send(interface{}) error } +// protocol specification of the pss capsule var pssSpec = &protocols.Spec{ Name: "pss", Version: 1, @@ -53,24 +236,12 @@ type pssCacheEntry struct { type pssDigest [digestLength]byte -// implements node.Service +// Toplevel pss object, taking care of message sending and receiving, message handler dispatchers and message forwarding. // -// pss provides sending messages to nodes without having to be directly connected to them. -// -// The messages are wrapped in a PssMsg structure and routed using the swarm kademlia routing. -// -// The top-level Pss object provides: -// -// - access to the swarm overlay and routing (kademlia) -// - a collection of remote overlay addresses mapped to MsgReadWriters, representing the virtually connected peers -// - a collection of remote underlay address, mapped to the overlay addresses above -// - a method to send a message to specific overlayaddr -// - a dispatcher lookup, mapping protocols to topics -// - a message cache to spot messages that previously have been forwarded +// Implements node.Service type Pss struct { network.Overlay // we can get the overlayaddress from this peerPool map[pot.Address]map[Topic]p2p.MsgReadWriter // keep track of all virtual p2p.Peers we are currently speaking to - //fwdPool map[pot.Address]*protocols.Peer // keep track of all peers sitting on the pssmsg routing layer fwdPool map[discover.NodeID]*protocols.Peer // keep track of all peers sitting on the pssmsg routing layer handlers map[Topic]map[*Handler]bool // topic and version based pss payload handlers fwdcache map[pssDigest]pssCacheEntry // checksum of unique fields from pssmsg mapped to expiry, cache to determine whether to drop msg @@ -83,7 +254,7 @@ type Pss struct { func (self *Pss) storeMsg(msg *PssMsg) (pssDigest, error) { swg := &sync.WaitGroup{} wwg := &sync.WaitGroup{} - buf := bytes.NewReader(msg.Serialize()) + buf := bytes.NewReader(msg.serialize()) key, err := self.dpa.Store(buf, int64(buf.Len()), swg, wwg) if err != nil { log.Warn("Could not store in swarm", "err", err) @@ -95,12 +266,13 @@ func (self *Pss) storeMsg(msg *PssMsg) (pssDigest, error) { return digest, nil } -// Creates a new Pss instance. A node should only need one of these +// Creates a new Pss instance. +// +// Needs a swarm network overlay, a DPA storage for message cache storage. func NewPss(k network.Overlay, dpa *storage.DPA, params *PssParams) *Pss { return &Pss{ Overlay: k, peerPool: make(map[pot.Address]map[Topic]p2p.MsgReadWriter, PssPeerCapacity), - //fwdPool: make(map[pot.Address]*protocols.Peer), fwdPool: make(map[discover.NodeID]*protocols.Peer), handlers: make(map[Topic]map[*Handler]bool), fwdcache: make(map[pssDigest]pssCacheEntry), @@ -110,18 +282,25 @@ func NewPss(k network.Overlay, dpa *storage.DPA, params *PssParams) *Pss { } } +// Convenience accessor to the swarm overlay address of the pss node func (self *Pss) BaseAddr() []byte { return self.Overlay.BaseAddr() } +// For node.Service implementation. Does nothing for now, but should be included in the code for backwards compatibility. func (self *Pss) Start(srv *p2p.Server) error { return nil } + +// For node.Service implementation. Does nothing for now, but should be included in the code for backwards compatibility. func (self *Pss) Stop() error { return nil } +// devp2p protocol object for the PssMsg struct. +// +// This represents the PssMsg capsule, and is the entry point for processing, receiving and sending pss messages between directly connected peers. func (self *Pss) Protocols() []p2p.Protocol { return []p2p.Protocol{ p2p.Protocol{ @@ -133,16 +312,19 @@ func (self *Pss) Protocols() []p2p.Protocol { } } +// Starts the PssMsg protocol func (self *Pss) Run(p *p2p.Peer, rw p2p.MsgReadWriter) error { pp := protocols.NewPeer(p, rw, pssSpec) //addr := network.NewAddrFromNodeID(id) //potaddr := pot.NewHashAddressFromBytes(addr.OAddr) - //self.fwdPool[potaddr.Address] = pp self.fwdPool[p.ID()] = pp return pp.Run(self.handlePssMsg) } +// Exposes the API methods +// +// If the debug-parameter was given to the top Pss object, the TestAPI methods will also be included func (self *Pss) APIs() []rpc.API { apis := []rpc.API{ rpc.API{ @@ -163,12 +345,11 @@ func (self *Pss) APIs() []rpc.API { return apis } -// Takes the generated Topic of a protocol/chatroom etc, and links a handler function to it -// This allows the implementer to retrieve the right handler functions (invoke the right protocol) -// for an incoming message by inspecting the topic on it. -// a topic allows for multiple handlers -// returns a deregister function which needs to be called to deregister the handler -// (similar to event.Subscription.Unsubscribe()) +// Links a handler function to a Topic +// +// After calling this, all incoming messages with an envelope Topic matching the Topic specified will be passed to the given Handler function. +// +// Returns a deregister function which needs to be called to deregister the handler, func (self *Pss) Register(topic *Topic, handler Handler) func() { self.lock.Lock() defer self.lock.Unlock() @@ -192,10 +373,7 @@ func (self *Pss) deregister(topic *Topic, h *Handler) { delete(handlers, h) } -// enables to set address of node, to avoid backwards forwarding -// -// currently not in use as forwarder address is not known in the handler function hooked to the pss dispatcher. -// it is included as a courtesy to custom transport layers that may want to implement this +// Adds an address/message pair to the cache func (self *Pss) AddToCache(addr []byte, msg *PssMsg) error { digest, err := self.storeMsg(msg) if err != nil { @@ -263,13 +441,13 @@ func (self *Pss) handlePssMsg(msg interface{}) error { return self.Process(pssmsg) } -// processes a message with self as recipient +// Entry point to processing a message for which the current node is the intended recipient. func (self *Pss) Process(pssmsg *PssMsg) error { env := pssmsg.Payload payload := env.Payload handlers := self.getHandlers(env.Topic) if len(handlers) == 0 { - return fmt.Errorf("No registered handler for topic '%s'", env.Topic) + return fmt.Errorf("No registered handler for topic '%x'", env.Topic) } nid, _ := discover.HexID("0x00") p := p2p.NewPeer(nid, fmt.Sprintf("%x", env.From), []p2p.Cap{}) @@ -282,9 +460,11 @@ func (self *Pss) Process(pssmsg *PssMsg) error { return nil } -// Sends a message using The message could be anything at all, and will be handled by whichever handler function is mapped to Topic using *Pss.Register() +// Sends a message using Pss. // -// The to address is a swarm overlay address +// This method is payload agnostic, and will accept any arbitrary byte slice as the payload for a message. +// +// It generates an envelope for the specified recipient and topic, and wraps the message payload in it. func (self *Pss) SendRaw(to []byte, topic Topic, msg []byte) error { sender := self.Overlay.BaseAddr() pssenv := NewEnvelope(sender, topic, msg) @@ -297,18 +477,20 @@ func (self *Pss) SendRaw(to []byte, topic Topic, msg []byte) error { // Forwards a pss message to the peer(s) closest to the to address // -// Handlers that want to pass on a message should call this directly +// Handlers that are merely passing on the PssMsg to its final recipient should call this directly func (self *Pss) Forward(msg *PssMsg) error { if self.isSelfRecipient(msg) { return errorForwardToSelf } + // cache it digest, err := self.storeMsg(msg) if err != nil { log.Warn(fmt.Sprintf("could not store message %v to cache: %v", msg, err)) } + // flood guard if self.checkFwdCache(nil, digest) { log.Trace(fmt.Sprintf("pss relay block-cache match: FROM %x TO %x", common.ByteLabel(self.Overlay.BaseAddr()), common.ByteLabel(msg.To))) return nil @@ -326,15 +508,12 @@ func (self *Pss) Forward(msg *PssMsg) error { return false } sendMsg := fmt.Sprintf("TO %x FROM %x VIA %x", common.ByteLabel(msg.To), common.ByteLabel(self.BaseAddr()), common.ByteLabel(op.Address())) - //h := pot.NewHashAddressFromBytes(op.Address()) - //pp := self.fwdPool[h.Address] pp := self.fwdPool[sp.ID()] if self.checkFwdCache(op.Address(), digest) { log.Info("%v: peer already forwarded to", sendMsg) return true } err := pp.Send(msg) - //err := sp.Send(msg) if err != nil { log.Warn(fmt.Sprintf("%v: failed forwarding: %v", sendMsg, err)) return true @@ -351,14 +530,16 @@ func (self *Pss) Forward(msg *PssMsg) error { if sent == 0 { log.Error("PSS: unable to forward to any peers") - return nil + return fmt.Errorf("unable to forward to any peers") } self.addFwdCacheExpire(digest) return nil } -// Links a pss peer address and topic to a dedicated p2p.MsgReadWriter in the pss peerpool, and runs the specificed protocol on this p2p.MsgReadWriter and the specified peer +// For devp2p protocol integration only. Analogous to an outgoing devp2p connection. +// +// Links a remote peer and Topic to a dedicated p2p.MsgReadWriter in the pss peerpool, and runs the specificed protocol using these resources. // // The effect is that now we have a "virtual" protocol running on an artificial p2p.Peer, which can be looked up and piped to through Pss using swarm overlay address and topic func (self *Pss) AddPeer(p *p2p.Peer, addr pot.Address, run func(*p2p.Peer, p2p.MsgReadWriter) error, topic Topic, rw p2p.MsgReadWriter) error { @@ -403,10 +584,9 @@ func (self *Pss) isActive(id pot.Address, topic Topic) bool { return self.peerPool[id][topic] != nil } -// Convenience object that: +// For devp2p protocol integration only. // -// - allows passing of the unwrapped PssMsg payload to the p2p level message handlers -// - interprets outgoing p2p.Msg from the p2p level to pass in to *Pss.Send() +// Bridges pss send/receive with devp2p protocol send/receive // // Implements p2p.MsgReadWriter type PssReadWriter struct { @@ -448,6 +628,9 @@ func (prw PssReadWriter) injectMsg(msg p2p.Msg) error { return nil } + +// For devp2p protocol integration only. +// // Convenience object for passing messages in and out of the p2p layer type PssProtocol struct { *Pss @@ -456,7 +639,9 @@ type PssProtocol struct { spec *protocols.Spec } -// Constructor +// For devp2p protocol integration only. +// +// Maps a Topic to a devp2p protocol. func RegisterPssProtocol(ps *Pss, topic *Topic, spec *protocols.Spec, targetprotocol *p2p.Protocol) *PssProtocol { pp := &PssProtocol{ Pss: ps, @@ -467,8 +652,12 @@ func RegisterPssProtocol(ps *Pss, topic *Topic, spec *protocols.Spec, targetprot return pp } +// For devp2p protocol integration only. +// +// Generic handler for initiating devp2p-like protocol connections +// +// This handler should be passed to Pss.Register with the associated ropic. func (self *PssProtocol) Handle(msg []byte, p *p2p.Peer, senderAddr []byte) error { - //hashoaddr := pot.NewHashAddressFromBytes(senderAddr).Address hashoaddr := pot.NewAddressFromBytes(senderAddr) if !self.isActive(hashoaddr, *self.topic) { rw := &PssReadWriter{ diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index 3702868ebc..b80bd476d5 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -4,14 +4,17 @@ import ( "bytes" "context" "encoding/hex" + "encoding/json" "fmt" "io/ioutil" + "math/rand" "os" "sync" "testing" "time" + "flag" - "github.com/ethereum/go-ethereum/common" +// "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" @@ -20,7 +23,6 @@ import ( "github.com/ethereum/go-ethereum/p2p/simulations" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" p2ptest "github.com/ethereum/go-ethereum/p2p/testing" - "github.com/ethereum/go-ethereum/pot" "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/storage" ) @@ -30,6 +32,10 @@ const ( bzzServiceName = "bzz" ) +var ( + snapshotfile string +) + var services = newServices() func init() { @@ -38,6 +44,8 @@ func init() { hf := log.LvlFilterHandler(log.LvlTrace, hs) h := log.CallerFileHandler(hf) log.Root().SetHandler(h) + + flag.StringVar(&snapshotfile, "file", "snapsnot.json", "snapshot file") } func TestCache(t *testing.T) { @@ -199,8 +207,9 @@ func TestSimpleLinear(t *testing.T) { Peer: pp, addr: network.ToOverlayAddr(id[:]), } - a := pot.NewAddressFromBytes(bp.addr) - ps.fwdPool[a] = pp + //a := pot.NewAddressFromBytes(bp.addr) + //ps.fwdPool[a] = pp + ps.fwdPool[id] = pp ps.Overlay.On(bp) defer ps.Overlay.Off(bp) log.Debug(fmt.Sprintf("%v", ps.Overlay)) @@ -236,32 +245,56 @@ func TestSimpleLinear(t *testing.T) { func TestFullRandom50n(t *testing.T) { adapter := adapters.NewSimAdapter(services) - testFullRandom(t, adapter, 50, 50, 50) + testFullRandom(t, adapter, 50, 50) } func TestFullRandom25n(t *testing.T) { adapter := adapters.NewSimAdapter(services) - testFullRandom(t, adapter, 25, 25, 25) + testFullRandom(t, adapter, 25, 25) } func TestFullRandom10n(t *testing.T) { adapter := adapters.NewSimAdapter(services) - testFullRandom(t, adapter, 10, 10, 10) + testFullRandom(t, adapter, 10, 10) } func TestFullRandom5n(t *testing.T) { - adapter := adapters.NewSimAdapter(services) - testFullRandom(t, adapter, 5, 5, 5) + baseDir, err := ioutil.TempDir("", "swarm-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(baseDir) + adapter := adapters.NewExecAdapter(baseDir) + testFullRandom(t, adapter, 5, 5) } -func testFullRandom(t *testing.T, adapter adapters.NodeAdapter, nodecount int, fullnodecount int, msgcount int) { - var i int - var msgfromids []discover.NodeID +func testFullRandom(t *testing.T, adapter adapters.NodeAdapter, nodecount int, msgcount int) { +} + +func TestFullRandomSnapshot50(t *testing.T) { + testFullRandomSnapshot(t, true, 5) +} + +func testFullRandomSnapshot(t *testing.T, sim bool, msgcount int) { + + var msgtoids []discover.NodeID var msgreceived []discover.NodeID var cancelmain func() - var triggerptr *chan discover.NodeID + //var triggerptr *chan discover.NodeID - msgtoids := make([]discover.NodeID, msgcount) + + baseDir, err := ioutil.TempDir("", "swarm-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(baseDir) + + var adapter adapters.NodeAdapter + if sim { + adapter = adapters.NewSimAdapter(services) + } else { + adapter = adapters.NewExecAdapter(baseDir) + } wg := sync.WaitGroup{} wg.Add(msgcount) @@ -269,141 +302,187 @@ func testFullRandom(t *testing.T, adapter adapters.NodeAdapter, nodecount int, f psslog := make(map[discover.NodeID]log.Logger) psslogmain := log.New("psslog", "*") + jsonsnapshot, err := ioutil.ReadFile(snapshotfile) + if err != nil { + t.Fatalf("cant read snapshot: %s", snapshotfile) + } + snapshot := &simulations.Snapshot{} + err = json.Unmarshal(jsonsnapshot, snapshot) + net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{ ID: "0", }) defer net.Shutdown() + err = net.Load(snapshot) + if err != nil { + t.Fatalf("invalid snapshot: %v", err) + } + + //msgtoids := make([]discover.NodeID, msgcount) + timeout := 15 * time.Second ctx, cancelmain := context.WithTimeout(context.Background(), timeout) defer cancelmain() trigger := make(chan discover.NodeID) - triggerptr = &trigger + //triggerptr = &trigger - ids := make([]discover.NodeID, nodecount) - fullids := ids[0:fullnodecount] - fullpeers := make(map[discover.NodeID][]byte) + recvaddrs := make(map[discover.NodeID][]byte) - for i = 0; i < nodecount; i++ { - nodeconfig := adapters.RandomNodeConfig() - nodeconfig.Services = []string{"bzz", "pss"} - node, err := net.NewNodeWithConfig(nodeconfig) - if err != nil { - t.Fatalf("error starting node: %s", err) - } - - if err := net.Start(node.ID()); err != nil { - t.Fatalf("error starting node %s: %s", node.ID().TerminalString(), err) - } - - if err := triggerChecks(ctx, &wg, triggerptr, net, node.ID()); err != nil { - t.Fatal("error triggering checks for node %s: %s", node.ID().TerminalString(), err) - } - ids[i] = node.ID() - if i < fullnodecount { - fullpeers[ids[i]] = network.ToOverlayAddr(node.ID().Bytes()) - psslog[ids[i]] = log.New("psslog", fmt.Sprintf("%x", fullpeers[ids[i]])) - } - log.Debug("psslog starting node", "id", nodeconfig.ID) - } - - for i, id := range fullids { - msgfromids = append(msgfromids, id) - msgtoids[i] = fullids[(i+(len(fullids)/2)+1)%len(fullids)] - } +// for i = 0; i < nodecount; i++ { +// nodeconfig := adapters.RandomNodeConfig() +// nodeconfig.Services = []string{"bzz", "pss"} +// node, err := net.NewNodeWithConfig(nodeconfig) +// if err != nil { +// t.Fatalf("error starting node: %s", err) +// } +// +// if err := net.Start(node.ID()); err != nil { +// t.Fatalf("error starting node %s: %s", node.ID().TerminalString(), err) +// } +// +// if err := triggerChecks(ctx, &wg, triggerptr, net, node.ID()); err != nil { +// t.Fatal("error triggering checks for node %s: %s", node.ID().TerminalString(), err) +// } +// ids[i] = node.ID() +// if i < fullnodecount { +// fullpeers[ids[i]] = network.ToOverlayAddr(node.ID().Bytes()) +// psslog[ids[i]] = log.New("psslog", fmt.Sprintf("%x", fullpeers[ids[i]])) +// } +// log.Debug("psslog starting node", "id", nodeconfig.ID) +// } +// +// for i, id := range fullids { +// msgfromids = append(msgfromids, id) +// msgtoids[i] = fullids[(i+(len(fullids)/2)+1)%len(fullids)] +// } // run a simulation which connects the 10 nodes in a ring and waits // for full peer discovery +// action := func(ctx context.Context) error { +// for i, id := range ids { +// peerID := ids[(i+1)%len(ids)] +// if net.GetConn(id, peerID) != nil { +// continue +// } +// if err := net.Connect(id, peerID); err != nil { +// return err +// } +// psslog[id].Debug("conn ok", "one", id, "other", peerID) +// } +// return nil +// } +// check := func(ctx context.Context, id discover.NodeID) (bool, error) { +// select { +// case <-ctx.Done(): +// wg.Done() +// psslog[id].Error("conn failed!", "id", id) +// return false, ctx.Err() +// default: +// } +// var tgt []byte +// var fwd struct { +// Addr []byte +// Count int +// } +// +// for i, fid := range msgfromids { +// if id == fid { +// tgt = network.ToOverlayAddr(msgtoids[(i+(len(msgtoids)/2)+1)%len(msgtoids)].Bytes()) +// break +// } +// } +// p := net.GetNode(id) +// if p == nil { +// return false, fmt.Errorf("Unknown node: %v", id) +// } +// c, err := p.Client() +// if err != nil { +// return false, err +// } +// for fwd.Count < 2 { +// c.CallContext(context.Background(), &fwd, "pss_getForwarder", tgt) +// time.Sleep(time.Microsecond * 250) +// } +// psslog[id].Debug("fwd check ok", "topaddr", fmt.Sprintf("%x", common.ByteLabel(fwd.Addr)), "kadcount", fwd.Count) +// return true, nil +// } +// +// result := simulations.NewSimulation(net).Run(ctx, &simulations.Step{ +// Action: action, +// Trigger: trigger, +// Expect: &simulations.Expectation{ +// Nodes: ids, +// Check: check, +// }, +// }) +// if result.Error != nil { +// t.Fatalf("simulation failed: %s", result.Error) +// cancelmain() +// } +// +// trigger = make(chan discover.NodeID) +// triggerptr = &trigger + + var ids []discover.NodeID + action := func(ctx context.Context) error { - for i, id := range ids { - peerID := ids[(i+1)%len(ids)] - if net.GetConn(id, peerID) != nil { - continue + var rpcerr error + var rpcbyte []byte + //for ii, id := range msgfromids { + for _, simnode := range net.Nodes { + ids = append(ids, simnode.ID()) + //node := net.GetNode(id) + if simnode == nil { + return fmt.Errorf("unknown node: %s", simnode.ID()) } - if err := net.Connect(id, peerID); err != nil { - return err + client, err := simnode.Client() + if err != nil { + return fmt.Errorf("error getting recp node client: %s", err) + } + + err = client.Call(&rpcbyte, "pss_baseAddr") + if err != nil { + t.Fatalf("cant get overlayaddr: %v", err) + } + + recvaddrs[simnode.ID()] = rpcbyte + err = client.Call(&rpcbyte, "pss_baseAddr") + if err != nil { + t.Fatalf("cant get overlayaddr: %v", err) + } + } + for i := 0; i < msgcount; i++ { + + idx := rand.Intn(len(net.Nodes)) + sendernode := net.Nodes[idx] + toidx := rand.Intn(len(net.Nodes)-1) + if idx >= toidx { + toidx++ + } + + recvnode := net.Nodes[toidx] + msg := PingMsg{Created: time.Now()} + code, _ := PingProtocol.GetCode(&PingMsg{}) + pmsg, _ := NewProtocolMsg(code, msg) + + client, err := sendernode.Client() + if err != nil { + return fmt.Errorf("error getting sendernode client: %s", err) + } + client.CallContext(ctx, &rpcerr, "pss_send", PingTopic, APIMsg{ + //Addr: fullpeers[msgtoids[ii]], + Addr: recvaddrs[recvnode.ID()], + Msg: pmsg, + }) + if rpcerr != nil { + return fmt.Errorf("error rpc send id %x: %v", sendernode.ID(), rpcerr) } - psslog[id].Debug("conn ok", "one", id, "other", peerID) } return nil } check := func(ctx context.Context, id discover.NodeID) (bool, error) { - var tgt []byte - var fwd struct { - Addr []byte - Count int - } - select { - case <-ctx.Done(): - wg.Done() - psslog[id].Error("conn failed!", "id", id) - return false, ctx.Err() - default: - } - for i, fid := range msgfromids { - if id == fid { - tgt = network.ToOverlayAddr(msgtoids[(i+(len(msgtoids)/2)+1)%len(msgtoids)].Bytes()) - break - } - } - p := net.GetNode(id) - if p == nil { - return false, fmt.Errorf("Unknown node: %v", id) - } - c, err := p.Client() - if err != nil { - return false, err - } - for fwd.Count < 2 { - c.CallContext(context.Background(), &fwd, "pss_getForwarder", tgt) - time.Sleep(time.Microsecond * 250) - } - psslog[id].Debug("fwd check ok", "topaddr", fmt.Sprintf("%x", common.ByteLabel(fwd.Addr)), "kadcount", fwd.Count) - return true, nil - } - - result := simulations.NewSimulation(net).Run(ctx, &simulations.Step{ - Action: action, - Trigger: trigger, - Expect: &simulations.Expectation{ - Nodes: ids, - Check: check, - }, - }) - if result.Error != nil { - t.Fatalf("simulation failed: %s", result.Error) - cancelmain() - } - - trigger = make(chan discover.NodeID) - triggerptr = &trigger - - action = func(ctx context.Context) error { - var rpcerr error - for ii, id := range msgfromids { - node := net.GetNode(id) - if node == nil { - return fmt.Errorf("unknown node: %s", id) - } - client, err := node.Client() - if err != nil { - return fmt.Errorf("error getting node client: %s", err) - } - msg := PingMsg{Created: time.Now()} - code, _ := PingProtocol.GetCode(&PingMsg{}) - pmsg, _ := NewProtocolMsg(code, msg) - client.CallContext(ctx, &rpcerr, "pss_send", PingTopic, APIMsg{ - Addr: fullpeers[msgtoids[ii]], - Msg: pmsg, - }) - if rpcerr != nil { - return fmt.Errorf("error rpc send id %x: %v", id, rpcerr) - } - } - return nil - } - check = func(ctx context.Context, id discover.NodeID) (bool, error) { select { case <-ctx.Done(): @@ -417,11 +496,12 @@ func testFullRandom(t *testing.T, adapter adapters.NodeAdapter, nodecount int, f return true, nil } - result = simulations.NewSimulation(net).Run(ctx, &simulations.Step{ + result := simulations.NewSimulation(net).Run(ctx, &simulations.Step{ Action: action, Trigger: trigger, Expect: &simulations.Expectation{ - Nodes: msgtoids, + //Nodes: msgtoids, + Nodes: ids, Check: check, }, }) @@ -555,7 +635,7 @@ func newServices() adapters.Services { "bzz": func(ctx *adapters.ServiceContext) (node.Service, error) { addr := network.NewAddrFromNodeID(ctx.Config.ID) hp := network.NewHiveParams() - hp.Discovery = true + hp.Discovery = false config := &network.BzzConfig{ OverlayAddr: addr.Over(), UnderlayAddr: addr.Under(), diff --git a/swarm/pss/pssapi.go b/swarm/pss/pssapi.go index e5221c7494..0b5173d556 100644 --- a/swarm/pss/pssapi.go +++ b/swarm/pss/pssapi.go @@ -11,17 +11,20 @@ import ( "github.com/ethereum/go-ethereum/swarm/network" ) -// API is the RPC API module for Pss +// Pss API services type API struct { *Pss } -// NewAPI constructs a PssAPI instance func NewAPI(ps *Pss) *API { return &API{Pss: ps} } -// NewMsg API endpoint creates an RPC subscription +// Creates a new subscription for the caller. Enables external handling of incoming messages. +// +// A new handler is registered in pss for the supplied topic +// +// All incoming messages to the node matching this topic will be encapsulated in the APIMsg struct and sent to the subscriber func (pssapi *API) Receive(ctx context.Context, topic Topic) (*rpc.Subscription, error) { notifier, supported := rpc.NotifierFromContext(ctx) if !supported { @@ -54,31 +57,47 @@ func (pssapi *API) Receive(ctx context.Context, topic Topic) (*rpc.Subscription, return psssub, nil } -// SendRaw sends the message (serialized into byte slice) to a peer with topic +// Sends the message wrapped in APIMsg through pss +// +// Wrapper method for the pss.SendRaw function. +// +// The method will pass on the error received from pss. +// +// Note that normally pss will report an error if an attempt is made to send a pss to oneself. However, if the debug flag has been set, and the address specified in APIMsg is the node's own, this method implements a short-circuit which injects the message as an incoming message (using Pss.Process). This can be useful for testing purposes, when only operating with one node. func (pssapi *API) Send(topic Topic, msg APIMsg) error { - if pssapi.debug && bytes.Equal(msg.Addr, pssapi.BaseAddr()) { + if pssapi.debug && bytes.Equal(msg.Addr, pssapi.Pss.BaseAddr()) { log.Warn("Pss debug enabled; send to self shortcircuit", "apimsg", msg, "topic", topic) env := NewEnvelope(msg.Addr, topic, msg.Msg) return pssapi.Process(&PssMsg{ - To: pssapi.BaseAddr(), + To: pssapi.Pss.BaseAddr(), Payload: env, }) } return pssapi.SendRaw(msg.Addr, topic, msg.Msg) } +// BaseAddr returns the pss node's swarm overlay address +// +// Note that the overlay address is NOT inferable. To really know the node's overlay address it must reveal it itself. +func (pssapi *API) BaseAddr() ([]byte, error) { + return pssapi.Pss.BaseAddr(), nil +} + // PssAPITest are temporary API calls for development use only -// These symbols should not be included in production environment +// +// These symbols should NOT be included in production environment type APITest struct { *Pss } -// NewAPI constructs a API instance +// Include these methods to the node.Service if test symbols should be used func NewAPITest(ps *Pss) *APITest { return &APITest{Pss: ps} } -// temporary for access to overlay while faking kademlia healthy routines +// Get the current nearest swarm node to the specified address +// +// (Can be used for diagnosing kademlia state) func (pssapitest *APITest) GetForwarder(addr []byte) (fwd struct { Addr []byte Count int @@ -93,7 +112,3 @@ func (pssapitest *APITest) GetForwarder(addr []byte) (fwd struct { return } -// BaseAddr gets our own overlayaddress -func (pssapitest *APITest) BaseAddr() ([]byte, error) { - return pssapitest.Pss.BaseAddr(), nil -} diff --git a/swarm/pss/simulations/service_test.go b/swarm/pss/simulations/service_test.go index 8fb1d10269..4ea40bab12 100644 --- a/swarm/pss/simulations/service_test.go +++ b/swarm/pss/simulations/service_test.go @@ -139,9 +139,13 @@ func (t *testWrapper) newTestService(ctx *adapters.ServiceContext) (node.Service if err != nil { panic(err) } + pssclient, err := client.NewClientWithRPC(context.Background(), rpcClient) + if err != nil { + return nil, err + } return &testService{ id: ctx.Config.ID, - pss: client.NewClientWithRPC(context.Background(), rpcClient), + pss: pssclient, handshakes: make(chan *testHandshake), }, nil } diff --git a/swarm/pss/types.go b/swarm/pss/types.go index 92b1f520df..9a76285778 100644 --- a/swarm/pss/types.go +++ b/swarm/pss/types.go @@ -18,13 +18,13 @@ const ( defaultDigestCacheTTL = time.Second ) -// Defines params for Pss +// Pss configuration parameters type PssParams struct { Cachettl time.Duration Debug bool } -// Initializes default params for Pss +// Sane defaults for Pss func NewPssParams(debug bool) *PssParams { return &PssParams{ Cachettl: defaultDigestCacheTTL, @@ -32,13 +32,14 @@ func NewPssParams(debug bool) *PssParams { } } -// Encapsulates the message transported over pss. +// Encapsulates messages transported over pss. type PssMsg struct { To []byte Payload *Envelope } -func (msg *PssMsg) Serialize() []byte { +// serializes the message for use in cache +func (msg *PssMsg) serialize() []byte { rlpdata, _ := rlp.EncodeToBytes(msg) return rlpdata } @@ -48,16 +49,8 @@ func (self *PssMsg) String() string { return fmt.Sprintf("PssMsg: Recipient: %x", common.ByteLabel(self.To)) } -// Topic defines the context of a message being transported over pss -// It is used by pss to determine what action is to be taken on an incoming message -// Typically, one can map protocol handlers for the message payloads by mapping topic to them; see *Pss.Register() -type Topic [TopicLength]byte -func (self *Topic) String() string { - return fmt.Sprintf("%x", self) -} - -// Pre-Whisper placeholder, payload of PssMsg +// Pre-Whisper placeholder, payload of PssMsg, sender address, Topic type Envelope struct { Topic Topic TTL uint16 @@ -65,7 +58,7 @@ type Envelope struct { From []byte } -// creates Pss envelope from sender address, topic and raw payload +// Creates A Pss envelope from sender address, topic and raw payload func NewEnvelope(addr []byte, topic Topic, payload []byte) *Envelope { return &Envelope{ From: addr, @@ -75,7 +68,7 @@ func NewEnvelope(addr []byte, topic Topic, payload []byte) *Envelope { } } -// encapsulates a protocol msg as PssEnvelope data +// Convenience wrapper for devp2p protocol messages for transport over pss type ProtocolMsg struct { Code uint64 Size uint32 @@ -83,13 +76,7 @@ type ProtocolMsg struct { ReceivedAt time.Time } -// PssAPIMsg is the type for messages, it extends the rlp encoded protocol Msg -// with the Sender's overlay address -type APIMsg struct { - Msg []byte - Addr []byte -} - +// Creates a ProtocolMsg func NewProtocolMsg(code uint64, msg interface{}) ([]byte, error) { rlpdata, err := rlp.EncodeToBytes(msg) @@ -97,8 +84,6 @@ func NewProtocolMsg(code uint64, msg interface{}) ([]byte, error) { return nil, err } - // previous attempts corrupted nested structs in the payload iself upon deserializing - // therefore we use two separate []byte fields instead of peerAddr // TODO verify that nested structs cannot be used in rlp smsg := &ProtocolMsg{ Code: code, @@ -109,12 +94,30 @@ func NewProtocolMsg(code uint64, msg interface{}) ([]byte, error) { return rlp.EncodeToBytes(smsg) } -// Message handler func for a topic +// Convenience wrapper for sending and receiving pss messages when using the pss API +type APIMsg struct { + Msg []byte + Addr []byte +} + +// Signature for a message handler function for a PssMsg +// +// Implementations of this type are passed to Pss.Register together with a topic, type Handler func(msg []byte, p *p2p.Peer, from []byte) error -// constructs a new PssTopic from a given name and version. +// Topic defines the context of a message being transported over pss +// It is used by pss to determine what action is to be taken on an incoming message +// Typically, one can map protocol handlers for the message payloads by mapping topic to them; see Pss.Register +type Topic [TopicLength]byte + +// String representation of Topic +func (self *Topic) String() string { + return fmt.Sprintf("%x", self) +} + +// Constructs a new PssTopic from a given name and version. // -// Analogous to the name and version members of p2p.Protocol +// Analogous to the name and version members of p2p.Protocol. func NewTopic(s string, v int) (topic Topic) { h := sha3.NewKeccak256() h.Write([]byte(s)) @@ -125,6 +128,11 @@ func NewTopic(s string, v int) (topic Topic) { return topic } +// For devp2p protocol integration only +// +// Creates a serialized (non-buffered) version of a p2p.Msg, used in the specialized p2p.MsgReadwriter implementations used internally by pss +// +// Should not normally be called outside the pss package hierarchy func ToP2pMsg(msg []byte) (p2p.Msg, error) { payload := &ProtocolMsg{} if err := rlp.DecodeBytes(msg, payload); err != nil { diff --git a/swarm/swarm_psschat.go b/swarm/swarm_psschat.go index 18d31b21c8..8f262abdbe 100644 --- a/swarm/swarm_psschat.go +++ b/swarm/swarm_psschat.go @@ -42,7 +42,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/pss" "github.com/ethereum/go-ethereum/swarm/storage" - psschat "github.com/ethereum/go-ethereum/swarm/pss/protocols/chat" + //psschat "github.com/ethereum/go-ethereum/swarm/pss/protocols/chat" ) // the swarm stack @@ -197,7 +197,7 @@ func (self *Swarm) Start(net *p2p.Server) error { // update uaddr to correct enode newaddr := self.bzz.UpdateLocalAddr([]byte(net.Self().String())) - log.Warn("Updated bzz local addr", "oaddr", fmt.Sprintf("%s", newaddr.OAddr), "uaddr", fmt.Sprintf("%x", newaddr.UAddr)) + log.Warn("Updated bzz local addr", "oaddr", fmt.Sprintf("%x", newaddr.OAddr), "uaddr", fmt.Sprintf("%x", newaddr.UAddr)) // set chequebook if self.swapEnabled { @@ -223,25 +223,30 @@ func (self *Swarm) Start(net *p2p.Server) error { if self.pss != nil { self.pss.Start(net) - log.Info("Pss started") - pss.RegisterPssProtocol(self.pss, &psschat.ChatTopic, psschat.ChatProtocol, psschat.New(nil, nil, func(ctrl *psschat.ChatCtrl) { - if ctrl.OutC != nil { - go func() { - for { - select { - case msg := <-ctrl.OutC: - err := ctrl.Peer.Send(msg) - if err != nil { - // do something with ctrl.ConnC here - //pp.Drop(err) - //return - } - } - } - }() - } - - })) + topic := pss.NewTopic("pssChat", 1) + self.pss.Register(&topic, func(msg []byte, p *p2p.Peer, from []byte) error { + log.Trace("placeholder handler for node got psschat proto incoming", "msg", msg, "from", from) + return nil + }) + log.Info("Pss started ... with 'pssChat' :)") +// pss.RegisterPssProtocol(self.pss, &psschat.ChatTopic, psschat.ChatProtocol, psschat.New(nil, nil, nil, func(ctrl *psschat.ChatCtrl) { +// if ctrl.OutC != nil { +// go func() { +// for { +// select { +// case msg := <-ctrl.OutC: +// err := ctrl.Peer.Send(msg) +// if err != nil { +// // do something with ctrl.ConnC here +// //pp.Drop(err) +// //return +// } +// } +// } +// }() +// } +// +// })) } self.dpa.Start() From 1598e6860048d2857d0f4f249070640135e7a68f Mon Sep 17 00:00:00 2001 From: nolash Date: Thu, 15 Jun 2017 18:20:06 +0200 Subject: [PATCH 08/17] swarm/network: removed debug lines --- swarm/network/protocol.go | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index 8243729100..c93f2968cf 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -23,7 +23,6 @@ import ( "net" "sync" "time" - "strings" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" @@ -292,21 +291,24 @@ func (self *bzzHandshake) Perform(p *p2p.Peer, rw p2p.MsgReadWriter) (err error) return fmt.Errorf("version mismatch %d (!= %d)", rhs.Version, self.Version) } self.peerAddr = rhs.Addr - peerid := p.ID() - log.Warn( - strings.Join( - []string{ - "handshake ok:", - fmt.Sprintf("p2p.Peer.ID(): ...... %v", p.ID()), - fmt.Sprintf("peerAddr.UAddr: ...... %x", self.peerAddr.UAddr[:32]), - fmt.Sprintf("peerAddr.OAddr: ...... %x", self.peerAddr.OAddr[:32]), - fmt.Sprintf("selfAddr.UAddr: ...... %x", self.Addr.UAddr[:32]), - fmt.Sprintf("selfAddr.OAddr: ...... %x", self.Addr.OAddr[:32]), - fmt.Sprintf("ToOverlayAddr(p2p.Peer.ID): .... %x", ToOverlayAddr(peerid[:])), - fmt.Sprintf("ToOverlayAddr(peerAddr.UAddr): .. %x", ToOverlayAddr(self.peerAddr.UAddr)), - }, "\n", - ), - ) + //log.Debug("L&R handshake", "local", fmt.Sprintf("%x", self.Addr.Over()), "remote", fmt.Sprintf("%x", rhs.Addr.Over())) + // log.Debug("RHS handshake", "addr", rhs.Addr, "peeraddr", rhs.peerAddr) + // log.Debug("LHS handshake", "addr", self.Addr, "peeraddr", self.peerAddr) + // + // log.Warn( + // strings.Join( + // []string{ + // "handshake ok:", + // fmt.Sprintf("p2p.Peer.ID(): ...... %v", p.ID()), + // fmt.Sprintf("peerAddr.UAddr: ...... %x", self.peerAddr.UAddr[:32]), + // fmt.Sprintf("peerAddr.OAddr: ...... %x", self.peerAddr.OAddr[:32]), + // fmt.Sprintf("selfAddr.UAddr: ...... %x", self.Addr.UAddr[:32]), + // fmt.Sprintf("selfAddr.OAddr: ...... %x", self.Addr.OAddr[:32]), + // fmt.Sprintf("ToOverlayAddr(p2p.Peer.ID): .... %x", ToOverlayAddr(peerid[:])), + // fmt.Sprintf("ToOverlayAddr(peerAddr.UAddr): .. %x", ToOverlayAddr(self.peerAddr.UAddr)), + // }, "\n", + // ), + // ) return nil } From 72517463c87b8946a61b3b48d939430a5cf6aa18 Mon Sep 17 00:00:00 2001 From: zelig Date: Sat, 17 Jun 2017 17:06:28 +0200 Subject: [PATCH 09/17] pot, swarm/network: cleanup, fix depth in kad table --- pot/address.go | 2 +- pot/address_test.go | 142 ----------------------- pot/doc.go | 40 +++---- pot/pot.go | 120 ++------------------ pot/pot_test.go | 2 - swarm/network/discovery.go | 104 ++++++++++------- swarm/network/discovery_test.go | 18 ++- swarm/network/hive.go | 2 +- swarm/network/hive_test.go | 16 +++ swarm/network/kademlia.go | 16 +-- swarm/network/kademlia_test.go | 48 ++++---- swarm/network/protocol.go | 15 +-- swarm/network/protocol_test.go | 26 ++++- swarm/network/test_overlay.go | 193 -------------------------------- 14 files changed, 185 insertions(+), 559 deletions(-) delete mode 100644 pot/address_test.go delete mode 100644 swarm/network/test_overlay.go diff --git a/pot/address.go b/pot/address.go index 8a396dac3b..a8e65fa93f 100644 --- a/pot/address.go +++ b/pot/address.go @@ -197,7 +197,7 @@ func ToBytes(v Val) []byte { return b } -// DefaultPof returns a proximity order operator function +// DefaultPof returns a proximity order comparison operator function // where all func DefaultPof(max int) func(one, other Val, pos int) (int, bool) { return func(one, other Val, pos int) (int, bool) { diff --git a/pot/address_test.go b/pot/address_test.go deleted file mode 100644 index b9b3f1f4e3..0000000000 --- a/pot/address_test.go +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . -package pot - -// -// import ( -// "math/rand" -// "reflect" -// "testing" -// -// "github.com/ethereum/go-ethereum/common" -// ) -// // -// // func (Address) Generate(rand *rand.Rand, size int) reflect.Value { -// // var id Address -// // for i := 0; i < len(id); i++ { -// // id[i] = byte(uint8(rand.Intn(255))) -// // } -// // return reflect.ValueOf(id) -// // } -// -// func TestCommonBitsAddrF(t *testing.T) { -// a := Address(common.HexToHash("0x0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef")) -// b := Address(common.HexToHash("0x8123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef")) -// c := Address(common.HexToHash("0x4123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef")) -// d := Address(common.HexToHash("0x0023456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef")) -// e := Address(common.HexToHash("0x01A3456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef")) -// ab := CommonBitsAddrF(a, b, func() byte { return byte(0x00) }, 10) -// expab := Address(common.HexToHash("0x8000000000000000000000000000000000000000000000000000000000000000")) -// -// if ab != expab { -// t.Fatalf("%v != %v", ab, expab) -// } -// ac := CommonBitsAddrF(a, c, func() byte { return byte(0x00) }, 10) -// expac := Address(common.HexToHash("0x4000000000000000000000000000000000000000000000000000000000000000")) -// -// if ac != expac { -// t.Fatalf("%v != %v", ac, expac) -// } -// ad := CommonBitsAddrF(a, d, func() byte { return byte(0x00) }, 10) -// expad := Address(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")) -// -// if ad != expad { -// t.Fatalf("%v != %v", ad, expad) -// } -// ae := CommonBitsAddrF(a, e, func() byte { return byte(0x00) }, 10) -// expae := Address(common.HexToHash("0x0180000000000000000000000000000000000000000000000000000000000000")) -// -// if ae != expae { -// t.Fatalf("%v != %v", ae, expae) -// } -// acf := CommonBitsAddrF(a, c, func() byte { return byte(0xff) }, 10) -// expacf := Address(common.HexToHash("0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")) -// -// if acf != expacf { -// t.Fatalf("%v != %v", acf, expacf) -// } -// aeo := CommonBitsAddrF(a, e, func() byte { return byte(0x00) }, 2) -// expaeo := Address(common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")) -// -// if aeo != expaeo { -// t.Fatalf("%v != %v", aeo, expaeo) -// } -// aep := CommonBitsAddrF(a, e, func() byte { return byte(0xff) }, 2) -// expaep := Address(common.HexToHash("0x3fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")) -// -// if aep != expaep { -// t.Fatalf("%v != %v", aep, expaep) -// } -// -// } -// -// func TestRandomAddressAt(t *testing.T) { -// var a Address -// for i := 0; i < 100; i++ { -// a = RandomAddress() -// prox := rand.Intn(255) -// b := RandomAddressAt(a, prox) -// p, _ := proximity(a, b) -// if p != prox { -// t.Fatalf("incorrect address prox(%v, %v) == %v (expected %v)", a, b, p, prox) -// } -// } -// } -// -// const ( -// maxTestPOs = 1000 -// testPOkeylen = 9 -// ) -// -// func TestPOs(t *testing.T) { -// for i := 0; i < maxTestPOs; i++ { -// length := rand.Intn(256) + 1 -// v0 := RandomAddress().Bin()[:length] -// v1 := RandomAddress().Bin()[:length] -// a0 := NewBoolAddress(v0) -// a1 := NewBoolAddress(v1) -// b0 := NewHashAddress(v0) -// b1 := NewHashAddress(v1) -// pos := rand.Intn(length) + 1 -// apo, aeq := a0.PO(a1, pos) -// bpo, beq := b0.PO(b1, pos) -// if bpo == 256 { -// bpo = length -// } -// a0s := a0.String() -// if a0s != v0 { -// t.Fatalf("incorrect bool address. expected %v, got %v", v0, a0s) -// } -// a1s := a1.String() -// if a1s != v1 { -// t.Fatalf("incorrect bool address. expected %v, got %v", v1, a1s) -// } -// b0s := b0.String()[:length] -// if b0s != v0 { -// t.Fatalf("incorrect hash address. expected %v, got %v", v0, b0s) -// } -// b1s := b1.String()[:length] -// if b1s != v1 { -// t.Fatalf("incorrect hash address. expected %v, got %v", v1, b1s) -// } -// if apo != bpo { -// t.Fatalf("PO does not match for %v X %v (pos: %v): expected %v, got %v", v0, v1, pos, apo, bpo) -// } -// if aeq != beq { -// t.Fatalf("PO equality does not match for %v X %v (pos: %v): expected %v, got %v", v0, v1, pos, aeq, beq) -// } -// } -// } diff --git a/pot/doc.go b/pot/doc.go index 5f01a91c17..47d0357d93 100644 --- a/pot/doc.go +++ b/pot/doc.go @@ -16,23 +16,21 @@ /* Package pot (proximity order tree) implements a container similar to a binary tree. -Value types implement the PoVal interface which provides the PO (proximity order) -comparison operator. +The elements are generic Val interface types. + Each fork in the trie is itself a value. Values of the subtree contained under a node all share the same order when compared to other elements in the tree. Example of proximity order is the length of the common prefix over bitvectors. -(which is equivalent to the order of magnitude of the XOR distance over integers). +(which is equivalent to the reverse rank of order of magnitude of the MSB first X +OR distance over finite set of integers). -The package provides two implementations of PoVal, - -* BoolAddress: an arbitrary length boolean vector -* HashAddress: a bitvector based address derived from 256 bit hash (common.Hash) +Methods take a comparison operator (pof, proximity order function) to compare two +value types. The default pof assumes Val to be or project to a byte slice using +the reverse rank on the MSB first XOR logarithmic disctance. If the address space if limited, equality is defined as the maximum proximity order. -Arbitrary value types can extend these base types or define their own PO method. - The container offers applicative (funcional) style methods on PO trees: * adding/removing en element * swap (value based add/remove) @@ -42,8 +40,10 @@ as well as iterator accessors that respect proximity order When synchronicity of membership if not 100% requirement (e.g. used as a database of network connections), applicative structures have the advantage that nodes -are immutable therefore manipulation does not need locking allowing for parallel retrievals. -For the use case where the entire container is supposed to allow changes by parallel routines, instance methods with write locks and access methods with readlock are provided. The latter only locks while reading the root node. +are immutable therefore manipulation does not need locking allowing for +concurrent retrievals. +For the use case where the entire container is supposed to allow changes by +concurrent routines, Pot * retrieval, insertion and deletion by key involves log(n) pointer lookups @@ -51,10 +51,11 @@ Pot * provide syncronous iterators respecting proximity ordering wrt any item * provide asyncronous iterator (for parallel execution of operations) over n items * allows cheap iteration over ranges -* asymmetric parallellised merge (union) +* asymmetric concurrent merge (union) Note: -* as is, union only makes sense for set representations since which of two values with equal keys survives is random +* as is, union only makes sense for set representations since which of two values +with equal keys survives is random * intersection is not implemented * simple get accessor is not implemented (but derivable from EachNeighbour) @@ -63,16 +64,17 @@ Pinned value on the node implies no need to copy keys of the item type. Note that * the same set of values allows for a large number of alternative POT representations. -* values on the top are accessed faster than lower ones and the steps needed to retrieve items has a logarithmic distribution. +* values on the top are accessed faster than lower ones and the steps needed to +retrieve items has a logarithmic distribution. -As a consequence on can organise the tree so that items that need faster access are torwards the top. -In particular for any subset where popularity has a power distriution that is independent of -proximity order (content addressed storage of chunks), it is in principle possible to create a pot where the steps needed to access an item is inversely proportional to its popularity. +As a consequence one can organise the tree so that items that need faster access +are torwards the top. In particular for any subset where popularity has a power +distriution that is independent of proximity order (content addressed storage of +chunks), it is in principle possible to create a pot where the steps needed to +access an item is inversely proportional to its popularity. Such organisation is not implemented as yet. TODO: -* swap -* get * overwrite-style merge * intersection * access frequency based optimisations diff --git a/pot/pot.go b/pot/pot.go index d7bc459151..18e51f8b3d 100644 --- a/pot/pot.go +++ b/pot/pot.go @@ -26,12 +26,6 @@ const ( maxkeylen = 256 ) -// Pot is the root node type, allows locked non-applicative manipulation -// type Pot struct { -// lock sync.RWMutex -// *Pot -// } - // Pot is the node type (same for root, branching node and leaf) type Pot struct { pin Val @@ -43,10 +37,10 @@ type Pot struct { // Val is the element type for Pots type Val interface{} -// Pof is the proximity order function +// Pof is the proximity order comparison operator function type Pof func(Val, Val, int) (int, bool) -// NewPot constructor. Requires value of type Val to pin +// NewPot constructor. Requires a value of type Val to pin // and po to point to a span in the Val key // The pinned item counts towards the size func NewPot(v Val, po int) *Pot { @@ -68,37 +62,20 @@ func (t *Pot) Pin() Val { // Size returns the number of values in the Pot func (t *Pot) Size() int { - // t.lock.RLock() - // defer t.lock.RUnlock() if t == nil { return 0 } return t.size } -// Add inserts v into the Pot and +// Add inserts a new value into the Pot and // returns the proximity order of v and a boolean // indicating if the item was found -// Add locks the Pot while using applicative add on its Pot -// func (t *Pot) Add(val Val) (po int, found bool) { -// // t.lock.Lock() -// // defer t.lock.Unlock() -// t.Pot, po, found = add(t.Pot, val) -// return po, found -// } - // Add called on (t, v) returns a new Pot that contains all the elements of t // plus the value v, using the applicative add // the second return value is the proximity order of the inserted element // the third is boolean indicating if the item was found -// it only readlocks the Pot while reading its Pot func Add(t *Pot, val Val, pof Pof) (*Pot, int, bool) { - // // t.lock.RLock() - // n := t.Pot - // // t.lock.RUnlock() - // r, po, found := add(n, val) - // return &Pot{Pot: r}, po, found - // // return &Pot{Pot: r}, po, found return add(t, val, pof) } @@ -170,25 +147,11 @@ func add(t *Pot, val Val, pof Pof) (*Pot, int, bool) { // Remove called on (v) deletes v from the Pot and returns // the proximity order of v and a boolean value indicating // if the value was found -// Remove locks Pot while using applicative remove on its Pot -// func (t *Pot) Remove(val Val) (po int, found bool) { -// // t.lock.Lock() -// // defer t.lock.Unlock() -// t.Pot, po, found = remove(t.Pot, val) -// return po, found -// } - // Remove called on (t, v) returns a new Pot that contains all the elements of t // minus the value v, using the applicative remove // the second return value is the proximity order of the inserted element // the third is boolean indicating if the item was found -// it only readlocks the Pot while reading its Pot func Remove(t *Pot, v Val, pof Pof) (*Pot, int, bool) { - // // t.lock.RLock() - // n := t.Pot - // // t.lock.RUnlock() - // r, po, found := remove(n, v) - // return &Pot{Pot: r}, po, found return remove(t, v, pof) } @@ -247,30 +210,11 @@ func remove(t *Pot, val Val, pof Pof) (r *Pot, po int, found bool) { } // Swap called on (k, f) looks up the item at k -// and applies the function f to the value v at k or nil if the item is not found -// if f returns nil, the element is removed -// if f returns v' <> v then v' is inserted into the Pot -// if v' == v the Pot is not changed -// it panics if v'.PO(k, 0) says v and k are not equal -// func (t *Pot) Swap(val Val, f func(v Val) Val) (po int, found bool, change bool) { -// t.lock.Lock() -// defer t.lock.Unlock() -// var t0 *Pot -// t0, po, found, change = swap(t.Pot, val, f) -// if change { -// t.Pot = t0 -// } -// return po, found, change -// } - -// Swap is applicative add, change, remove on a Pot -// func Swap(t *Pot, val Val, f func(v Val) Val) (_ *Pot, po int, found bool, change bool) { -// var t0 *Pot -// t0, po, found, change = swap(t.Pot, val, f) -// return &Pot{Pot: t0}, po, found, change -// } - -// func swap(t *Pot, k Val, f func(v Val) Val) (r *Pot, po int, found bool, change bool) { +// and applies the function f to the value v at k or to nil if the item is not found +// if f(v) returns nil, the element is removed +// if f(v) returns v' <> v then v' is inserted into the Pot +// if (v) == v the Pot is not changed +// it panics if Pof(f(v), k) show that v' and v are not key-equal func Swap(t *Pot, k Val, pof Pof, f func(v Val) Val) (r *Pot, po int, found bool, change bool) { var val Val if t.pin == nil { @@ -325,7 +269,6 @@ func Swap(t *Pot, k Val, pof Pof, f func(v Val) Val) (r *Pot, po int, found bool return t, po, found, false } // recursive change - //size += p.size - n.size bins := append([]*Pot{}, t.bins[:i]...) if p.size == 0 { size-- @@ -371,33 +314,10 @@ func Swap(t *Pot, k Val, pof Pof, f func(v Val) Val) (r *Pot, po int, found bool return r, po, found, true } -// Merge called on (t1) changes t0 to contain all the elements of t1 -// it locks t0, but only readlocks t1 while taking its Pot -// uses applicative union -// func (t *Pot) Merge(t1 *Pot) (c int) { -// t.lock.Lock() -// defer t.lock.Unlock() -// t1.lock.RLock() -// n1 := t1.Pot -// t1.lock.RUnlock() -// t.Pot, c = union(t.Pot, n1) -// return c -// } - -// Union returns the union of t0 and t1 -// it only readlocks the Pot-s to read their Pots and +// Union called on (t0, t1, pof) returns the union of t0 and t1 // calculates the union using the applicative union // the second return value is the number of common elements func Union(t0, t1 *Pot, pof Pof) (*Pot, int) { - // t0.lock.RLock() - // n0 := t0.Pot - // t0.lock.RUnlock() - // t1.lock.RLock() - // n1 := t1.Pot - // t1.lock.RUnlock() - // - // p, c := union(n0, n1) - // return &Pot{Pot: p}, c return union(t0, t1, pof) } @@ -537,10 +457,6 @@ func union(t0, t1 *Pot, pof Pof) (*Pot, int) { // respecting an ordering // proximity > pinnedness func (t *Pot) Each(f func(Val, int) bool) bool { - // t.lock.RLock() - // n := t.Pot - // t.lock.RUnlock() - // return n.each(f) return t.each(f) } @@ -570,10 +486,6 @@ func (t *Pot) each(f func(Val, int) bool) bool { // the iteration ends if the function return false or there are no more elements // end of a po range can be implemented since po is passed to the function func (t *Pot) EachFrom(f func(Val, int) bool, po int) bool { - // t.lock.RLock() - // n := t.Pot - // t.lock.RUnlock() - // return n.eachFrom(f, po) return t.eachFrom(f, po) } @@ -595,10 +507,6 @@ func (t *Pot) eachFrom(f func(Val, int) bool, po int) bool { // the iteration continues until the function's return value is false // or there are no more subtries func (t *Pot) EachBin(val Val, pof Pof, po int, f func(int, int, func(func(val Val, i int) bool) bool) bool) { - // t.lock.RLock() - // n := t.Pot - // t.lock.RUnlock() - // n.eachBin(val, po, f) t.eachBin(val, pof, po, f) } @@ -649,10 +557,6 @@ func (t *Pot) eachBin(val Val, pof Pof, po int, f func(int, int, func(func(val V // the order of elements retrieved reflect proximity order to the target // TODO: add maximum proxbin to start range of iteration func (t *Pot) EachNeighbour(val Val, pof Pof, f func(Val, int) bool) bool { - // t.lock.RLock() - // n := t.Pot - // t.lock.RUnlock() - // return n.eachNeighbour(val, f) return t.eachNeighbour(val, pof, f) } @@ -717,9 +621,6 @@ func (t *Pot) eachNeighbour(val Val, pof Pof, f func(Val, int) bool) bool { // if wait is true, the iterator returns only if all calls to f are finished // TODO: implement minPos for proper prox range iteration func (t *Pot) EachNeighbourAsync(val Val, pof Pof, max int, maxPos int, f func(Val, int), wait bool) { - // t.lock.RLock() - // n := t.Pot - // t.lock.RUnlock() if max > t.size { max = t.size } @@ -727,8 +628,7 @@ func (t *Pot) EachNeighbourAsync(val Val, pof Pof, max int, maxPos int, f func(V if wait { wg = &sync.WaitGroup{} } - // _ = n.eachNeighbourAsync(val, max, maxPos, f, wg) - _ = t.eachNeighbourAsync(val, pof, max, maxPos, f, wg) + t.eachNeighbourAsync(val, pof, max, maxPos, f, wg) if wait { wg.Wait() } diff --git a/pot/pot_test.go b/pot/pot_test.go index b41c64b076..7befdf71ba 100644 --- a/pot/pot_test.go +++ b/pot/pot_test.go @@ -158,7 +158,6 @@ func TestPotRemove(t *testing.T) { func TestPotSwap(t *testing.T) { for i := 0; i < maxSwapTests; i++ { - // alen := maxkeylen alen := maxkeylen pof := DefaultPof(alen) max := rand.Intn(maxSwap) @@ -292,7 +291,6 @@ func TestPotMergeCommon(t *testing.T) { vs := make([]*testAddr, mergeTestCount) for i := 0; i < maxEachNeighbourTests; i++ { alen := maxkeylen - // alen := maxkeylen pof := DefaultPof(alen) for j := 0; j < len(vs); j++ { diff --git a/swarm/network/discovery.go b/swarm/network/discovery.go index db2e2be60b..8b8c83a9cb 100644 --- a/swarm/network/discovery.go +++ b/swarm/network/discovery.go @@ -1,3 +1,19 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + package network import ( @@ -19,27 +35,27 @@ type discPeer struct { } // NewDiscovery discovery peer contructor -func NewDiscovery(p *bzzPeer, o Overlay) *discPeer { - self := &discPeer{ +func newDiscovery(p *bzzPeer, o Overlay) *discPeer { + d := &discPeer{ overlay: o, bzzPeer: p, peers: make(map[string]bool), } - self.seen(self) - return self + d.seen(d) + return d } -func (self *discPeer) HandleMsg(msg interface{}) error { +func (d *discPeer) HandleMsg(msg interface{}) error { switch msg := msg.(type) { case *peersMsg: - return self.handlePeersMsg(msg) + return d.handlePeersMsg(msg) case *getPeersMsg: - return self.handleGetPeersMsg(msg) + return d.handleGetPeersMsg(msg) case *subPeersMsg: - return self.handleSubPeersMsg(msg) + return d.handleSubPeersMsg(msg) default: return fmt.Errorf("unknown message type: %T", msg) @@ -48,8 +64,8 @@ func (self *discPeer) HandleMsg(msg interface{}) error { // NotifyPeer notifies the receiver remote end of a peer p or PO po. // callback for overlay driver -func (self *discPeer) NotifyPeer(a OverlayAddr, po uint8) error { - if po < self.depth || self.seen(a) { +func (d *discPeer) NotifyPeer(a OverlayAddr, po uint8) error { + if po < d.depth || d.seen(a) { return nil } log.Warn(fmt.Sprintf("notification about %x", a.Address())) @@ -57,15 +73,15 @@ func (self *discPeer) NotifyPeer(a OverlayAddr, po uint8) error { resp := &peersMsg{ Peers: []*bzzAddr{ToAddr(a)}, // perhaps the PeerAddr interface is unnecessary generalization } - return self.Send(resp) + return d.Send(resp) } // NotifyDepth sends a subPeers Msg to the receiver notifying them about // a change in the prox limit (radius of the set including the nearest X peers // or first empty row) // callback for overlay driver -func (self *discPeer) NotifyDepth(po uint8) error { - return self.Send(&subPeersMsg{Depth: po}) +func (d *discPeer) NotifyDepth(po uint8) error { + return d.Send(&subPeersMsg{Depth: po}) } /* @@ -91,8 +107,8 @@ type peersMsg struct { Peers []*bzzAddr } -func (self peersMsg) String() string { - return fmt.Sprintf("%T: %v", self, self.Peers) +func (msg peersMsg) String() string { + return fmt.Sprintf("%T: %v", msg, msg.Peers) } // getPeersMsg is sent to (random) peers to request (Max) peers of a specific order @@ -101,8 +117,8 @@ type getPeersMsg struct { Max uint8 } -func (self getPeersMsg) String() string { - return fmt.Sprintf("%T: accept max %v peers of PO%03d", self, self.Max, self.Order) +func (msg getPeersMsg) String() string { + return fmt.Sprintf("%T: accept max %v peers of PO%03d", msg, msg.Max, msg.Order) } // subPeers msg is communicating the depth/sharpness/focus of the overlay table of a peer @@ -110,41 +126,41 @@ type subPeersMsg struct { Depth uint8 } -func (self subPeersMsg) String() string { - return fmt.Sprintf("%T: request peers > PO%02d. ", self, self.Depth) +func (msg subPeersMsg) String() string { + return fmt.Sprintf("%T: request peers > PO%02d. ", msg, msg.Depth) } -func (self *discPeer) handleSubPeersMsg(msg *subPeersMsg) error { - self.depth = msg.Depth - if !self.sentPeers { +func (d *discPeer) handleSubPeersMsg(msg *subPeersMsg) error { + d.depth = msg.Depth + if !d.sentPeers { var peers []*bzzAddr - self.overlay.EachConn(self.Over(), 255, func(p OverlayConn, po int, isproxbin bool) bool { - if uint8(po) < self.depth { + d.overlay.EachConn(d.Over(), 255, func(p OverlayConn, po int, isproxbin bool) bool { + if uint8(po) < d.depth { return false } - if !self.seen(p) { + if !d.seen(p) { peers = append(peers, ToAddr(p.Off())) } return true }) - log.Warn(fmt.Sprintf("found initial %v peers not farther than %v", len(peers), self.depth)) + log.Warn(fmt.Sprintf("found initial %v peers not farther than %v", len(peers), d.depth)) if len(peers) > 0 { - if err := self.Send(&peersMsg{Peers: peers}); err != nil { + if err := d.Send(&peersMsg{Peers: peers}); err != nil { return err } } } - self.sentPeers = true + d.sentPeers = true return nil } // handlePeersMsg called by the protocol when receiving peerset (for target address) // list of nodes ([]PeerAddr in peersMsg) is added to the overlay db using the // Register interface method -func (self *discPeer) handlePeersMsg(msg *peersMsg) error { +func (d *discPeer) handlePeersMsg(msg *peersMsg) error { // register all addresses if len(msg.Peers) == 0 { - log.Debug(fmt.Sprintf("whoops, no peers in incoming peersMsg from %v", self)) + log.Debug(fmt.Sprintf("whoops, no peers in incoming peersMsg from %v", d)) return nil } @@ -152,12 +168,12 @@ func (self *discPeer) handlePeersMsg(msg *peersMsg) error { go func() { defer close(c) for _, a := range msg.Peers { - self.seen(a) + d.seen(a) c <- a } }() log.Info("discovery overlay register") - return self.overlay.Register(c) + return d.overlay.Register(c) } // handleGetPeersMsg is called by the protocol when receiving a @@ -165,29 +181,31 @@ func (self *discPeer) handlePeersMsg(msg *peersMsg) error { // peers suggestions are retrieved from the overlay topology driver // using the EachConn interface iterator method // peers sent are remembered throughout a session and not sent twice -func (self *discPeer) handleGetPeersMsg(msg *getPeersMsg) error { +func (d *discPeer) handleGetPeersMsg(msg *getPeersMsg) error { var peers []*bzzAddr i := 0 - self.overlay.EachConn(self.Over(), int(msg.Order), func(p OverlayConn, po int, isproxbin bool) bool { + d.overlay.EachConn(d.Over(), int(msg.Order), func(p OverlayConn, po int, isproxbin bool) bool { i++ // only send peers we have not sent before in this session a := ToAddr(p.Off()) - if self.seen(a) { + if d.seen(a) { peers = append(peers, a) } return len(peers) < int(msg.Max) }) if len(peers) == 0 { - log.Debug(fmt.Sprintf("no peers found for %v", self)) + log.Debug(fmt.Sprintf("no peers found for %v", d)) return nil } - log.Debug(fmt.Sprintf("%v peers sent to %v", len(peers), self)) + log.Debug(fmt.Sprintf("%v peers sent to %v", len(peers), d)) resp := &peersMsg{ Peers: peers, } - return self.Send(resp) + return d.Send(resp) } +// RequestOrder broadcasts to trageted peers a request for peers of a particular +// proximity order func RequestOrder(k Overlay, order, broadcastSize, maxPeers uint8) { req := &getPeersMsg{ Order: uint8(order), @@ -207,13 +225,13 @@ func RequestOrder(k Overlay, order, broadcastSize, maxPeers uint8) { log.Info(fmt.Sprintf("requesting bees of PO%03d from %v/%v (each max %v)", order, i, broadcastSize, maxPeers)) } -func (self *discPeer) seen(p OverlayPeer) bool { - self.mtx.Lock() - defer self.mtx.Unlock() +func (d *discPeer) seen(p OverlayPeer) bool { + d.mtx.Lock() + defer d.mtx.Unlock() k := string(p.Address()) - if self.peers[k] { + if d.peers[k] { return true } - self.peers[k] = true + d.peers[k] = true return false } diff --git a/swarm/network/discovery_test.go b/swarm/network/discovery_test.go index 70f1fc7c94..ee90683a73 100644 --- a/swarm/network/discovery_test.go +++ b/swarm/network/discovery_test.go @@ -1,3 +1,19 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + package network import ( @@ -18,7 +34,7 @@ func TestDiscovery(t *testing.T) { to := NewKademlia(addr.OAddr, NewKadParams()) run := func(p *bzzPeer) error { - dp := NewDiscovery(p, to) + dp := newDiscovery(p, to) to.On(p) defer to.Off(p) log.Trace(fmt.Sprintf("kademlia on %v", p)) diff --git a/swarm/network/hive.go b/swarm/network/hive.go index ab492201c5..c7b6c62dd6 100644 --- a/swarm/network/hive.go +++ b/swarm/network/hive.go @@ -163,7 +163,7 @@ func (h *Hive) Stop() { // Run protocol run function func (h *Hive) Run(p *bzzPeer) error { - dp := NewDiscovery(p, h) + dp := newDiscovery(p, h) log.Debug(fmt.Sprintf("to add new bee %v", p)) h.On(dp) h.wake() diff --git a/swarm/network/hive_test.go b/swarm/network/hive_test.go index c621e97241..14b1fc2f0b 100644 --- a/swarm/network/hive_test.go +++ b/swarm/network/hive_test.go @@ -1,3 +1,19 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + package network import ( diff --git a/swarm/network/kademlia.go b/swarm/network/kademlia.go index e2cfb20443..2483f58ff1 100644 --- a/swarm/network/kademlia.go +++ b/swarm/network/kademlia.go @@ -299,10 +299,10 @@ func (k *Kademlia) On(p OverlayConn) { } depth := uint8(k.depth()) + var depthChanged bool if depth != k.currentDepth { + depthChanged = true k.currentDepth = depth - } else { - depth = 0 } go np.NotifyDepth(depth) @@ -310,7 +310,7 @@ func (k *Kademlia) On(p OverlayConn) { dp := val.(*entry).OverlayPeer.(Notifier) dp.NotifyPeer(p.Off(), uint8(po)) log.Trace(fmt.Sprintf("peer %v notified of %v (%v)", dp, p, po)) - if depth > 0 { + if depthChanged { dp.NotifyDepth(depth) log.Trace(fmt.Sprintf("peer %v notified of new depth %v", dp, depth)) } @@ -442,9 +442,8 @@ func (k *Kademlia) String() string { liverows := make([]string, k.MaxProxDisplay) peersrows := make([]string, k.MaxProxDisplay) - var depth int - prev := -1 - var depthSet bool + + depth := k.depth() rest := k.conns.Size() k.conns.EachBin(k.base, pof, 0, func(po, size int, f func(func(val pot.Val, i int) bool) bool) bool { var rowlen int @@ -459,14 +458,9 @@ func (k *Kademlia) String() string { rowlen++ return rowlen < 4 }) - if !depthSet && (po > prev+1 || rest < k.MinProxBinSize) { - depthSet = true - depth = prev + 1 - } r := strings.Join(row, " ") r = r + wsrow liverows[po] = r[:31] - prev = po return true }) diff --git a/swarm/network/kademlia_test.go b/swarm/network/kademlia_test.go index ddf3613923..28d182fa5e 100644 --- a/swarm/network/kademlia_test.go +++ b/swarm/network/kademlia_test.go @@ -13,6 +13,7 @@ // // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . + package network import ( @@ -27,9 +28,8 @@ import ( ) func init() { - h := log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))) - // h := log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stderr, log.TerminalFormat(true))) - // h := log.CallerFileHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(true))) + // h := log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))) + h := log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stderr, log.TerminalFormat(true))) log.Root().SetHandler(h) } @@ -65,26 +65,26 @@ type dropError struct { addr string } -func (self *testDropPeer) Drop(err error) { - err2 := &dropError{err, binStr(self)} - self.dropc <- err2 +func (d *testDropPeer) Drop(err error) { + err2 := &dropError{err, binStr(d)} + d.dropc <- err2 } -func (self *testDiscPeer) NotifyDepth(po uint8) error { - key := binStr(self) - self.lock.Lock() - defer self.lock.Unlock() - self.notifications[key] = po +func (d *testDiscPeer) NotifyDepth(po uint8) error { + key := binStr(d) + d.lock.Lock() + defer d.lock.Unlock() + d.notifications[key] = po return nil } -func (self *testDiscPeer) NotifyPeer(p OverlayAddr, po uint8) error { - key := binStr(self) +func (d *testDiscPeer) NotifyPeer(p OverlayAddr, po uint8) error { + key := binStr(d) key += binStr(p) - self.lock.Lock() - defer self.lock.Unlock() + d.lock.Lock() + defer d.lock.Unlock() log.Trace(fmt.Sprintf("key %v=>%v", key, po)) - self.notifications[key] = po + d.notifications[key] = po return nil } @@ -432,31 +432,31 @@ func TestPruning(t *testing.T) { func TestKademliaHiveString(t *testing.T) { k := newTestKademlia("00000000").On("01000000", "00100000").Register("10000000", "10000001") h := k.String() - expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), MinProxBinSize: 2, MinBinSize: 1, MaxBinSize: 4\n============ DEPTH: 0 ==========================================\n000 0 | 2 8100 (0) 8000 (0)\n001 1 4000 | 1 4000 (0)\n002 1 2000 | 1 2000 (0)\n003 0 | 0\n004 0 | 0\n005 0 | 0\n006 0 | 0\n007 0 | 0\n=========================================================================" + expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), MinProxBinSize: 2, MinBinSize: 1, MaxBinSize: 4\n000 0 | 2 8100 (0) 8000 (0)\n============ DEPTH: 1 ==========================================\n001 1 4000 | 1 4000 (0)\n002 1 2000 | 1 2000 (0)\n003 0 | 0\n004 0 | 0\n005 0 | 0\n006 0 | 0\n007 0 | 0\n=========================================================================" if expH[100:] != h[100:] { t.Fatalf("incorrect hive output. expected %v, got %v", expH, h) } } -func (self *testKademlia) checkNotifications(npeers []*testPeerNotification, nprox []*testDepthNotification) error { +func (k *testKademlia) checkNotifications(npeers []*testPeerNotification, nprox []*testDepthNotification) error { for _, pn := range npeers { key := pn.rec + pn.addr - po, found := self.notifications[key] + po, found := k.notifications[key] if !found || pn.po != po { return fmt.Errorf("%v, expected to have notified %v about peer %v (%v)", key, pn.rec, pn.addr, pn.po) } - delete(self.notifications, key) + delete(k.notifications, key) } for _, pn := range nprox { key := pn.rec - po, found := self.notifications[key] + po, found := k.notifications[key] if !found || pn.po != po { return fmt.Errorf("expected to have notified %v about new prox limit %v", pn.rec, pn.po) } - delete(self.notifications, key) + delete(k.notifications, key) } - if len(self.notifications) > 0 { - return fmt.Errorf("%v unexpected notifications", len(self.notifications)) + if len(k.notifications) > 0 { + return fmt.Errorf("%v unexpected notifications", len(k.notifications)) } return nil } diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index c93f2968cf..3d29b9b9ea 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -33,10 +33,11 @@ import ( ) const ( - NetworkId = 322 // BZZ in l33t + NetworkID = 322 // BZZ in l33t ProtocolMaxMsgSize = 10 * 1024 * 1024 ) +// BzzHandshakeSpec is the spec of the generic swarm handshake var BzzHandshakeSpec = &protocols.Spec{ Name: "bzz", Version: 1, @@ -211,7 +212,7 @@ func (b *Bzz) getHandshake(peerID discover.NodeID) *bzzHandshake { if !ok { handshake = &bzzHandshake{ Version: uint64(BzzHandshakeSpec.Version), - NetworkId: uint64(NetworkId), + NetworkID: uint64(NetworkID), Addr: b.localAddr, done: make(chan struct{}), } @@ -250,12 +251,12 @@ func (self *bzzPeer) LastActive() time.Time { Handshake * Version: 8 byte integer version of the protocol -* NetworkId: 8 byte integer network identifier +* NetworkID: 8 byte integer network identifier * Addr: the address advertised by the node including underlay and overlay connecctions */ type bzzHandshake struct { Version uint64 - NetworkId uint64 + NetworkID uint64 Addr *bzzAddr // peerAddr is the address received in the peer handshake @@ -266,7 +267,7 @@ type bzzHandshake struct { } func (self *bzzHandshake) String() string { - return fmt.Sprintf("Handshake: Version: %v, NetworkId: %v, Addr: %v", self.Version, self.NetworkId, self.Addr) + return fmt.Sprintf("Handshake: Version: %v, NetworkID: %v, Addr: %v", self.Version, self.NetworkID, self.Addr) } const bzzHandshakeTimeout = time.Second @@ -284,8 +285,8 @@ func (self *bzzHandshake) Perform(p *p2p.Peer, rw p2p.MsgReadWriter) (err error) return err } rhs := hs.(*bzzHandshake) - if rhs.NetworkId != self.NetworkId { - return fmt.Errorf("network id mismatch %d (!= %d)", rhs.NetworkId, self.NetworkId) + if rhs.NetworkID != self.NetworkID { + return fmt.Errorf("network id mismatch %d (!= %d)", rhs.NetworkID, self.NetworkID) } if rhs.Version != self.Version { return fmt.Errorf("version mismatch %d (!= %d)", rhs.Version, self.Version) diff --git a/swarm/network/protocol_test.go b/swarm/network/protocol_test.go index 897003d8c3..002fcf596d 100644 --- a/swarm/network/protocol_test.go +++ b/swarm/network/protocol_test.go @@ -1,3 +1,19 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + package network import ( @@ -140,12 +156,12 @@ func (s *bzzTester) runHandshakes(ids ...discover.NodeID) { func correctBzzHandshake(addr *bzzAddr) *bzzHandshake { return &bzzHandshake{ Version: 0, - NetworkId: 322, + NetworkID: 322, Addr: addr, } } -func TestBzzHandshakeNetworkIdMismatch(t *testing.T) { +func TestBzzHandshakeNetworkIDMismatch(t *testing.T) { pp := p2ptest.NewTestPeerPool() addr := RandomAddr() s := newBzzTester(t, 1, addr, pp, nil, nil) @@ -154,7 +170,7 @@ func TestBzzHandshakeNetworkIdMismatch(t *testing.T) { id := s.IDs[0] s.testHandshake( correctBzzHandshake(addr), - &bzzHandshake{Version: 0, NetworkId: 321, Addr: NewAddrFromNodeID(id)}, + &bzzHandshake{Version: 0, NetworkID: 321, Addr: NewAddrFromNodeID(id)}, &p2ptest.Disconnect{Peer: id, Error: fmt.Errorf("network id mismatch 321 (!= 322)")}, ) } @@ -168,7 +184,7 @@ func TestBzzHandshakeVersionMismatch(t *testing.T) { id := s.IDs[0] s.testHandshake( correctBzzHandshake(addr), - &bzzHandshake{Version: 1, NetworkId: 322, Addr: NewAddrFromNodeID(id)}, + &bzzHandshake{Version: 1, NetworkID: 322, Addr: NewAddrFromNodeID(id)}, &p2ptest.Disconnect{Peer: id, Error: fmt.Errorf("version mismatch 1 (!= 0)")}, ) } @@ -182,6 +198,6 @@ func TestBzzHandshakeSuccess(t *testing.T) { id := s.IDs[0] s.testHandshake( correctBzzHandshake(addr), - &bzzHandshake{Version: 0, NetworkId: 322, Addr: NewAddrFromNodeID(id)}, + &bzzHandshake{Version: 0, NetworkID: 322, Addr: NewAddrFromNodeID(id)}, ) } diff --git a/swarm/network/test_overlay.go b/swarm/network/test_overlay.go deleted file mode 100644 index ebcbbc3108..0000000000 --- a/swarm/network/test_overlay.go +++ /dev/null @@ -1,193 +0,0 @@ -package network - -// -// import ( -// "fmt" -// "strings" -// "sync" -// -// "github.com/ethereum/go-ethereum/log" -// ) -// -// const orders = 8 -// -// type testOverlay struct { -// mu sync.Mutex -// addr []byte -// pos [][]OverlayAddr -// posMap map[string]OverlayAddr -// } -// -// type testPeerAddr struct { -// Addr -// Peer -// } -// -// func (self *testPeerAddr) Address() []byte { -// return nil -// } -// -// func (self *testPeerAddr) Update(a OverlayAddr) OverlayAddr { -// return self -// } -// -// func (self *testPeerAddr) On(p OverlayConn) OverlayConn { -// return self -// } -// -// func (self *testPeerAddr) Off() OverlayAddr { -// return self -// } -// -// func (self *testOverlay) Register(peers chan OverlayAddr) error { -// self.mu.Lock() -// defer self.mu.Unlock() -// var nas []OverlayAddr -// for a := range peers { -// nas = append(nas, a) -// } -// return self.register(nas...) -// } -// -// func (self *testOverlay) BaseAddr() []byte { -// return nil -// } -// -// func (self *testOverlay) register(nas ...OverlayAddr) error { -// for _, na := range nas { -// addr := na.Address() -// if self.posMap[string(addr)] != nil { -// continue -// } -// self.posMap[string(addr)] = na -// o := order(addr) -// log.Trace(fmt.Sprintf("PO: %v, orders: %v", o, orders)) -// self.pos[o] = append(self.pos[o], na) -// } -// return nil -// } -// -// func order(addr []byte) int { -// return int(addr[0]) / 32 -// } -// -// func (self *testOverlay) On(n OverlayConn) { -// self.mu.Lock() -// defer self.mu.Unlock() -// addr := n.Address() -// na := self.posMap[string(addr)] -// if na == nil { -// self.register(n) -// na = self.posMap[string(addr)] -// } else if na.Peer != nil { -// return -// } -// log.Trace(fmt.Sprintf("Online: %x", addr[:4])) -// na.Peer = n -// return -// } -// -// func (self *testOverlay) Off(n OverlayConn) { -// self.mu.Lock() -// defer self.mu.Unlock() -// addr := n.Over() -// na := self.posMap[string(addr)] -// if na == nil { -// return -// } -// delete(self.posMap, string(addr)) -// na.Peer = nil -// } -// -// // caller must hold the lock -// func (self *testOverlay) on(po []*testPeerAddr) (nodes []OverlayConn) { -// for _, na := range po { -// if na.Peer != nil { -// nodes = append(nodes, na) -// } -// } -// return nodes -// } -// -// // caller must hold the lock -// func (self *testOverlay) off(po []*testPeerAddr) (nas []OverlayAddr) { -// for _, na := range po { -// if na.Peer == (*bzzPeer)(nil) { -// nas = append(nas, Addr(na)) -// } -// } -// return nas -// } -// -// func (self *testOverlay) EachConn(base []byte, o int, f func(OverlayConn, int, bool) bool) { -// for i := o; i < len(self.pos); i++ { -// for _, na := range self.pos[i] { -// if na.Peer != nil { -// if !f(na, o, false) { -// return -// } -// } -// } -// } -// } -// -// func (self *testOverlay) EachAddr(base []byte, o int, f func(OverlayAddr, int) bool) { -// for i := o; i < len(self.pos); i++ { -// for _, na := range self.pos[i] { -// if !f(na, i) { -// return -// } -// } -// } -// } -// -// func (self *testOverlay) SuggestPeer() (OverlayAddr, int, bool) { -// self.mu.Lock() -// defer self.mu.Unlock() -// for i, po := range self.pos { -// ons := self.on(po) -// if len(ons) < 2 { -// offs := self.off(po) -// if len(offs) > 0 { -// log.Trace(fmt.Sprintf("node %v is off", offs[0])) -// return offs[0], i, true -// } -// } -// } -// return nil, 0, true -// } -// -// func (self *testOverlay) String() string { -// self.mu.Lock() -// defer self.mu.Unlock() -// var t []string -// var ons, offs int -// var ns []Peer -// var nas []Addr -// for o, po := range self.pos { -// var row []string -// ns = self.on(po) -// nas = self.off(po) -// ons = len(ns) -// for _, n := range ns { -// addr := n.Over() -// row = append(row, fmt.Sprintf("%x", addr[:4])) -// } -// row = append(row, "|") -// offs = len(nas) -// for _, na := range nas { -// addr := na.Over() -// row = append(row, fmt.Sprintf("%x", addr[:4])) -// } -// t = append(t, fmt.Sprintf("%v: (%v/%v) %v", o, ons, offs, strings.Join(row, " "))) -// } -// return strings.Join(t, "\n") -// } -// -// func NewTestOverlay(addr []byte) *testOverlay { -// return &testOverlay{ -// addr: addr, -// posMap: make(map[string]*testPeerAddr), -// pos: make([][]*testPeerAddr, orders), -// } -// } From 61b45b767abca2cdec2707f0f8781c69e0fdbcac Mon Sep 17 00:00:00 2001 From: zelig Date: Sat, 17 Jun 2017 17:06:28 +0200 Subject: [PATCH 10/17] pot, swarm/network: cleanup, fix depth in kad table --- swarm/api/config.go | 2 +- swarm/network/kademlia.go | 4 ++++ swarm/swarm.go | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/swarm/api/config.go b/swarm/api/config.go index 3095e89517..ad16e3be84 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -88,7 +88,7 @@ func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, n // if not set in function param, then set default for swarm network, will be overwritten by config file if present if networkId == 0 { - self.NetworkId = network.NetworkId + self.NetworkId = network.NetworkID } if err != nil { diff --git a/swarm/network/kademlia.go b/swarm/network/kademlia.go index 2483f58ff1..0205c8404a 100644 --- a/swarm/network/kademlia.go +++ b/swarm/network/kademlia.go @@ -345,6 +345,10 @@ func (k *Kademlia) Off(p OverlayConn) { func (k *Kademlia) EachConn(base []byte, o int, f func(OverlayConn, int, bool) bool) { k.lock.RLock() defer k.lock.RUnlock() + k.eachConn(base, o, f) +} + +func (k *Kademlia) eachConn(base []byte, o int, f func(OverlayConn, int, bool) bool) { if len(base) == 0 { base = k.base } diff --git a/swarm/swarm.go b/swarm/swarm.go index 71334ba661..d2b764e600 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -133,7 +133,7 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api. dpaChunkStore := storage.NewDpaChunkStore(self.lstore, self.storage) log.Debug(fmt.Sprintf("-> Local Access to Swarm")) - // Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage + // Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage self.dpa = storage.NewDPA(dpaChunkStore, self.config.ChunkerParams) log.Debug(fmt.Sprintf("-> Content Store API")) @@ -368,7 +368,7 @@ func NewLocalSwarm(datadir, port string) (self *Swarm, err error) { return } - config, err := api.NewConfig(datadir, common.Address{}, prvKey, network.NetworkId) + config, err := api.NewConfig(datadir, common.Address{}, prvKey, network.NetworkID) if err != nil { return } From 65023d7d4bf63cc9e985ff0411048d6cdc3aba7b Mon Sep 17 00:00:00 2001 From: lash Date: Tue, 20 Jun 2017 21:30:46 +0200 Subject: [PATCH 11/17] swarm, swarm/network, swarm/pss, p2p/simulations: cleanup + doc --- .../simulations/discovery/discovery_test.go | 41 ++- swarm/pss/client/client.go | 34 +-- swarm/pss/doc.go | 183 ++++++++++++++ swarm/pss/pss.go | 202 +-------------- swarm/pss/pss_test.go | 235 +++++------------- swarm/pss/pssapi.go | 3 +- swarm/pss/testdata/snapshot_10.json | 1 + swarm/pss/testdata/snapshot_5.json | 1 + swarm/pss/testdata/snapshot_50.json | 1 + swarm/pss/types.go | 6 +- swarm/swarm.go | 20 +- swarm/swarm_psschat.go | 18 -- 12 files changed, 331 insertions(+), 414 deletions(-) create mode 100644 swarm/pss/doc.go create mode 100755 swarm/pss/testdata/snapshot_10.json create mode 100755 swarm/pss/testdata/snapshot_5.json create mode 100755 swarm/pss/testdata/snapshot_50.json diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go index d91d8ba33e..caeb227d47 100644 --- a/swarm/network/simulations/discovery/discovery_test.go +++ b/swarm/network/simulations/discovery/discovery_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "flag" "io/ioutil" "os" "testing" @@ -27,13 +28,23 @@ var services = adapters.Services{ serviceName: newService, } +var ( + snapshotFile = flag.String("snapshot", "", "create snapshot") + nodeCount = flag.Int("nodes", 10, "number of nodes to create (default 10)") + verbose = flag.Bool("verbose", false, "output extra logs") +) + func init() { + flag.Parse() // register the discovery service which will run as a devp2p // protocol when using the exec adapter adapters.RegisterServices(services) - // log.Root().SetHandler(log.LvlFilterHandler(log.LvlError, log.StreamHandler(os.Stderr, log.TerminalFormat(false)))) - log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(false)))) + if *verbose { + log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(false)))) + } else { + log.Root().SetHandler(log.LvlFilterHandler(log.LvlError, log.StreamHandler(os.Stderr, log.TerminalFormat(false)))) + } } func TestDiscoverySimulationDockerAdapter(t *testing.T) { @@ -59,15 +70,14 @@ func TestDiscoverySimulationSimAdapter(t *testing.T) { func testDiscoverySimulation(t *testing.T, adapter adapters.NodeAdapter) { // create network - nodeCount := 10 net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{ ID: "0", DefaultService: serviceName, }) defer net.Shutdown() trigger := make(chan discover.NodeID) - ids := make([]discover.NodeID, nodeCount) - for i := 0; i < nodeCount; i++ { + ids := make([]discover.NodeID, *nodeCount) + for i := 0; i < *nodeCount; i++ { node, err := net.NewNode() if err != nil { t.Fatalf("error starting node: %s", err) @@ -136,13 +146,22 @@ func testDiscoverySimulation(t *testing.T, adapter adapters.NodeAdapter) { t.Fatalf("simulation failed: %s", result.Error) } - snap, err := net.Snapshot() - if err != nil { - t.Fatalf("no shapshot dude") + if *snapshotFile != "" { + snap, err := net.Snapshot() + if err != nil { + t.Fatalf("no shapshot dude") + } + jsonsnapshot, err := json.Marshal(snap) + if err != nil { + t.Fatalf("corrupt json snapshot: %v", err) + } + log.Info("writing snapshot", "file", *snapshotFile) + err = ioutil.WriteFile(*snapshotFile, jsonsnapshot, os.ModePerm) + if err != nil { + t.Fatal(err) + } } - jsonsnapshot, err := json.Marshal(snap) - ioutil.WriteFile("jsonsnapshot.txt", jsonsnapshot, os.ModePerm) - + t.Log("Simulation Passed:") t.Logf("Duration: %s", result.FinishedAt.Sub(result.StartedAt)) for _, id := range ids { diff --git a/swarm/pss/client/client.go b/swarm/pss/client/client.go index 0b3f7918ab..48942d912c 100644 --- a/swarm/pss/client/client.go +++ b/swarm/pss/client/client.go @@ -18,12 +18,12 @@ // "github.com/ethereum/go-ethereum/pot" // "github.com/ethereum/go-ethereum/log" // ) -// +// // type FooMsg struct { // Bar int // } -// -// +// +// // func fooHandler (msg interface{}) error { // foomsg, ok := msg.(*FooMsg) // if ok { @@ -31,7 +31,7 @@ // } // return fmt.Errorf("Unknown message") // } -// +// // spec := &protocols.Spec{ // Name: "foo", // Version: 1, @@ -40,7 +40,7 @@ // FooMsg{}, // }, // } -// +// // proto := &p2p.Protocol{ // Name: spec.Name, // Version: spec.Version, @@ -51,7 +51,7 @@ // }, // } // -// func implementation() { +// func implementation() { // cfg := pss.NewClientConfig() // psc := pss.NewClient(context.Background(), nil, cfg) // err := psc.Start() @@ -61,21 +61,21 @@ // } // // log.Debug("connected to pss node", "bzz addr", psc.BaseAddr) -// -// err = psc.RunProtocol(proto) +// +// err = psc.RunProtocol(proto) // if err != nil { // log.Crit("can't start protocol on pss websocket") // os.Exit(1) // } -// +// // addr := pot.RandomAddress() // should be a real address, of course // psc.AddPssPeer(addr, spec) -// +// // // use the protocol for something -// +// // psc.Stop() -// } -// +// } +// // BUG(test): TestIncoming test times out due to deadlock issues in the swarm hive package client @@ -86,11 +86,11 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/protocols" "github.com/ethereum/go-ethereum/pot" - "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm/pss" @@ -125,7 +125,7 @@ func NewClientConfig() *ClientConfig { // After a successful connection with Client.Start, BaseAddr contains the swarm overlay address of the pss node type Client struct { - BaseAddr []byte + BaseAddr []byte localuri string remoteuri string ctx context.Context @@ -234,7 +234,7 @@ func NewClientWithRPC(ctx context.Context, rpcclient *rpc.Client) (*Client, erro protos: make(map[pss.Topic]*p2p.Protocol), ws: rpcclient, ctx: ctx, - BaseAddr: oaddr, + BaseAddr: oaddr, }, nil } @@ -267,7 +267,7 @@ func (self *Client) Start() error { } // Mounts a new devp2p protcool on the pss connection -// the protocol is aliased as a "pss topic" +// the protocol is aliased as a "pss topic" // uses normal devp2p Send and incoming message handler routines from the p2p/protocols package // // when an incoming message is received from a peer that is not yet known to the client, this peer object is instantiated, and the protocol is run on it. diff --git a/swarm/pss/doc.go b/swarm/pss/doc.go new file mode 100644 index 0000000000..5ab4ed80d1 --- /dev/null +++ b/swarm/pss/doc.go @@ -0,0 +1,183 @@ +// pss provides devp2p functionality for swarm nodes without the need for a direct tcp connection between them. +// +// It uses swarm kademlia routing to send and receive messages. Routing is deterministic and will seek the shortest route available on the network. +// +// Messages are encapsulated in a devp2p message structure `PssMsg`. These capsules are forwarded from node to node using ordinary tcp devp2p until it reaches it's destination. The destination address is hinted in `PssMsg.To` +// +// The content of a PssMsg can be anything at all, down to a simple, non-descript byte-slices. But convenience methods are made available to implement devp2p protocol functionality on top of it. +// +// In its final implementation, pss is intended to become "shh over bzz," that is; "whisper over swarm." Specifically, this means that the emphemeral encryption envelopes of whisper will be used to obfuscate the correspondance. Ideally, the unencrypted content of the PssMsg will only contain a part of the address of the recipient, where the final recipient is the one who matches this partial address *and* successfully can encrypt the message. +// +// For the current state and roadmap of pss development please see https://github.com/ethersphere/swarm/wiki/swarm-dev-progress. +// +// Please report issues on https://github.com/ethersphere/go-ethereum +// +// Feel free to ask questions in https://gitter.im/ethersphere/pss +// +// TLDR IMPLEMENTATION +// +// Most developers will most probably want to use the protocol-wrapping convenience client in swarm/pss/client. Documentation and a minimal code example for the latter is found in the package documentation. The pss API can of course also be used directly. The client implementation provides a clear illustration of its intended usage. +// +// pss implements the node.Service interface. This means that the API methods will be auto-magically exposed to any RPC layer the node activates. In particular, pss provides subscription to incoming messages using the go-ethereum rpc websocket layer. +// +// The important API methods are: +// - Receive() - start a subscription to receive new incoming messages matching specific "topics" +// - Send() - send content over pss to a specified recipient +// +// +// LOWLEVEL IMPLEMENTATION +// +// code speaks louder than words: +// +// import ( +// "io/ioutil" +// "os" +// "github.com/ethereum/go-ethereum/p2p" +// "github.com/ethereum/go-ethereum/log" +// "github.com/ethereum/go-ethereum/swarm/pss" +// "github.com/ethereum/go-ethereum/swarm/network" +// "github.com/ethereum/go-ethereum/swarm/storage" +// ) +// +// var ( +// righttopic = pss.NewTopic("foo", 4) +// wrongtopic = pss.NewTopic("bar", 2) +// ) +// +// func init() { +// hs := log.StreamHandler(os.Stderr, log.TerminalFormat(true)) +// hf := log.LvlFilterHandler(log.LvlTrace, hs) +// h := log.CallerFileHandler(hf) +// log.Root().SetHandler(h) +// } +// +// +// // Pss.Handler type +// func handler(msg []byte, p *p2p.Peer, from []byte) error { +// log.Debug("received", "msg", msg, "from", from, "forwarder", p.ID()) +// return nil +// } +// +// func implementation() { +// +// // bogus addresses for illustration purposes +// meaddr := network.RandomAddr() +// toaddr := network.RandomAddr() +// fwdaddr := network.RandomAddr() +// +// // new kademlia for routing +// kp := network.NewKadParams() +// to := network.NewKademlia(meaddr.Over(), kp) +// +// // new (local) storage for cache +// cachedir, err := ioutil.TempDir("", "pss-cache") +// if err != nil { +// panic("overlay") +// } +// dpa, err := storage.NewLocalDPA(cachedir) +// if err != nil { +// panic("storage") +// } +// +// // setup pss +// psp := pss.NewPssParams(false) +// ps := pss.NewPss(to, dpa, psp) +// +// // does nothing but please include it +// ps.Start(nil) +// +// dereg := ps.Register(&righttopic, handler) +// +// // in its simplest form a message is just a byteslice +// payload := []byte("foobar") +// +// // send a raw message +// err = ps.SendRaw(toaddr.Over(), righttopic, payload) +// log.Error("Fails. Not connect, so nothing in kademlia. But it illustrates the point.", "err", err) +// +// // forward a full message +// envfwd := pss.NewEnvelope(fwdaddr.Over(), righttopic, payload) +// msgfwd := &pss.PssMsg{ +// To: toaddr.Over(), +// Payload: envfwd, +// } +// err = ps.Forward(msgfwd) +// log.Error("Also fails, same reason. I wish, I wish, I wish there was somebody out there.", "err", err) +// +// // process an incoming message +// // (this is the first step after the devp2p PssMsg message handler) +// envme := pss.NewEnvelope(toaddr.Over(), righttopic, payload) +// msgme := &pss.PssMsg{ +// To: meaddr.Over(), +// Payload: envme, +// } +// err = ps.Process(msgme) +// if err == nil { +// log.Info("this works :)") +// } +// +// // if we don't have a registered topic it fails +// dereg() // remove the previously registered topic-handler link +// ps.Process(msgme) +// log.Error("It fails as we expected", "err", err) +// +// // does nothing but please include it +// ps.Stop() +// } +// +// MESSAGE STRUCTURE +// +// NOTE! This part is subject to change. In particular the envelope structure will be re-implemented using whisper. +// +// A pss message has the following layers: +// +// PssMsg +// Contains (eventually only part of) recipient address, and (eventually) encrypted Envelope. +// +// Envelope +// Currently rlp-encoded. Contains the Payload, along with sender address, topic and expiry information. +// +// Payload +// Byte-slice of arbitrary data +// +// ProtocolMsg +// An optional convenience structure for implementation of devp2p protocols. Contains Code, Size and Payload analogous to the p2p.Msg structure, where the payload is a rlp-encoded byteslice. For transport, this struct is serialized and used as the "payload" above. +// +// TOPICS AND PROTOCOLS +// +// Pure pss is protocol agnostic. Instead it uses the notion of Topic. This is NOT the "subject" of a message. Instead this type is used to internally register handlers for messages matching respective Topics. +// +// Topic in this context virtually mean anything; protocols, chatrooms, or social media groups. +// +// When implementing devp2p protocols, topics are direct mappings to protocols name and version. The pss package provides the PssProtocol convenience structure, and a generic Handler that can be passed to Pss.Register. This makes it possible to use the same message handler code for pss that are used for direct connected peers. +// +// CONNECTIONS +// +// A "connection" in pss is a purely virtual construct. There is no mechanisms in place to ensure that the remote peer actually is there. In fact, "adding" a peer involves merely the node's opinion that the peer is there. It may issue messages to that remote peer to a directly connected peer, which in turn passes it on. But if it is not present on the network - or if there is no route to it - the message will never reach its destination through mere forwarding. +// +// When implementing the devp2p protocol stack, the "adding" of a remote peer is a prerequisite for the side actually initiating the protocol communication. Adding a peer in effect "runs" the protocol on that peer, and adds an internal mapping between a topic and that peer. It also enables sending and receiving messages using the main io-construct in devp2p - the p2p.MsgReadWriter. +// +// Under the hood, pss implements its own MsgReadWriter, which bridges MsgReadWriter.WriteMsg with Pss.SendRaw, and deftly adds an InjectMsg method which pipes incoming messages to appear on the MsgReadWriter.ReadMsg channel. +// +// An incoming connection is nothing more than an actual PssMsg appearing with a certain Topic. If a Handler har been registered to that Topic, the message will be passed to it. This constitutes a "new" connection if: +// +// - The pss node never called AddPeer with this combination of remote peer address and topic, and +// +// - The pss node never received a PssMsg from this remote peer with this specific Topic before. +// +// If it is a "new" connection, the protocol will be "run" on the remote peer, in the same manner as if it was pre-emptively added. +// +// ROUTING AND CACHING +// +// (please refer to swarm kademlia routing for an explanation of the routing algorithm used for pss) +// +// pss implements a simple caching mechanism, using the swarm DPA for storage of the messages and generation of the digest keys used in the cache table. The caching is intended to alleviate the following: +// +// - save messages so that they can be delivered later if the recipient was not online at the time of sending. +// +// - drop an identical message to the same recipient if received within a given time interval +// +// - prevent backwards routing of messages +// +// the latter may occur if only one entry is in the receiving node's kademlia. In this case the forwarder will be provided as the "nearest node" to the final recipient. The cache keeps the address of who the message was forwarded from, and if the cache lookup matches, the message will be dropped. +package pss diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index ccf7ad03b6..1a2b4328a7 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -1,185 +1,3 @@ -// pss provides devp2p functionality for swarm nodes without the need for a direct tcp connection between them. -// -// It uses swarm kademlia routing to send and receive messages. Routing is deterministic and will seek the shortest route available on the network. -// -// Messages are encapsulated in a devp2p message structure `PssMsg`. These capsules are forwarded from node to node using ordinary tcp devp2p until it reaches it's destination. The destination address is hinted in `PssMsg.To` -// -// The content of a PssMsg can be anything at all, down to a simple, non-descript byte-slices. But convenience methods are made available to implement devp2p protocol functionality on top of it. -// -// In its final implementation, pss is intended to become "shh over bzz," that is; "whisper over swarm." Specifically, this means that the emphemeral encryption envelopes of whisper will be used to obfuscate the correspondance. Ideally, the unencrypted content of the PssMsg will only contain a part of the address of the recipient, where the final recipient is the one who matches this partial address *and* successfully can encrypt the message. -// -// For the current state and roadmap of pss development please see https://github.com/ethersphere/swarm/wiki/swarm-dev-progress. -// -// Please report issues on https://github.com/ethersphere/go-ethereum -// -// Feel free to ask questions in https://gitter.im/ethersphere/pss -// -// TLDR IMPLEMENTATION -// -// Most developers will most probably want to use the protocol-wrapping convenience client in swarm/pss/client. Documentation and a minimal code example for the latter is found in the package documentation. The pss API can of course also be used directly. The client implementation provides a clear illustration of its intended usage. -// -// pss implements the node.Service interface. This means that the API methods will be auto-magically exposed to any RPC layer the node activates. In particular, pss provides subscription to incoming messages using the go-ethereum rpc websocket layer. -// -// The important API methods are: -// - Receive() - start a subscription to receive new incoming messages matching specific "topics" -// - Send() - send content over pss to a specified recipient -// -// -// LOWLEVEL IMPLEMENTATION -// -// code speaks louder than words: -// -// import ( -// "io/ioutil" -// "os" -// "github.com/ethereum/go-ethereum/p2p" -// "github.com/ethereum/go-ethereum/log" -// "github.com/ethereum/go-ethereum/swarm/pss" -// "github.com/ethereum/go-ethereum/swarm/network" -// "github.com/ethereum/go-ethereum/swarm/storage" -// ) -// -// var ( -// righttopic = pss.NewTopic("foo", 4) -// wrongtopic = pss.NewTopic("bar", 2) -// ) -// -// func init() { -// hs := log.StreamHandler(os.Stderr, log.TerminalFormat(true)) -// hf := log.LvlFilterHandler(log.LvlTrace, hs) -// h := log.CallerFileHandler(hf) -// log.Root().SetHandler(h) -// } -// -// -// // Pss.Handler type -// func handler(msg []byte, p *p2p.Peer, from []byte) error { -// log.Debug("received", "msg", msg, "from", from, "forwarder", p.ID()) -// return nil -// } -// -// func implementation() { -// -// // bogus addresses for illustration purposes -// meaddr := network.RandomAddr() -// toaddr := network.RandomAddr() -// fwdaddr := network.RandomAddr() -// -// // new kademlia for routing -// kp := network.NewKadParams() -// to := network.NewKademlia(meaddr.Over(), kp) -// -// // new (local) storage for cache -// cachedir, err := ioutil.TempDir("", "pss-cache") -// if err != nil { -// panic("overlay") -// } -// dpa, err := storage.NewLocalDPA(cachedir) -// if err != nil { -// panic("storage") -// } -// -// // setup pss -// psp := pss.NewPssParams(false) -// ps := pss.NewPss(to, dpa, psp) -// -// // does nothing but please include it -// ps.Start(nil) -// -// dereg := ps.Register(&righttopic, handler) -// -// // in its simplest form a message is just a byteslice -// payload := []byte("foobar") -// -// // send a raw message -// err = ps.SendRaw(toaddr.Over(), righttopic, payload) -// log.Error("Fails. Not connect, so nothing in kademlia. But it illustrates the point.", "err", err) -// -// // forward a full message -// envfwd := pss.NewEnvelope(fwdaddr.Over(), righttopic, payload) -// msgfwd := &pss.PssMsg{ -// To: toaddr.Over(), -// Payload: envfwd, -// } -// err = ps.Forward(msgfwd) -// log.Error("Also fails, same reason. I wish, I wish, I wish there was somebody out there.", "err", err) -// -// // process an incoming message -// // (this is the first step after the devp2p PssMsg message handler) -// envme := pss.NewEnvelope(toaddr.Over(), righttopic, payload) -// msgme := &pss.PssMsg{ -// To: meaddr.Over(), -// Payload: envme, -// } -// err = ps.Process(msgme) -// if err == nil { -// log.Info("this works :)") -// } -// -// // if we don't have a registered topic it fails -// dereg() // remove the previously registered topic-handler link -// ps.Process(msgme) -// log.Error("It fails as we expected", "err", err) -// -// // does nothing but please include it -// ps.Stop() -// } -// -// MESSAGE STRUCTURE -// -// NOTE! This part is subject to change. In particular the envelope structure will be re-implemented using whisper. -// -// A pss message has the following layers: -// -// PssMsg -// Contains (eventually only part of) recipient address, and (eventually) encrypted Envelope. -// -// Envelope -// Currently rlp-encoded. Contains the Payload, along with sender address, topic and expiry information. -// -// Payload -// Byte-slice of arbitrary data -// -// ProtocolMsg -// An optional convenience structure for implementation of devp2p protocols. Contains Code, Size and Payload analogous to the p2p.Msg structure, where the payload is a rlp-encoded byteslice. For transport, this struct is serialized and used as the "payload" above. -// -// TOPICS AND PROTOCOLS -// -// Pure pss is protocol agnostic. Instead it uses the notion of Topic. This is NOT the "subject" of a message. Instead this type is used to internally register handlers for messages matching respective Topics. -// -// Topic in this context virtually mean anything; protocols, chatrooms, or social media groups. -// -// When implementing devp2p protocols, topics are direct mappings to protocols name and version. The pss package provides the PssProtocol convenience structure, and a generic Handler that can be passed to Pss.Register. This makes it possible to use the same message handler code for pss that are used for direct connected peers. -// -// CONNECTIONS -// -// A "connection" in pss is a purely virtual construct. There is no mechanisms in place to ensure that the remote peer actually is there. In fact, "adding" a peer involves merely the node's opinion that the peer is there. It may issue messages to that remote peer to a directly connected peer, which in turn passes it on. But if it is not present on the network - or if there is no route to it - the message will never reach its destination through mere forwarding. -// -// When implementing the devp2p protocol stack, the "adding" of a remote peer is a prerequisite for the side actually initiating the protocol communication. Adding a peer in effect "runs" the protocol on that peer, and adds an internal mapping between a topic and that peer. It also enables sending and receiving messages using the main io-construct in devp2p - the p2p.MsgReadWriter. -// -// Under the hood, pss implements its own MsgReadWriter, which bridges MsgReadWriter.WriteMsg with Pss.SendRaw, and deftly adds an InjectMsg method which pipes incoming messages to appear on the MsgReadWriter.ReadMsg channel. -// -// An incoming connection is nothing more than an actual PssMsg appearing with a certain Topic. If a Handler har been registered to that Topic, the message will be passed to it. This constitutes a "new" connection if: -// -// - The pss node never called AddPeer with this combination of remote peer address and topic, and -// -// - The pss node never received a PssMsg from this remote peer with this specific Topic before. -// -// If it is a "new" connection, the protocol will be "run" on the remote peer, in the same manner as if it was pre-emptively added. -// -// ROUTING AND CACHING -// -// (please refer to swarm kademlia routing for an explanation of the routing algorithm used for pss) -// -// pss implements a simple caching mechanism, using the swarm DPA for storage of the messages and generation of the digest keys used in the cache table. The caching is intended to alleviate the following: -// -// - save messages so that they can be delivered later if the recipient was not online at the time of sending. -// -// - drop an identical message to the same recipient if received within a given time interval -// -// - prevent backwards routing of messages -// -// the latter may occur if only one entry is in the receiving node's kademlia. In this case the forwarder will be provided as the "nearest node" to the final recipient. The cache keeps the address of who the message was forwarded from, and if the cache lookup matches, the message will be dropped. package pss import ( @@ -203,8 +21,8 @@ import ( const ( PssPeerCapacity = 256 // limit of peers kept in cache. (not implemented) - PssPeerTopicDefaultCapacity = 8 // limit of topics kept per peer. (not implemented) - digestLength = 32 // byte length of digest used for pss cache (currently same as swarm chunk hash) + PssPeerTopicDefaultCapacity = 8 // limit of topics kept per peer. (not implemented) + digestLength = 32 // byte length of digest used for pss cache (currently same as swarm chunk hash) digestCapacity = 256 // cache entry limit (not implement) ) @@ -242,7 +60,7 @@ type pssDigest [digestLength]byte type Pss struct { network.Overlay // we can get the overlayaddress from this peerPool map[pot.Address]map[Topic]p2p.MsgReadWriter // keep track of all virtual p2p.Peers we are currently speaking to - fwdPool map[discover.NodeID]*protocols.Peer // keep track of all peers sitting on the pssmsg routing layer + fwdPool map[discover.NodeID]*protocols.Peer // keep track of all peers sitting on the pssmsg routing layer handlers map[Topic]map[*Handler]bool // topic and version based pss payload handlers fwdcache map[pssDigest]pssCacheEntry // checksum of unique fields from pssmsg mapped to expiry, cache to determine whether to drop msg cachettl time.Duration // how long to keep messages in fwdcache @@ -292,13 +110,12 @@ func (self *Pss) Start(srv *p2p.Server) error { return nil } - // For node.Service implementation. Does nothing for now, but should be included in the code for backwards compatibility. func (self *Pss) Stop() error { return nil } -// devp2p protocol object for the PssMsg struct. +// devp2p protocol object for the PssMsg struct. // // This represents the PssMsg capsule, and is the entry point for processing, receiving and sending pss messages between directly connected peers. func (self *Pss) Protocols() []p2p.Protocol { @@ -507,10 +324,11 @@ func (self *Pss) Forward(msg *PssMsg) error { log.Crit("Pss cannot use kademlia peer type") return false } + //sendMsg := fmt.Sprintf("MSG %x TO %x FROM %x VIA %x", digest, common.ByteLabel(msg.To), common.ByteLabel(self.BaseAddr()), common.ByteLabel(op.Address())) sendMsg := fmt.Sprintf("TO %x FROM %x VIA %x", common.ByteLabel(msg.To), common.ByteLabel(self.BaseAddr()), common.ByteLabel(op.Address())) pp := self.fwdPool[sp.ID()] if self.checkFwdCache(op.Address(), digest) { - log.Info("%v: peer already forwarded to", sendMsg) + log.Info(fmt.Sprintf("%v: peer already forwarded to", sendMsg)) return true } err := pp.Send(msg) @@ -518,7 +336,7 @@ func (self *Pss) Forward(msg *PssMsg) error { log.Warn(fmt.Sprintf("%v: failed forwarding: %v", sendMsg, err)) return true } - log.Trace(fmt.Sprintf("%v: successfully forwarded", sendMsg)) + log.Debug(fmt.Sprintf("%v: successfully forwarded", sendMsg)) sent++ // if equality holds, p is always the first peer given in the iterator if bytes.Equal(msg.To, op.Address()) || !isproxbin { @@ -628,7 +446,6 @@ func (prw PssReadWriter) injectMsg(msg p2p.Msg) error { return nil } - // For devp2p protocol integration only. // // Convenience object for passing messages in and out of the p2p layer @@ -640,7 +457,7 @@ type PssProtocol struct { } // For devp2p protocol integration only. -// +// // Maps a Topic to a devp2p protocol. func RegisterPssProtocol(ps *Pss, topic *Topic, spec *protocols.Spec, targetprotocol *p2p.Protocol) *PssProtocol { pp := &PssProtocol{ @@ -654,7 +471,7 @@ func RegisterPssProtocol(ps *Pss, topic *Topic, spec *protocols.Spec, targetprot // For devp2p protocol integration only. // -// Generic handler for initiating devp2p-like protocol connections +// Generic handler for initiating devp2p-like protocol connections // // This handler should be passed to Pss.Register with the associated ropic. func (self *PssProtocol) Handle(msg []byte, p *p2p.Peer, senderAddr []byte) error { @@ -680,4 +497,3 @@ func (self *PssProtocol) Handle(msg []byte, p *p2p.Peer, senderAddr []byte) erro return nil } - diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index b80bd476d5..2cfdedb512 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -34,18 +34,33 @@ const ( var ( snapshotfile string + debugflag = flag.Bool("v", false, "verbose") + + // custom logging + psslogmain log.Logger + ) var services = newServices() func init() { + + flag.Parse() + rand.Seed(time.Now().Unix()) + adapters.RegisterServices(services) + + loglevel := log.LvlDebug + if *debugflag { + loglevel = log.LvlTrace + } + + psslogmain = log.New("psslog", "*") hs := log.StreamHandler(os.Stderr, log.TerminalFormat(true)) - hf := log.LvlFilterHandler(log.LvlTrace, hs) + hf := log.LvlFilterHandler(loglevel, hs) h := log.CallerFileHandler(hf) log.Root().SetHandler(h) - flag.StringVar(&snapshotfile, "file", "snapsnot.json", "snapshot file") } func TestCache(t *testing.T) { @@ -243,72 +258,49 @@ func TestSimpleLinear(t *testing.T) { } } -func TestFullRandom50n(t *testing.T) { - adapter := adapters.NewSimAdapter(services) - testFullRandom(t, adapter, 50, 50) +func TestSnapshot_50_5(t *testing.T) { + testSnapshot(t, "testdata/snapshot_50.json", 5, true) } -func TestFullRandom25n(t *testing.T) { - adapter := adapters.NewSimAdapter(services) - testFullRandom(t, adapter, 25, 25) +func TestSnapshot_5_50(t *testing.T) { + testSnapshot(t, "testdata/snapshot_5.json", 50, true) } -func TestFullRandom10n(t *testing.T) { - adapter := adapters.NewSimAdapter(services) - testFullRandom(t, adapter, 10, 10) +func TestSnapshot_5_5(t *testing.T) { + testSnapshot(t, "testdata/snapshot_5.json", 5, true) } -func TestFullRandom5n(t *testing.T) { - baseDir, err := ioutil.TempDir("", "swarm-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(baseDir) - adapter := adapters.NewExecAdapter(baseDir) - testFullRandom(t, adapter, 5, 5) -} - -func testFullRandom(t *testing.T, adapter adapters.NodeAdapter, nodecount int, msgcount int) { -} - -func TestFullRandomSnapshot50(t *testing.T) { - testFullRandomSnapshot(t, true, 5) -} - -func testFullRandomSnapshot(t *testing.T, sim bool, msgcount int) { - - var msgtoids []discover.NodeID - var msgreceived []discover.NodeID - var cancelmain func() - //var triggerptr *chan discover.NodeID +func testSnapshot(t *testing.T, snapshotfile string, msgcount int, sim bool) { - baseDir, err := ioutil.TempDir("", "swarm-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(baseDir) - + // choose the adapter to use var adapter adapters.NodeAdapter if sim { adapter = adapters.NewSimAdapter(services) } else { + baseDir, err := ioutil.TempDir("", "swarm-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(baseDir) adapter = adapters.NewExecAdapter(baseDir) } - wg := sync.WaitGroup{} - wg.Add(msgcount) - - psslog := make(map[discover.NodeID]log.Logger) - psslogmain := log.New("psslog", "*") - + // process shapshot jsonsnapshot, err := ioutil.ReadFile(snapshotfile) if err != nil { t.Fatalf("cant read snapshot: %s", snapshotfile) } snapshot := &simulations.Snapshot{} err = json.Unmarshal(jsonsnapshot, snapshot) + if err != nil { + t.Fatalf("snapshot file unreadable: %v", err) + } + for _, node := range snapshot.Nodes { + node.Config.Services = []string{"bzz", "pss"} + } + // setup network with snapshot net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{ ID: "0", }) @@ -319,121 +311,30 @@ func testFullRandomSnapshot(t *testing.T, sim bool, msgcount int) { t.Fatalf("invalid snapshot: %v", err) } - //msgtoids := make([]discover.NodeID, msgcount) - timeout := 15 * time.Second ctx, cancelmain := context.WithTimeout(context.Background(), timeout) defer cancelmain() - trigger := make(chan discover.NodeID) - //triggerptr = &trigger + // nodes expecting messages + recvids := make([]discover.NodeID, msgcount) + // the overlay address map to recvids recvaddrs := make(map[discover.NodeID][]byte) -// for i = 0; i < nodecount; i++ { -// nodeconfig := adapters.RandomNodeConfig() -// nodeconfig.Services = []string{"bzz", "pss"} -// node, err := net.NewNodeWithConfig(nodeconfig) -// if err != nil { -// t.Fatalf("error starting node: %s", err) -// } -// -// if err := net.Start(node.ID()); err != nil { -// t.Fatalf("error starting node %s: %s", node.ID().TerminalString(), err) -// } -// -// if err := triggerChecks(ctx, &wg, triggerptr, net, node.ID()); err != nil { -// t.Fatal("error triggering checks for node %s: %s", node.ID().TerminalString(), err) -// } -// ids[i] = node.ID() -// if i < fullnodecount { -// fullpeers[ids[i]] = network.ToOverlayAddr(node.ID().Bytes()) -// psslog[ids[i]] = log.New("psslog", fmt.Sprintf("%x", fullpeers[ids[i]])) -// } -// log.Debug("psslog starting node", "id", nodeconfig.ID) -// } -// -// for i, id := range fullids { -// msgfromids = append(msgfromids, id) -// msgtoids[i] = fullids[(i+(len(fullids)/2)+1)%len(fullids)] -// } + // messages actually received (registered through trigger and test check) + var msgreceived []discover.NodeID - // run a simulation which connects the 10 nodes in a ring and waits - // for full peer discovery -// action := func(ctx context.Context) error { -// for i, id := range ids { -// peerID := ids[(i+1)%len(ids)] -// if net.GetConn(id, peerID) != nil { -// continue -// } -// if err := net.Connect(id, peerID); err != nil { -// return err -// } -// psslog[id].Debug("conn ok", "one", id, "other", peerID) -// } -// return nil -// } -// check := func(ctx context.Context, id discover.NodeID) (bool, error) { -// select { -// case <-ctx.Done(): -// wg.Done() -// psslog[id].Error("conn failed!", "id", id) -// return false, ctx.Err() -// default: -// } -// var tgt []byte -// var fwd struct { -// Addr []byte -// Count int -// } -// -// for i, fid := range msgfromids { -// if id == fid { -// tgt = network.ToOverlayAddr(msgtoids[(i+(len(msgtoids)/2)+1)%len(msgtoids)].Bytes()) -// break -// } -// } -// p := net.GetNode(id) -// if p == nil { -// return false, fmt.Errorf("Unknown node: %v", id) -// } -// c, err := p.Client() -// if err != nil { -// return false, err -// } -// for fwd.Count < 2 { -// c.CallContext(context.Background(), &fwd, "pss_getForwarder", tgt) -// time.Sleep(time.Microsecond * 250) -// } -// psslog[id].Debug("fwd check ok", "topaddr", fmt.Sprintf("%x", common.ByteLabel(fwd.Addr)), "kadcount", fwd.Count) -// return true, nil -// } -// -// result := simulations.NewSimulation(net).Run(ctx, &simulations.Step{ -// Action: action, -// Trigger: trigger, -// Expect: &simulations.Expectation{ -// Nodes: ids, -// Check: check, -// }, -// }) -// if result.Error != nil { -// t.Fatalf("simulation failed: %s", result.Error) -// cancelmain() -// } -// -// trigger = make(chan discover.NodeID) -// triggerptr = &trigger + // trigger for expect in test + trigger := make(chan discover.NodeID) - var ids []discover.NodeID + // one wait for every message + wg := sync.WaitGroup{} + wg.Add(msgcount) action := func(ctx context.Context) error { var rpcerr error var rpcbyte []byte - //for ii, id := range msgfromids { for _, simnode := range net.Nodes { - ids = append(ids, simnode.ID()) - //node := net.GetNode(id) if simnode == nil { return fmt.Errorf("unknown node: %s", simnode.ID()) } @@ -452,17 +353,22 @@ func testFullRandomSnapshot(t *testing.T, sim bool, msgcount int) { if err != nil { t.Fatalf("cant get overlayaddr: %v", err) } + + err = triggerChecks(ctx, &wg, &trigger, net, simnode.ID()) + if err != nil { + t.Fatalf("trigger setup failed: %v", err) + } } for i := 0; i < msgcount; i++ { idx := rand.Intn(len(net.Nodes)) sendernode := net.Nodes[idx] toidx := rand.Intn(len(net.Nodes)-1) - if idx >= toidx { + if toidx >= idx { toidx++ } - recvnode := net.Nodes[toidx] + recvids[i] = recvnode.ID() msg := PingMsg{Created: time.Now()} code, _ := PingProtocol.GetCode(&PingMsg{}) pmsg, _ := NewProtocolMsg(code, msg) @@ -472,7 +378,6 @@ func testFullRandomSnapshot(t *testing.T, sim bool, msgcount int) { return fmt.Errorf("error getting sendernode client: %s", err) } client.CallContext(ctx, &rpcerr, "pss_send", PingTopic, APIMsg{ - //Addr: fullpeers[msgtoids[ii]], Addr: recvaddrs[recvnode.ID()], Msg: pmsg, }) @@ -483,15 +388,14 @@ func testFullRandomSnapshot(t *testing.T, sim bool, msgcount int) { return nil } check := func(ctx context.Context, id discover.NodeID) (bool, error) { - select { - case <-ctx.Done(): - wg.Done() - return false, ctx.Err() - default: + case <-ctx.Done(): + wg.Done() + return false, ctx.Err() + default: } msgreceived = append(msgreceived, id) - psslog[id].Info("trigger received", "id", id, "len", len(msgreceived)) + psslogmain.Info("trigger received", "id", id, "len", len(msgreceived)) wg.Done() return true, nil } @@ -500,8 +404,7 @@ func testFullRandomSnapshot(t *testing.T, sim bool, msgcount int) { Action: action, Trigger: trigger, Expect: &simulations.Expectation{ - //Nodes: msgtoids, - Nodes: ids, + Nodes: recvids, Check: check, }, }) @@ -511,13 +414,14 @@ func testFullRandomSnapshot(t *testing.T, sim bool, msgcount int) { t.Fatalf("simulation failed: %s", result.Error) } - if len(msgreceived) != len(msgtoids) { - t.Fatalf("Simulation Failed, got %d of %d msgs", len(msgreceived), len(msgtoids)) + wg.Wait() + + if len(msgreceived) != msgcount { + t.Fatalf("Simulation Failed, got %d of %d msgs", len(msgreceived), msgcount) } - wg.Wait() psslogmain.Info("done!") - t.Logf("Simulation Passed, got %d of %d msgs", len(msgreceived), len(msgtoids)) + t.Logf("Simulation Passed, got %d of %d msgs", len(msgreceived), msgcount) //t.Logf("Duration: %s", result.FinishedAt.Sub(result.StartedAt)) } @@ -527,7 +431,6 @@ func testFullRandomSnapshot(t *testing.T, sim bool, msgcount int) { func triggerChecks(ctx context.Context, wg *sync.WaitGroup, trigger *chan discover.NodeID, net *simulations.Network, id discover.NodeID) error { quitC := make(chan struct{}) - got := false node := net.GetNode(id) if node == nil { @@ -555,12 +458,8 @@ func triggerChecks(ctx context.Context, wg *sync.WaitGroup, trigger *chan discov defer peersub.Unsubscribe() for { select { - case event := <-peerevents: - if event.Type == "add" && !got { - got = true - *trigger <- id - } case <-msgevents: + psslogmain.Debug("incoming msg", "node", id) *trigger <- id case err := <-peersub.Err(): if err != nil { diff --git a/swarm/pss/pssapi.go b/swarm/pss/pssapi.go index 0b5173d556..5ac25aecd1 100644 --- a/swarm/pss/pssapi.go +++ b/swarm/pss/pssapi.go @@ -24,7 +24,7 @@ func NewAPI(ps *Pss) *API { // // A new handler is registered in pss for the supplied topic // -// All incoming messages to the node matching this topic will be encapsulated in the APIMsg struct and sent to the subscriber +// All incoming messages to the node matching this topic will be encapsulated in the APIMsg struct and sent to the subscriber func (pssapi *API) Receive(ctx context.Context, topic Topic) (*rpc.Subscription, error) { notifier, supported := rpc.NotifierFromContext(ctx) if !supported { @@ -111,4 +111,3 @@ func (pssapitest *APITest) GetForwarder(addr []byte) (fwd struct { }) return } - diff --git a/swarm/pss/testdata/snapshot_10.json b/swarm/pss/testdata/snapshot_10.json new file mode 100755 index 0000000000..1ea8203525 --- /dev/null +++ b/swarm/pss/testdata/snapshot_10.json @@ -0,0 +1 @@ +{"nodes":[{"config":{"id":"6bcea2a040f92c275e8c35a9c5dda5ed630b68a643cdb24bab1aecff717517fac479f086763bbcda64c97b8367f78656c41176310940eb026da5d4115129196f","private_key":"bb415e2cd8967b7ebb5f075bd9bab3573a0de5f53a10b8871730d83ae67a1875","name":"node01","services":["discovery"]},"up":true},{"config":{"id":"04afa3365d7d5e6e98888d2b1bfd527abcd5aa771aa8b65b207370b0ad161046b9d1138f455662b712c957c0e3e3c91f71dc4b43d137739eefcdde50fd24738c","private_key":"807019cf99c358ce2885c8d7a6e8e1de6b6b28d53fd851d9cd6ad0fbdedd1d40","name":"node02","services":["discovery"]},"up":true},{"config":{"id":"5a2e841b4fccae7af62694cf941ffabe52bb083ee5b6f47d8639b305b0961fe009355af77e039b0305d24b9e13e82b3431999d759a39900e298d84ef8d360ad2","private_key":"a2abcb3cdff46fe4e565fa692476dff5e38d7fbeb74fc31a8e900488c91c7bce","name":"node03","services":["discovery"]},"up":true},{"config":{"id":"4b2410992c11e220c4eebe63cde453f87201dce7e90568a18ae2fe54d99274823c96920b7daf67ffc8a3c167d594a7691d5a562dbfab5bcf8d4db43cc8f01752","private_key":"5a7fda28aa3e26e49eba40c0e0cbb5dddf829f157ca8cafcaff6c078d9aee418","name":"node04","services":["discovery"]},"up":true},{"config":{"id":"551fdbcd23e1f6e479409663ed5874f473272d1a7774e3d07a66b86067e0f4c82a8f2443d1583efc8b1ab0e5e8dedab7370911a2e4031c3b1f0a52d8caf56c70","private_key":"84a006d28ac13c6b3f297cb0965ae10b1c08598513125d7d7914533535f6e05e","name":"node05","services":["discovery"]},"up":true},{"config":{"id":"c8dfb4203273de9fa04a9e539114a852fcced30ebfab9d1b5aec07e0618278d0f00b4c7659b1fbb7f3faa89875a412ac0581c2db27cc27c6a5e21e6a1fa60575","private_key":"d727b5e9f2680b0f8a5d3b8afec74eedc6ffbc7a16e23f20a0f9a6286c9377c4","name":"node06","services":["discovery"]},"up":true},{"config":{"id":"3869317fb9824e6249a78004395f736f46738431ef2f60af7b51bb6e78ce5c9138504f32683a5faf61027c81da63e64f4afeb739b22cc83337e85ab90a52f9f5","private_key":"55af4c0788f39b9aa5a4d990c87f0ee2880e4c9008b25e1ac6ae704ca6a98c05","name":"node07","services":["discovery"]},"up":true},{"config":{"id":"35bc83bc11ecb374a9c6d9039346fca5e67bb699c69c93fbdf945c675f64ca3c65a6ed4f96e224f89551c45c696cb4d223928f61ef62d27b160a270ce29eab93","private_key":"4241aecb9bfb72b0c9795d87de2130c328a69bd1bcf46e466f2979ecacfd1120","name":"node08","services":["discovery"]},"up":true},{"config":{"id":"fe86dc105ebe680de6e1b42addd61844791eb7931ccafdb8aede566198384f1193bab26033d6e88210345de5a0cb88148575dde247a044c4e72289603b94eb36","private_key":"9941d05811e966083376b24bb99692358e140f3447f9bfdb6d287eee41bd09e6","name":"node09","services":["discovery"]},"up":true},{"config":{"id":"99c06185c7ebf35d2b7448b0c747d5c7ceb5ae4da653e0d62fbc6611835743ff253be7a5151a147d28d852356e803bb60c2030466b50c8d9deab6416293c6462","private_key":"de3537adbb50ef6610754c06a721f1a8ba78d98a60eb035400a914c2672823aa","name":"node10","services":["discovery"]},"up":true}],"conns":[{"one":"6bcea2a040f92c275e8c35a9c5dda5ed630b68a643cdb24bab1aecff717517fac479f086763bbcda64c97b8367f78656c41176310940eb026da5d4115129196f","other":"99c06185c7ebf35d2b7448b0c747d5c7ceb5ae4da653e0d62fbc6611835743ff253be7a5151a147d28d852356e803bb60c2030466b50c8d9deab6416293c6462","up":true,"reverse":true,"distance":68},{"one":"04afa3365d7d5e6e98888d2b1bfd527abcd5aa771aa8b65b207370b0ad161046b9d1138f455662b712c957c0e3e3c91f71dc4b43d137739eefcdde50fd24738c","other":"6bcea2a040f92c275e8c35a9c5dda5ed630b68a643cdb24bab1aecff717517fac479f086763bbcda64c97b8367f78656c41176310940eb026da5d4115129196f","up":true,"reverse":false,"distance":69},{"one":"5a2e841b4fccae7af62694cf941ffabe52bb083ee5b6f47d8639b305b0961fe009355af77e039b0305d24b9e13e82b3431999d759a39900e298d84ef8d360ad2","other":"04afa3365d7d5e6e98888d2b1bfd527abcd5aa771aa8b65b207370b0ad161046b9d1138f455662b712c957c0e3e3c91f71dc4b43d137739eefcdde50fd24738c","up":true,"reverse":false,"distance":69},{"one":"4b2410992c11e220c4eebe63cde453f87201dce7e90568a18ae2fe54d99274823c96920b7daf67ffc8a3c167d594a7691d5a562dbfab5bcf8d4db43cc8f01752","other":"5a2e841b4fccae7af62694cf941ffabe52bb083ee5b6f47d8639b305b0961fe009355af77e039b0305d24b9e13e82b3431999d759a39900e298d84ef8d360ad2","up":true,"reverse":true,"distance":71},{"one":"551fdbcd23e1f6e479409663ed5874f473272d1a7774e3d07a66b86067e0f4c82a8f2443d1583efc8b1ab0e5e8dedab7370911a2e4031c3b1f0a52d8caf56c70","other":"4b2410992c11e220c4eebe63cde453f87201dce7e90568a18ae2fe54d99274823c96920b7daf67ffc8a3c167d594a7691d5a562dbfab5bcf8d4db43cc8f01752","up":true,"reverse":false,"distance":71},{"one":"c8dfb4203273de9fa04a9e539114a852fcced30ebfab9d1b5aec07e0618278d0f00b4c7659b1fbb7f3faa89875a412ac0581c2db27cc27c6a5e21e6a1fa60575","other":"551fdbcd23e1f6e479409663ed5874f473272d1a7774e3d07a66b86067e0f4c82a8f2443d1583efc8b1ab0e5e8dedab7370911a2e4031c3b1f0a52d8caf56c70","up":true,"reverse":true,"distance":65},{"one":"3869317fb9824e6249a78004395f736f46738431ef2f60af7b51bb6e78ce5c9138504f32683a5faf61027c81da63e64f4afeb739b22cc83337e85ab90a52f9f5","other":"c8dfb4203273de9fa04a9e539114a852fcced30ebfab9d1b5aec07e0618278d0f00b4c7659b1fbb7f3faa89875a412ac0581c2db27cc27c6a5e21e6a1fa60575","up":false,"reverse":false,"distance":65},{"one":"35bc83bc11ecb374a9c6d9039346fca5e67bb699c69c93fbdf945c675f64ca3c65a6ed4f96e224f89551c45c696cb4d223928f61ef62d27b160a270ce29eab93","other":"3869317fb9824e6249a78004395f736f46738431ef2f60af7b51bb6e78ce5c9138504f32683a5faf61027c81da63e64f4afeb739b22cc83337e85ab90a52f9f5","up":true,"reverse":false,"distance":76},{"one":"fe86dc105ebe680de6e1b42addd61844791eb7931ccafdb8aede566198384f1193bab26033d6e88210345de5a0cb88148575dde247a044c4e72289603b94eb36","other":"35bc83bc11ecb374a9c6d9039346fca5e67bb699c69c93fbdf945c675f64ca3c65a6ed4f96e224f89551c45c696cb4d223928f61ef62d27b160a270ce29eab93","up":true,"reverse":false,"distance":65},{"one":"99c06185c7ebf35d2b7448b0c747d5c7ceb5ae4da653e0d62fbc6611835743ff253be7a5151a147d28d852356e803bb60c2030466b50c8d9deab6416293c6462","other":"fe86dc105ebe680de6e1b42addd61844791eb7931ccafdb8aede566198384f1193bab26033d6e88210345de5a0cb88148575dde247a044c4e72289603b94eb36","up":true,"reverse":false,"distance":65}]} \ No newline at end of file diff --git a/swarm/pss/testdata/snapshot_5.json b/swarm/pss/testdata/snapshot_5.json new file mode 100755 index 0000000000..88938605be --- /dev/null +++ b/swarm/pss/testdata/snapshot_5.json @@ -0,0 +1 @@ +{"nodes":[{"config":{"id":"78c9669ee7b6b66642c69547501bddc4ffb5b3887c2b889b30b761fe245fee05db56bc892c6e6502a4d036c9c86ce7f9f0975d7b8b1ffebfdb8109335f364b4a","private_key":"d264be17451dbbeec8cffa553fdc9717dcc76b4ad895db7bbc672016470b6e3a","name":"node01","services":["discovery"]},"up":true},{"config":{"id":"0eec1942b5c7bea584e03d9d0f8b3219161daa68635457fd33e703b1b1512b1dd2ae27bd575a72b4748f6c85b00006dd5a08be0b8eacd6cbd8d90124b3874c7a","private_key":"b5057fc754be217f79b594072c0ef2d35c306435c8b1de75358852e92ab33ca7","name":"node02","services":["discovery"]},"up":true},{"config":{"id":"8d84d276b03161f58af3911b10da2dc33cfc93391130ed89db0f7a84bd6323b0237dab2f3c8b5c7d9953cdc745125dd67ee1122cb73a583caa98359f01afb415","private_key":"a00eea5fae5de6ca8e8d7b622ab1ede0f8adeb6fbb55a00dcd9783b760812cdd","name":"node03","services":["discovery"]},"up":true},{"config":{"id":"fdeaa376c517fe1b4574eb859d92df4272d344409ae9ec42f1c3f21e386c88b8dddd5d2febb9851e450c7e2a72113d419a6e52bf63e6bfbdde00259062ed3098","private_key":"3de00ba717baa58589e0562051f4ce6a2650c0bdab330d9eb55786fecbab2ec7","name":"node04","services":["discovery"]},"up":true},{"config":{"id":"a26035f22d085ad5c8f3892f16ed4867b6a09952c589d0177af8372666e44a454a3e49b43456c25a8365015f450432f4a6b328810e740dd2eadb898857b83e7b","private_key":"1aa2ca63a68c03f8dbc1a5a50cd7994af15fe860e1d2c7e97c0b99cd9f295532","name":"node05","services":["discovery"]},"up":true}],"conns":[{"one":"78c9669ee7b6b66642c69547501bddc4ffb5b3887c2b889b30b761fe245fee05db56bc892c6e6502a4d036c9c86ce7f9f0975d7b8b1ffebfdb8109335f364b4a","other":"a26035f22d085ad5c8f3892f16ed4867b6a09952c589d0177af8372666e44a454a3e49b43456c25a8365015f450432f4a6b328810e740dd2eadb898857b83e7b","up":true,"reverse":true,"distance":65},{"one":"0eec1942b5c7bea584e03d9d0f8b3219161daa68635457fd33e703b1b1512b1dd2ae27bd575a72b4748f6c85b00006dd5a08be0b8eacd6cbd8d90124b3874c7a","other":"78c9669ee7b6b66642c69547501bddc4ffb5b3887c2b889b30b761fe245fee05db56bc892c6e6502a4d036c9c86ce7f9f0975d7b8b1ffebfdb8109335f364b4a","up":true,"reverse":false,"distance":69},{"one":"8d84d276b03161f58af3911b10da2dc33cfc93391130ed89db0f7a84bd6323b0237dab2f3c8b5c7d9953cdc745125dd67ee1122cb73a583caa98359f01afb415","other":"0eec1942b5c7bea584e03d9d0f8b3219161daa68635457fd33e703b1b1512b1dd2ae27bd575a72b4748f6c85b00006dd5a08be0b8eacd6cbd8d90124b3874c7a","up":true,"reverse":true,"distance":68},{"one":"fdeaa376c517fe1b4574eb859d92df4272d344409ae9ec42f1c3f21e386c88b8dddd5d2febb9851e450c7e2a72113d419a6e52bf63e6bfbdde00259062ed3098","other":"8d84d276b03161f58af3911b10da2dc33cfc93391130ed89db0f7a84bd6323b0237dab2f3c8b5c7d9953cdc745125dd67ee1122cb73a583caa98359f01afb415","up":true,"reverse":false,"distance":65},{"one":"a26035f22d085ad5c8f3892f16ed4867b6a09952c589d0177af8372666e44a454a3e49b43456c25a8365015f450432f4a6b328810e740dd2eadb898857b83e7b","other":"fdeaa376c517fe1b4574eb859d92df4272d344409ae9ec42f1c3f21e386c88b8dddd5d2febb9851e450c7e2a72113d419a6e52bf63e6bfbdde00259062ed3098","up":true,"reverse":true,"distance":69}]} \ No newline at end of file diff --git a/swarm/pss/testdata/snapshot_50.json b/swarm/pss/testdata/snapshot_50.json new file mode 100755 index 0000000000..269f2952bf --- /dev/null +++ b/swarm/pss/testdata/snapshot_50.json @@ -0,0 +1 @@ +{"nodes":[{"config":{"id":"6d65a381994e9c50911e14806c77d7a21aecc50c2c3199a9c70cffb76184810bcbe0231acea7cc671a321690bf8fad407e5457dcb3b7450031923f3244a85dd8","private_key":"c59b26bdf2d2549ef146a525fe6b08b0fb6d3198f83b762ed2c75b0b98dfb57b","name":"node01","services":["discovery"]},"up":true},{"config":{"id":"50599a16f2459fcc22b72d339fe8fe3dea7d171045e33d42da5eb1df4cd1473259e0b91b3b95abf50d39ec08927221d1836079c76fd357cba55aee6b718d3969","private_key":"cf2f3388fbff4dfe0a110844fad832fd199b49e81d01a67d8a38cebdf27e34e4","name":"node02","services":["discovery"]},"up":true},{"config":{"id":"5d9701b7bca361717d51cf7a006668bbd937d3169dd4a28b665db7e08ccf8b0b4742479ad58906e3a9b3be4688f188806b427c4256fa7d8d9054dfe200b44d89","private_key":"9d9605dc75308d3c0dd32e3797669323506303819eecc37d3d8674b3070f1343","name":"node03","services":["discovery"]},"up":true},{"config":{"id":"5ca10bb8eb1ca4b2624cd883589cda6b7bd05b3367f7505a628c7ead6445d7cde338532beabe67d39c272c6dca779651102c33fd7f6c10fb32d07bf4a3e6c476","private_key":"a3bfd4d50dfcc1d1b061a41b37ac8dc2291b8b639a45600ed7dcccf36b58416f","name":"node04","services":["discovery"]},"up":true},{"config":{"id":"1efb1ae1623b6b34acb0972b0f3424e744c3e71537ba16a4c965b85d665efcfe2ac427a36ba9bb05ca957b65b3f1ea025d97174307cc47e46727538c59ee4075","private_key":"01b1a0fb956ad3a8460290181f5f220b4a1ef44fbf0f47514e0271d0a1ac96f9","name":"node05","services":["discovery"]},"up":true},{"config":{"id":"129104b47421b79491405085da14399b27619b094bcb72428b1b885582e2b0f321eea43fc7715270011de064f29c2e8aebeca7a102828d8293c4c66106777cb9","private_key":"a5acf3ad59a5a281a51a1c8b96d567fe09a8ad6228eddfdf1ab3e587d0a236a4","name":"node06","services":["discovery"]},"up":true},{"config":{"id":"e30e5461d270be1cbb1a71d9170a1b4c02e460721bcecd7bebf2afa84afe1e35c5c91ae741453ee3788de44bcba8a9cb88bc958dbc0a9d7a8eecc2e4bcbf55b7","private_key":"8ed2c9ef6177acf5626e42b331944fc2b729345836b4fb7716ef2238e014bed7","name":"node07","services":["discovery"]},"up":true},{"config":{"id":"263bb9c8994dc8466e94f2a63034eeee3110d34aa112bbc1a03cf69d53526b056084baf90b37ad9ea53b0022857f466303e2b674eb81d4c60860e83208204953","private_key":"1c9f8eb9903768a7d442d7edac25280c24f4183253680bc23fa208b86c274140","name":"node08","services":["discovery"]},"up":true},{"config":{"id":"21f264c073513940306599d0424381e447eaae60e9e20ffdd3aa78aa0ca0fd3764d510ff7b3cdaab3567cf9f9ab348405f2691780324642896e51ae9f734025a","private_key":"dcdc028e5c7b72733592c6d4c23757819bd07b7a6c496b4299de04ac2c21607e","name":"node09","services":["discovery"]},"up":true},{"config":{"id":"d122e7ba13f4d4237f2ea82d57cf5657ae1adb4d7f37ccf5f76784071ea55ab1ba39b66e23d8d620ad794821a2a17e8017d18bd350af5e7cd8b629286e712061","private_key":"d4e91e7c5dda6efc3bac4a7f3442973a8c966bccd6e886ccb9439aef1fa89ae5","name":"node10","services":["discovery"]},"up":true},{"config":{"id":"9a9e30d7b81d88706fd08ffdef62f1de66ff72d0e6a95d4ffb704aeaab440d55fd2cae269f5074c8529197075590d83b622eab83f3181f44170f82fa68192d05","private_key":"c1431e21b8a0a3924d926da11c07ae0aa6110a88bf0ea318e9e9970202be1f0a","name":"node11","services":["discovery"]},"up":true},{"config":{"id":"6e8a066612c06ea3f45da90f94283efcffb932824fa1b3b5bc0fe4ef5a4254d1b8b3e171a7ab0babd22a438e0b2740ea21e132569526c8b6c66248569458c818","private_key":"de30e3d48cc8a5bdb9af62aec4e6c566c7997151e016376a4653421bd78c07b9","name":"node12","services":["discovery"]},"up":true},{"config":{"id":"3a7b261fe3a4eff5574eb65750496387fbea76911fea73c2c1a8f7b1cc9ffe012de59d2dddc023fa91d9f98c4f443f041319641e93ddae8e9e2ccf09aba47cab","private_key":"2bd746112ae8abe08f9ac74d2d8c117faa9746e6db9b472d802c67bdcf14998d","name":"node13","services":["discovery"]},"up":true},{"config":{"id":"f4a467e4438ee926c12c64d1cea08a2afdcf35d43ff79fc0350c1820aa00f9a63b7ae68c716b4d6c58bea8df0cd2b7308086b9ecf643f2d28de8ab2082e3d997","private_key":"734e9c80bd1f602a54819d72a71db9513ac0bf710b9737bbdffd8c0c528e2878","name":"node14","services":["discovery"]},"up":true},{"config":{"id":"8fbc6a4c0f3affceadb1caf4879495276e14f8b2da8a1ac0e6216af2f6aee24d2f6a3183730bdff414fd1ec0d3fd9ab75b71aa86cd4f26c574e452e6c2ffeb62","private_key":"9a2cf963ac8e78310a138fa9a4051bdecd45653453c77d452e09a466b0b0fc51","name":"node15","services":["discovery"]},"up":true},{"config":{"id":"7eb346b41bd347c85309113b5defb55754712d2f65f8504df55cd6593e406f0bc74d83e7116a2c24d1450f10e154afd8796a29e594e56b2c3baa04aa35701b3f","private_key":"c8c262fe8b8d1ad48d0c4f8b2c0d7b0ec1c450e20868c3c7c170fde9778b5106","name":"node16","services":["discovery"]},"up":true},{"config":{"id":"da1b8633a480c6000d6041b8a2cd3ca9ca5d22b166fabac008c841c7f166878ab84992750298bd524f8ff076d205106fdf8203e27aac5c1d3d1dbc0a60fc1c55","private_key":"35ee91bd4526b1e351dc0ad3fe1c1d69b7d9c1baf6417b1aba424f71884513db","name":"node17","services":["discovery"]},"up":true},{"config":{"id":"f2fd5041b4a368e02897eeff5fa5cab99036f4242a69712fcd85f72995318b8ec53b1430eb1947386ab627c32e4d616ede61045bed4c83ae8b4cfc30e36542bf","private_key":"50a50cd1db37a123f1b7f27786dbd76a6ecb0c29c9f36bbdf7f7ccdd9de9696c","name":"node18","services":["discovery"]},"up":true},{"config":{"id":"8ddf978d1b720e176707ceedfa1084ddef995da315efdb85b740b3f28f15e4345f4b9426245d77214549cc9acfab2753e54c3a3370b705c8105d8fcc1a493e8f","private_key":"ef2e6ea6c75e4d5827538121280f9e10365471d5c9f6c537b73b726ceb15d3e5","name":"node19","services":["discovery"]},"up":true},{"config":{"id":"74a411f8223da3a3af3bf7f46b7864808e0baa99959319a00c4cfa1df865857950b991d71bf426fd7327e67686aab5925d1b01cc937baabde4a7f2a93ed2b012","private_key":"0282d75fdcae218180e5b552586c3e2febd133f0381bb142714a9cee571aa4f7","name":"node20","services":["discovery"]},"up":true},{"config":{"id":"4e4239a7eefd7332409038a04c2cb2bcb50261b5a07fce3ab1205e08015b0249a799ba663cebc81f5b93ff8040c3eb88a19796fa249944a2a85db03ca0a47c07","private_key":"4e758ff820885b504fc65f430d108a61edcef9cef077f9c7b2d495ed06b75d8a","name":"node21","services":["discovery"]},"up":true},{"config":{"id":"87481536fed1ebcea41a642cd3d9644894fdf47cee84c04664a188884fc7a61bf175a8a803acd79657a00ee1f386110f81dc893259ed62a23e542ebfda4ae8a4","private_key":"f51588c3b24f167ad1dea7a216798419dc1e87abc98cf1394c17a7e9b7ed5bf5","name":"node22","services":["discovery"]},"up":true},{"config":{"id":"e12bc82fc0b4493cd8cd38c3ca9529c1fda391f5927799506fe83504ba04d0d455befa12bf3c611edc8173e3092230fde011f720644bca9a393b3e46b7b478fa","private_key":"20aa7feff8d74a5a9c593466e3a583028525b71e6c102f480bedd8e4e2dd6bb3","name":"node23","services":["discovery"]},"up":true},{"config":{"id":"c644219e651396eb2a27d87d09a78127654e75a8169ccb68f652a2dc692f5b90c97abbf7cedaf49644073196e31cf28cb070fd013c8d7e38fc4ca269b34e7d53","private_key":"b2c697d1409f1d517321cc8e61a2bec294062ccb33aacc13eb9359336308711f","name":"node24","services":["discovery"]},"up":true},{"config":{"id":"08865c0205512d9a86e80c4a6861bd6eee1d55ac97fd485c6129e0427dc3ac8dd07dcb149d6fe071f4c81ad77c000f520cde076109f40e4a22c6b96ade6367f1","private_key":"1487c00433a2745fdca2bc8b32f1e96e0c7db5dccdd75cfaeb426d760886291d","name":"node25","services":["discovery"]},"up":true},{"config":{"id":"ad9c1beb27e76de1c3102ae1a977672f46103908f0a1d1ade9d31126737aac0b3c38488c95c81a7ca7b69c8029c20535f573fedd3d5da4feb2eb3f9b2187d884","private_key":"1f2cb6dccf75ebd1bc9e6b3c90c5ca16850b70f46717f1877d93f43545959fa3","name":"node26","services":["discovery"]},"up":true},{"config":{"id":"ba0618974968465e0c3da7a67fb32088b0c85088830cd1e7d6e5cc69c90df9ca426490eb6167e67e0ad3d4eb22edf817429d13e3fbb8702f2fc3e7f54078ffd4","private_key":"5f816afc1b0d33dc426ddf7275e04a31fb1edb9ae136d8e1ea7278ca536e5d44","name":"node27","services":["discovery"]},"up":true},{"config":{"id":"f2590e4fb9453af327b4726295c6076e91bb618504b2951e6c219b6cd29145f9c2d12252d6468d074c96386e47346ccc75cd98c490ba4589ee245474b255aba6","private_key":"1027d7205516393fae64a16f686e4eed95e1a2cac8a1eb855e9c39b5ce6cc5af","name":"node28","services":["discovery"]},"up":true},{"config":{"id":"39def8b81d64f73e90e00ce235372a8af664c44d2d850a1eb206bfa16a1de4d505100925b914e1597eb33d3592ee5a1e661b2f2e1a7da71985ebaca48743995c","private_key":"b281e4be8d7085c9700c1d0475d078f9da8ee68cec20fa9c14c171f081cd6a54","name":"node29","services":["discovery"]},"up":true},{"config":{"id":"ef276e88157832f458ed79d5660493a1ab6387fcb0977eb9e99a6880d1c7b6cb7b64cb87f383b37dd3edfdc27fdfc93cb473c0998320112c26d950f727907613","private_key":"9432d8eb12536aee1f895b96381b87dda63cf7e3e82702875633aa227f9dca7b","name":"node30","services":["discovery"]},"up":true},{"config":{"id":"55a0c890b8b62a00069f9fd373af5fbe089d9ab6a2b81535b0b2a1880baa38e94462ae83befda6ebf904aea53f6b646a14d3ce72eedc935012c37433f2d551cc","private_key":"9f4d102902de0850d7cb2894fa127afd0e0aba9409b9c1f1c481cf3f023fc1c2","name":"node31","services":["discovery"]},"up":true},{"config":{"id":"930d87d768d5436f7a6c4cc561962856b7117334bdc6d03cb427c4750b1b6f66e50ab45483681516f54581a278ee64243ad9373f8e4c2c29deb5a6f792aa83e1","private_key":"eb7f2c0c970baa75a6d4f044036f020c66c7d666b0d44cdf00480aace49d1587","name":"node32","services":["discovery"]},"up":true},{"config":{"id":"6f1b9f4ef817edec35a9c6bfb762fb5fac66257830cf19977a80b23dee13ab2f196ffdaf7d62092f9cc8988c38b45af7348ff9ca0130b9337bd76e40846a9181","private_key":"7c03e86a80473d54030d20355304c2ffe2d5aa0f206c149e36666dd21944b5f6","name":"node33","services":["discovery"]},"up":true},{"config":{"id":"d274bf8cefa5713fdd306478c05e5bcdbeb303a926f1aec9bfa19fd2bbcb6f47b74ed150dffafdeae77785f0d03892d7a0dc7ebb5232425770077d721977f142","private_key":"de7db60511126dfce0315d9a550fb54593e33e06a7a8be60cf2029981a3a2e5e","name":"node34","services":["discovery"]},"up":true},{"config":{"id":"8dc5fc624500d8ec3b9cd1a9a1195dcbae1ba3a2c094d664277684fc7a5244fba0038382c9ea6f9c4512f725fe4bbb954e0d47e8e85eb30b4af502d074bc9e89","private_key":"53f8f25a614c7c3ff68be8a72e5283b05efee005dab6435d1b7fbbf0047dc2ad","name":"node35","services":["discovery"]},"up":true},{"config":{"id":"e31e3ad36eb9b19ec153ebb640b2db8c7125e493e77023f122c98f7870ee1371c298137ba642cf7198553df11d9a36f993d2689691446633e76e56f34830e224","private_key":"832ac19752cfdac0e80a00217374e2d16cd80e315ed3f9e33ae74aec874cdd94","name":"node36","services":["discovery"]},"up":true},{"config":{"id":"2a66ca910cb35fb890a4da20e633da373d86d6cdf0f11410d95a80c35d1ec428c38048eb8b05546e8a42eab530e02af53dafd236ce08e0d6d3c480b7fcfa9de4","private_key":"d74e99bf9fb2f905da35732dbf7146ff6d7be5a8cb2616ecf2bc02acd930ac16","name":"node37","services":["discovery"]},"up":true},{"config":{"id":"5f76667754fc2025d4c48a6dba3915c40b8fc3071b747d44447bd7dc6e60408ea58a38a1013a386a8fe38afe865f0518d54fe3a2e56b9df7f2fb8a9917ec6bda","private_key":"b04f2dceba8774cefb9b4452ed02606b77746a5cec45165e1f9f2443ec3ffce3","name":"node38","services":["discovery"]},"up":true},{"config":{"id":"299066c8215f1db1a7519de2b5be55b00a290ef7c4f740c6c7e624f348e8348fb102ca2120c9905f63e19c357eaf59c968edba2e1f173c8913ee437ce0759cd3","private_key":"f53e3eb735daf7635802e3e032b15fdaf94410e6daaae89e72af1a632e49531a","name":"node39","services":["discovery"]},"up":true},{"config":{"id":"148839415bf1ad2587f9665cb30be5308c1f41ec2b92d62bc57dbbb5f3d38f5fc7f54111e1ad7d78631ec66b9ca29472256664960d03134fd9b39e02bec98383","private_key":"b6efac9740ad0657054c8fd489744726822f41af08f5bf3a140c0936467187e5","name":"node40","services":["discovery"]},"up":true},{"config":{"id":"bfad49ad4d9b3bf364001de5adfe6d06c774e2c6762b7e029add90c4be84078016ecce689481ed6da4a21eb703b09c7562eb3cae99d9f9aa8a20456ac3aaea0a","private_key":"5450221f2b8c346b6011889760199c35649c3ecd78d290877fb3e41760f386b8","name":"node41","services":["discovery"]},"up":true},{"config":{"id":"c66f1993d60b10eff83712b9b9a9ea25a7e8564edf2295ab7e347f11f31023c3061037fa81c75ce209005dc3cc0ddf9ba08f3d80d952b56701ca6448c644d5ee","private_key":"285531392e9893b739f4b3fe550a63d590261723381f99af4babb3f1ebfa3b12","name":"node42","services":["discovery"]},"up":true},{"config":{"id":"73b5c8bbce025f0a14b7c6f3a5dd86977b4e8d3814fb7a7ae4928cc48e62af434def88f493823264c24e8615402c69e1ea29f1baaa67f799171b12ee0c2a54ca","private_key":"97c535cff467a4dd7dd033ce11d5841e095b73849562ec7db1b4a90243cf6858","name":"node43","services":["discovery"]},"up":true},{"config":{"id":"45e70f42d1b82a6d198cb0134c283ed04063fc397e04e971688140e9a49e09a4fd4da8f327b1916c3d6f7e6e267f0ed44bacb1e1c6214579d37798934d384d52","private_key":"537ed5998f3d9c060ae01a5556a33e2a0db1f022a633899e956611870ad0b80a","name":"node44","services":["discovery"]},"up":true},{"config":{"id":"468cd912d48cb0394a4725ff71066eba0573018deddef2915931f20f42700a564ef54b4bd8083fc87c3fffd20430686adbf930089e4dcf1024be772dc280e6d5","private_key":"7552f023eb0235ea17d0887ee60040bb3f0611cdb166d449fbce58a78b4c3644","name":"node45","services":["discovery"]},"up":true},{"config":{"id":"68ca69880a3f905069387ca0e2f8d16be7dc47f4ecc57659bb87ab8c1849586ae643e94480f46318443a30f29cfbf3bbfe8883c99d87b8177a4bc8dcfb359808","private_key":"e7a7519a2213e1976f3d7cc32c8a6a407a2b16c15b9ec3190cfb4035da5ffd44","name":"node46","services":["discovery"]},"up":true},{"config":{"id":"13be094b2682fc6b97f624d39fd05778a7acac30ee8005e24f4b9a8fcb33cf00f329ec5f9f0cac5c5d45c825d3641e787856fd658e7e65bb858238b241be60ac","private_key":"1f953de115e315162c3dedd4962ced98ed1a24e5831c5a35b3498c9660ee25ab","name":"node47","services":["discovery"]},"up":true},{"config":{"id":"bb6c676b089092de9bac981004844c77658436703931e9ffb2d56c965ae9402f5d077be8c9278ecc8174352f615d23289e4dcd4bbd8c71addde4e5e47febd69a","private_key":"77ab76fce304ef9d3900f4b059400102385d0b4dbc68210f14f5d0b7544d8eb8","name":"node48","services":["discovery"]},"up":true},{"config":{"id":"339c4b9c35e4d99ff71e338465d19d6d8df6e7459a3e58242dbd0c04a198deba3caedd076b9c340d4a5d611e074cb41fb4b896a4d2eb6f02414e75034fd5f493","private_key":"6d3d787b038d8f18e3b656401962b9b509ff68f53cecc584250dda7a685cd6f4","name":"node49","services":["discovery"]},"up":true},{"config":{"id":"c87ac1c26d351001b25fcd6715238746e66dcd3d6cc7e8bd8e212a9c5987016d358727f445665a6e51dd3f454f97240440c5392f149f94385ed98f9cdc987870","private_key":"3a1a788c32b60fb22055167c28094156c1d90bca7f274b6394521921fd2f6ef1","name":"node50","services":["discovery"]},"up":true}],"conns":[{"one":"6d65a381994e9c50911e14806c77d7a21aecc50c2c3199a9c70cffb76184810bcbe0231acea7cc671a321690bf8fad407e5457dcb3b7450031923f3244a85dd8","other":"c87ac1c26d351001b25fcd6715238746e66dcd3d6cc7e8bd8e212a9c5987016d358727f445665a6e51dd3f454f97240440c5392f149f94385ed98f9cdc987870","up":true,"reverse":true,"distance":65},{"one":"50599a16f2459fcc22b72d339fe8fe3dea7d171045e33d42da5eb1df4cd1473259e0b91b3b95abf50d39ec08927221d1836079c76fd357cba55aee6b718d3969","other":"6d65a381994e9c50911e14806c77d7a21aecc50c2c3199a9c70cffb76184810bcbe0231acea7cc671a321690bf8fad407e5457dcb3b7450031923f3244a85dd8","up":true,"reverse":false,"distance":70},{"one":"5d9701b7bca361717d51cf7a006668bbd937d3169dd4a28b665db7e08ccf8b0b4742479ad58906e3a9b3be4688f188806b427c4256fa7d8d9054dfe200b44d89","other":"50599a16f2459fcc22b72d339fe8fe3dea7d171045e33d42da5eb1df4cd1473259e0b91b3b95abf50d39ec08927221d1836079c76fd357cba55aee6b718d3969","up":true,"reverse":false,"distance":73},{"one":"5ca10bb8eb1ca4b2624cd883589cda6b7bd05b3367f7505a628c7ead6445d7cde338532beabe67d39c272c6dca779651102c33fd7f6c10fb32d07bf4a3e6c476","other":"5d9701b7bca361717d51cf7a006668bbd937d3169dd4a28b665db7e08ccf8b0b4742479ad58906e3a9b3be4688f188806b427c4256fa7d8d9054dfe200b44d89","up":true,"reverse":false,"distance":77},{"one":"1efb1ae1623b6b34acb0972b0f3424e744c3e71537ba16a4c965b85d665efcfe2ac427a36ba9bb05ca957b65b3f1ea025d97174307cc47e46727538c59ee4075","other":"5ca10bb8eb1ca4b2624cd883589cda6b7bd05b3367f7505a628c7ead6445d7cde338532beabe67d39c272c6dca779651102c33fd7f6c10fb32d07bf4a3e6c476","up":true,"reverse":true,"distance":69},{"one":"129104b47421b79491405085da14399b27619b094bcb72428b1b885582e2b0f321eea43fc7715270011de064f29c2e8aebeca7a102828d8293c4c66106777cb9","other":"1efb1ae1623b6b34acb0972b0f3424e744c3e71537ba16a4c965b85d665efcfe2ac427a36ba9bb05ca957b65b3f1ea025d97174307cc47e46727538c59ee4075","up":true,"reverse":false,"distance":73},{"one":"e30e5461d270be1cbb1a71d9170a1b4c02e460721bcecd7bebf2afa84afe1e35c5c91ae741453ee3788de44bcba8a9cb88bc958dbc0a9d7a8eecc2e4bcbf55b7","other":"129104b47421b79491405085da14399b27619b094bcb72428b1b885582e2b0f321eea43fc7715270011de064f29c2e8aebeca7a102828d8293c4c66106777cb9","up":true,"reverse":false,"distance":65},{"one":"263bb9c8994dc8466e94f2a63034eeee3110d34aa112bbc1a03cf69d53526b056084baf90b37ad9ea53b0022857f466303e2b674eb81d4c60860e83208204953","other":"e30e5461d270be1cbb1a71d9170a1b4c02e460721bcecd7bebf2afa84afe1e35c5c91ae741453ee3788de44bcba8a9cb88bc958dbc0a9d7a8eecc2e4bcbf55b7","up":true,"reverse":true,"distance":65},{"one":"21f264c073513940306599d0424381e447eaae60e9e20ffdd3aa78aa0ca0fd3764d510ff7b3cdaab3567cf9f9ab348405f2691780324642896e51ae9f734025a","other":"263bb9c8994dc8466e94f2a63034eeee3110d34aa112bbc1a03cf69d53526b056084baf90b37ad9ea53b0022857f466303e2b674eb81d4c60860e83208204953","up":true,"reverse":false,"distance":77},{"one":"d122e7ba13f4d4237f2ea82d57cf5657ae1adb4d7f37ccf5f76784071ea55ab1ba39b66e23d8d620ad794821a2a17e8017d18bd350af5e7cd8b629286e712061","other":"21f264c073513940306599d0424381e447eaae60e9e20ffdd3aa78aa0ca0fd3764d510ff7b3cdaab3567cf9f9ab348405f2691780324642896e51ae9f734025a","up":true,"reverse":true,"distance":65},{"one":"9a9e30d7b81d88706fd08ffdef62f1de66ff72d0e6a95d4ffb704aeaab440d55fd2cae269f5074c8529197075590d83b622eab83f3181f44170f82fa68192d05","other":"d122e7ba13f4d4237f2ea82d57cf5657ae1adb4d7f37ccf5f76784071ea55ab1ba39b66e23d8d620ad794821a2a17e8017d18bd350af5e7cd8b629286e712061","up":true,"reverse":false,"distance":65},{"one":"6e8a066612c06ea3f45da90f94283efcffb932824fa1b3b5bc0fe4ef5a4254d1b8b3e171a7ab0babd22a438e0b2740ea21e132569526c8b6c66248569458c818","other":"9a9e30d7b81d88706fd08ffdef62f1de66ff72d0e6a95d4ffb704aeaab440d55fd2cae269f5074c8529197075590d83b622eab83f3181f44170f82fa68192d05","up":true,"reverse":true,"distance":68},{"one":"3a7b261fe3a4eff5574eb65750496387fbea76911fea73c2c1a8f7b1cc9ffe012de59d2dddc023fa91d9f98c4f443f041319641e93ddae8e9e2ccf09aba47cab","other":"6e8a066612c06ea3f45da90f94283efcffb932824fa1b3b5bc0fe4ef5a4254d1b8b3e171a7ab0babd22a438e0b2740ea21e132569526c8b6c66248569458c818","up":true,"reverse":true,"distance":69},{"one":"f4a467e4438ee926c12c64d1cea08a2afdcf35d43ff79fc0350c1820aa00f9a63b7ae68c716b4d6c58bea8df0cd2b7308086b9ecf643f2d28de8ab2082e3d997","other":"3a7b261fe3a4eff5574eb65750496387fbea76911fea73c2c1a8f7b1cc9ffe012de59d2dddc023fa91d9f98c4f443f041319641e93ddae8e9e2ccf09aba47cab","up":true,"reverse":false,"distance":65},{"one":"8fbc6a4c0f3affceadb1caf4879495276e14f8b2da8a1ac0e6216af2f6aee24d2f6a3183730bdff414fd1ec0d3fd9ab75b71aa86cd4f26c574e452e6c2ffeb62","other":"f4a467e4438ee926c12c64d1cea08a2afdcf35d43ff79fc0350c1820aa00f9a63b7ae68c716b4d6c58bea8df0cd2b7308086b9ecf643f2d28de8ab2082e3d997","up":true,"reverse":true,"distance":65},{"one":"7eb346b41bd347c85309113b5defb55754712d2f65f8504df55cd6593e406f0bc74d83e7116a2c24d1450f10e154afd8796a29e594e56b2c3baa04aa35701b3f","other":"8fbc6a4c0f3affceadb1caf4879495276e14f8b2da8a1ac0e6216af2f6aee24d2f6a3183730bdff414fd1ec0d3fd9ab75b71aa86cd4f26c574e452e6c2ffeb62","up":true,"reverse":false,"distance":68},{"one":"da1b8633a480c6000d6041b8a2cd3ca9ca5d22b166fabac008c841c7f166878ab84992750298bd524f8ff076d205106fdf8203e27aac5c1d3d1dbc0a60fc1c55","other":"7eb346b41bd347c85309113b5defb55754712d2f65f8504df55cd6593e406f0bc74d83e7116a2c24d1450f10e154afd8796a29e594e56b2c3baa04aa35701b3f","up":true,"reverse":false,"distance":65},{"one":"f2fd5041b4a368e02897eeff5fa5cab99036f4242a69712fcd85f72995318b8ec53b1430eb1947386ab627c32e4d616ede61045bed4c83ae8b4cfc30e36542bf","other":"da1b8633a480c6000d6041b8a2cd3ca9ca5d22b166fabac008c841c7f166878ab84992750298bd524f8ff076d205106fdf8203e27aac5c1d3d1dbc0a60fc1c55","up":true,"reverse":true,"distance":70},{"one":"8ddf978d1b720e176707ceedfa1084ddef995da315efdb85b740b3f28f15e4345f4b9426245d77214549cc9acfab2753e54c3a3370b705c8105d8fcc1a493e8f","other":"f2fd5041b4a368e02897eeff5fa5cab99036f4242a69712fcd85f72995318b8ec53b1430eb1947386ab627c32e4d616ede61045bed4c83ae8b4cfc30e36542bf","up":true,"reverse":false,"distance":65},{"one":"74a411f8223da3a3af3bf7f46b7864808e0baa99959319a00c4cfa1df865857950b991d71bf426fd7327e67686aab5925d1b01cc937baabde4a7f2a93ed2b012","other":"8ddf978d1b720e176707ceedfa1084ddef995da315efdb85b740b3f28f15e4345f4b9426245d77214549cc9acfab2753e54c3a3370b705c8105d8fcc1a493e8f","up":true,"reverse":false,"distance":68},{"one":"4e4239a7eefd7332409038a04c2cb2bcb50261b5a07fce3ab1205e08015b0249a799ba663cebc81f5b93ff8040c3eb88a19796fa249944a2a85db03ca0a47c07","other":"74a411f8223da3a3af3bf7f46b7864808e0baa99959319a00c4cfa1df865857950b991d71bf426fd7327e67686aab5925d1b01cc937baabde4a7f2a93ed2b012","up":true,"reverse":true,"distance":70},{"one":"87481536fed1ebcea41a642cd3d9644894fdf47cee84c04664a188884fc7a61bf175a8a803acd79657a00ee1f386110f81dc893259ed62a23e542ebfda4ae8a4","other":"4e4239a7eefd7332409038a04c2cb2bcb50261b5a07fce3ab1205e08015b0249a799ba663cebc81f5b93ff8040c3eb88a19796fa249944a2a85db03ca0a47c07","up":true,"reverse":false,"distance":68},{"one":"e12bc82fc0b4493cd8cd38c3ca9529c1fda391f5927799506fe83504ba04d0d455befa12bf3c611edc8173e3092230fde011f720644bca9a393b3e46b7b478fa","other":"87481536fed1ebcea41a642cd3d9644894fdf47cee84c04664a188884fc7a61bf175a8a803acd79657a00ee1f386110f81dc893259ed62a23e542ebfda4ae8a4","up":true,"reverse":true,"distance":65},{"one":"c644219e651396eb2a27d87d09a78127654e75a8169ccb68f652a2dc692f5b90c97abbf7cedaf49644073196e31cf28cb070fd013c8d7e38fc4ca269b34e7d53","other":"e12bc82fc0b4493cd8cd38c3ca9529c1fda391f5927799506fe83504ba04d0d455befa12bf3c611edc8173e3092230fde011f720644bca9a393b3e46b7b478fa","up":true,"reverse":false,"distance":69},{"one":"08865c0205512d9a86e80c4a6861bd6eee1d55ac97fd485c6129e0427dc3ac8dd07dcb149d6fe071f4c81ad77c000f520cde076109f40e4a22c6b96ade6367f1","other":"c644219e651396eb2a27d87d09a78127654e75a8169ccb68f652a2dc692f5b90c97abbf7cedaf49644073196e31cf28cb070fd013c8d7e38fc4ca269b34e7d53","up":true,"reverse":true,"distance":65},{"one":"ad9c1beb27e76de1c3102ae1a977672f46103908f0a1d1ade9d31126737aac0b3c38488c95c81a7ca7b69c8029c20535f573fedd3d5da4feb2eb3f9b2187d884","other":"08865c0205512d9a86e80c4a6861bd6eee1d55ac97fd485c6129e0427dc3ac8dd07dcb149d6fe071f4c81ad77c000f520cde076109f40e4a22c6b96ade6367f1","up":true,"reverse":false,"distance":65},{"one":"ba0618974968465e0c3da7a67fb32088b0c85088830cd1e7d6e5cc69c90df9ca426490eb6167e67e0ad3d4eb22edf817429d13e3fbb8702f2fc3e7f54078ffd4","other":"ad9c1beb27e76de1c3102ae1a977672f46103908f0a1d1ade9d31126737aac0b3c38488c95c81a7ca7b69c8029c20535f573fedd3d5da4feb2eb3f9b2187d884","up":true,"reverse":true,"distance":70},{"one":"f2590e4fb9453af327b4726295c6076e91bb618504b2951e6c219b6cd29145f9c2d12252d6468d074c96386e47346ccc75cd98c490ba4589ee245474b255aba6","other":"ba0618974968465e0c3da7a67fb32088b0c85088830cd1e7d6e5cc69c90df9ca426490eb6167e67e0ad3d4eb22edf817429d13e3fbb8702f2fc3e7f54078ffd4","up":true,"reverse":false,"distance":69},{"one":"39def8b81d64f73e90e00ce235372a8af664c44d2d850a1eb206bfa16a1de4d505100925b914e1597eb33d3592ee5a1e661b2f2e1a7da71985ebaca48743995c","other":"f2590e4fb9453af327b4726295c6076e91bb618504b2951e6c219b6cd29145f9c2d12252d6468d074c96386e47346ccc75cd98c490ba4589ee245474b255aba6","up":true,"reverse":true,"distance":65},{"one":"ef276e88157832f458ed79d5660493a1ab6387fcb0977eb9e99a6880d1c7b6cb7b64cb87f383b37dd3edfdc27fdfc93cb473c0998320112c26d950f727907613","other":"39def8b81d64f73e90e00ce235372a8af664c44d2d850a1eb206bfa16a1de4d505100925b914e1597eb33d3592ee5a1e661b2f2e1a7da71985ebaca48743995c","up":true,"reverse":false,"distance":65},{"one":"55a0c890b8b62a00069f9fd373af5fbe089d9ab6a2b81535b0b2a1880baa38e94462ae83befda6ebf904aea53f6b646a14d3ce72eedc935012c37433f2d551cc","other":"ef276e88157832f458ed79d5660493a1ab6387fcb0977eb9e99a6880d1c7b6cb7b64cb87f383b37dd3edfdc27fdfc93cb473c0998320112c26d950f727907613","up":true,"reverse":true,"distance":65},{"one":"930d87d768d5436f7a6c4cc561962856b7117334bdc6d03cb427c4750b1b6f66e50ab45483681516f54581a278ee64243ad9373f8e4c2c29deb5a6f792aa83e1","other":"55a0c890b8b62a00069f9fd373af5fbe089d9ab6a2b81535b0b2a1880baa38e94462ae83befda6ebf904aea53f6b646a14d3ce72eedc935012c37433f2d551cc","up":true,"reverse":false,"distance":68},{"one":"6f1b9f4ef817edec35a9c6bfb762fb5fac66257830cf19977a80b23dee13ab2f196ffdaf7d62092f9cc8988c38b45af7348ff9ca0130b9337bd76e40846a9181","other":"930d87d768d5436f7a6c4cc561962856b7117334bdc6d03cb427c4750b1b6f66e50ab45483681516f54581a278ee64243ad9373f8e4c2c29deb5a6f792aa83e1","up":true,"reverse":false,"distance":68},{"one":"d274bf8cefa5713fdd306478c05e5bcdbeb303a926f1aec9bfa19fd2bbcb6f47b74ed150dffafdeae77785f0d03892d7a0dc7ebb5232425770077d721977f142","other":"6f1b9f4ef817edec35a9c6bfb762fb5fac66257830cf19977a80b23dee13ab2f196ffdaf7d62092f9cc8988c38b45af7348ff9ca0130b9337bd76e40846a9181","up":true,"reverse":false,"distance":65},{"one":"8dc5fc624500d8ec3b9cd1a9a1195dcbae1ba3a2c094d664277684fc7a5244fba0038382c9ea6f9c4512f725fe4bbb954e0d47e8e85eb30b4af502d074bc9e89","other":"d274bf8cefa5713fdd306478c05e5bcdbeb303a926f1aec9bfa19fd2bbcb6f47b74ed150dffafdeae77785f0d03892d7a0dc7ebb5232425770077d721977f142","up":true,"reverse":true,"distance":65},{"one":"e31e3ad36eb9b19ec153ebb640b2db8c7125e493e77023f122c98f7870ee1371c298137ba642cf7198553df11d9a36f993d2689691446633e76e56f34830e224","other":"8dc5fc624500d8ec3b9cd1a9a1195dcbae1ba3a2c094d664277684fc7a5244fba0038382c9ea6f9c4512f725fe4bbb954e0d47e8e85eb30b4af502d074bc9e89","up":true,"reverse":true,"distance":65},{"one":"2a66ca910cb35fb890a4da20e633da373d86d6cdf0f11410d95a80c35d1ec428c38048eb8b05546e8a42eab530e02af53dafd236ce08e0d6d3c480b7fcfa9de4","other":"e31e3ad36eb9b19ec153ebb640b2db8c7125e493e77023f122c98f7870ee1371c298137ba642cf7198553df11d9a36f993d2689691446633e76e56f34830e224","up":true,"reverse":false,"distance":65},{"one":"5f76667754fc2025d4c48a6dba3915c40b8fc3071b747d44447bd7dc6e60408ea58a38a1013a386a8fe38afe865f0518d54fe3a2e56b9df7f2fb8a9917ec6bda","other":"2a66ca910cb35fb890a4da20e633da373d86d6cdf0f11410d95a80c35d1ec428c38048eb8b05546e8a42eab530e02af53dafd236ce08e0d6d3c480b7fcfa9de4","up":true,"reverse":false,"distance":69},{"one":"299066c8215f1db1a7519de2b5be55b00a290ef7c4f740c6c7e624f348e8348fb102ca2120c9905f63e19c357eaf59c968edba2e1f173c8913ee437ce0759cd3","other":"5f76667754fc2025d4c48a6dba3915c40b8fc3071b747d44447bd7dc6e60408ea58a38a1013a386a8fe38afe865f0518d54fe3a2e56b9df7f2fb8a9917ec6bda","up":true,"reverse":true,"distance":69},{"one":"148839415bf1ad2587f9665cb30be5308c1f41ec2b92d62bc57dbbb5f3d38f5fc7f54111e1ad7d78631ec66b9ca29472256664960d03134fd9b39e02bec98383","other":"299066c8215f1db1a7519de2b5be55b00a290ef7c4f740c6c7e624f348e8348fb102ca2120c9905f63e19c357eaf59c968edba2e1f173c8913ee437ce0759cd3","up":true,"reverse":true,"distance":70},{"one":"bfad49ad4d9b3bf364001de5adfe6d06c774e2c6762b7e029add90c4be84078016ecce689481ed6da4a21eb703b09c7562eb3cae99d9f9aa8a20456ac3aaea0a","other":"148839415bf1ad2587f9665cb30be5308c1f41ec2b92d62bc57dbbb5f3d38f5fc7f54111e1ad7d78631ec66b9ca29472256664960d03134fd9b39e02bec98383","up":true,"reverse":false,"distance":65},{"one":"c66f1993d60b10eff83712b9b9a9ea25a7e8564edf2295ab7e347f11f31023c3061037fa81c75ce209005dc3cc0ddf9ba08f3d80d952b56701ca6448c644d5ee","other":"bfad49ad4d9b3bf364001de5adfe6d06c774e2c6762b7e029add90c4be84078016ecce689481ed6da4a21eb703b09c7562eb3cae99d9f9aa8a20456ac3aaea0a","up":true,"reverse":true,"distance":71},{"one":"73b5c8bbce025f0a14b7c6f3a5dd86977b4e8d3814fb7a7ae4928cc48e62af434def88f493823264c24e8615402c69e1ea29f1baaa67f799171b12ee0c2a54ca","other":"c66f1993d60b10eff83712b9b9a9ea25a7e8564edf2295ab7e347f11f31023c3061037fa81c75ce209005dc3cc0ddf9ba08f3d80d952b56701ca6448c644d5ee","up":true,"reverse":false,"distance":65},{"one":"45e70f42d1b82a6d198cb0134c283ed04063fc397e04e971688140e9a49e09a4fd4da8f327b1916c3d6f7e6e267f0ed44bacb1e1c6214579d37798934d384d52","other":"73b5c8bbce025f0a14b7c6f3a5dd86977b4e8d3814fb7a7ae4928cc48e62af434def88f493823264c24e8615402c69e1ea29f1baaa67f799171b12ee0c2a54ca","up":true,"reverse":false,"distance":70},{"one":"468cd912d48cb0394a4725ff71066eba0573018deddef2915931f20f42700a564ef54b4bd8083fc87c3fffd20430686adbf930089e4dcf1024be772dc280e6d5","other":"45e70f42d1b82a6d198cb0134c283ed04063fc397e04e971688140e9a49e09a4fd4da8f327b1916c3d6f7e6e267f0ed44bacb1e1c6214579d37798934d384d52","up":true,"reverse":false,"distance":78},{"one":"68ca69880a3f905069387ca0e2f8d16be7dc47f4ecc57659bb87ab8c1849586ae643e94480f46318443a30f29cfbf3bbfe8883c99d87b8177a4bc8dcfb359808","other":"468cd912d48cb0394a4725ff71066eba0573018deddef2915931f20f42700a564ef54b4bd8083fc87c3fffd20430686adbf930089e4dcf1024be772dc280e6d5","up":true,"reverse":true,"distance":70},{"one":"13be094b2682fc6b97f624d39fd05778a7acac30ee8005e24f4b9a8fcb33cf00f329ec5f9f0cac5c5d45c825d3641e787856fd658e7e65bb858238b241be60ac","other":"68ca69880a3f905069387ca0e2f8d16be7dc47f4ecc57659bb87ab8c1849586ae643e94480f46318443a30f29cfbf3bbfe8883c99d87b8177a4bc8dcfb359808","up":true,"reverse":false,"distance":69},{"one":"bb6c676b089092de9bac981004844c77658436703931e9ffb2d56c965ae9402f5d077be8c9278ecc8174352f615d23289e4dcd4bbd8c71addde4e5e47febd69a","other":"13be094b2682fc6b97f624d39fd05778a7acac30ee8005e24f4b9a8fcb33cf00f329ec5f9f0cac5c5d45c825d3641e787856fd658e7e65bb858238b241be60ac","up":true,"reverse":true,"distance":65},{"one":"339c4b9c35e4d99ff71e338465d19d6d8df6e7459a3e58242dbd0c04a198deba3caedd076b9c340d4a5d611e074cb41fb4b896a4d2eb6f02414e75034fd5f493","other":"bb6c676b089092de9bac981004844c77658436703931e9ffb2d56c965ae9402f5d077be8c9278ecc8174352f615d23289e4dcd4bbd8c71addde4e5e47febd69a","up":true,"reverse":true,"distance":65},{"one":"c87ac1c26d351001b25fcd6715238746e66dcd3d6cc7e8bd8e212a9c5987016d358727f445665a6e51dd3f454f97240440c5392f149f94385ed98f9cdc987870","other":"339c4b9c35e4d99ff71e338465d19d6d8df6e7459a3e58242dbd0c04a198deba3caedd076b9c340d4a5d611e074cb41fb4b896a4d2eb6f02414e75034fd5f493","up":true,"reverse":false,"distance":65}]} \ No newline at end of file diff --git a/swarm/pss/types.go b/swarm/pss/types.go index 9a76285778..32075fa224 100644 --- a/swarm/pss/types.go +++ b/swarm/pss/types.go @@ -49,7 +49,6 @@ func (self *PssMsg) String() string { return fmt.Sprintf("PssMsg: Recipient: %x", common.ByteLabel(self.To)) } - // Pre-Whisper placeholder, payload of PssMsg, sender address, Topic type Envelope struct { Topic Topic @@ -100,6 +99,11 @@ type APIMsg struct { Addr []byte } +// for debugging, show nice hex version +func (self *APIMsg) String() string { + return fmt.Sprintf("APIMsg: from: %s..., msg: %s...", common.ByteLabel(self.Msg), common.ByteLabel(self.Addr)) +} + // Signature for a message handler function for a PssMsg // // Implementations of this type are passed to Pss.Register together with a topic, diff --git a/swarm/swarm.go b/swarm/swarm.go index d2b764e600..8dfe124e15 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/contracts/chequebook" "github.com/ethereum/go-ethereum/contracts/ens" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -41,6 +42,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/pss" "github.com/ethereum/go-ethereum/swarm/storage" + //psschat "github.com/ethereum/go-ethereum/swarm/pss/protocols/chat" ) // the swarm stack @@ -110,9 +112,12 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api. ) config.HiveParams.Discovery = true + + nodeid := discover.PubkeyID(crypto.ToECDSAPub(common.FromHex(config.PublicKey))) + addr := network.NewAddrFromNodeID(nodeid) bzzconfig := &network.BzzConfig{ - UnderlayAddr: common.FromHex(config.PublicKey), OverlayAddr: common.FromHex(config.BzzKey), + UnderlayAddr: addr.UAddr, HiveParams: config.HiveParams, } self.bzz = network.NewBzz(bzzconfig, to, nil) @@ -189,6 +194,11 @@ Start is called when the stack is started */ // implements the node.Service interface func (self *Swarm) Start(net *p2p.Server) error { + + // update uaddr to correct enode + newaddr := self.bzz.UpdateLocalAddr([]byte(net.Self().String())) + log.Warn("Updated bzz local addr", "oaddr", fmt.Sprintf("%x", newaddr.OAddr), "uaddr", fmt.Sprintf("%x", newaddr.UAddr)) + // set chequebook if self.swapEnabled { ctx := context.Background() // The initial setup has no deadline. @@ -213,6 +223,7 @@ func (self *Swarm) Start(net *p2p.Server) error { if self.pss != nil { self.pss.Start(net) + log.Info("Pss started") } self.dpa.Start() @@ -242,9 +253,9 @@ func (self *Swarm) Stop() error { self.dpa.Stop() //self.hive.Stop() self.bzz.Stop() - if self.pss != nil { - self.pss.Stop() - } + //if self.pss != nil { + // self.pss.Stop() + //} if ch := self.config.Swap.Chequebook(); ch != nil { ch.Stop() ch.Save() @@ -265,6 +276,7 @@ func (self *Swarm) Protocols() (protos []p2p.Protocol) { } if self.pss != nil { + log.Warn("adding pss protos") for _, p := range self.pss.Protocols() { protos = append(protos, p) } diff --git a/swarm/swarm_psschat.go b/swarm/swarm_psschat.go index 8f262abdbe..8b3ba10b84 100644 --- a/swarm/swarm_psschat.go +++ b/swarm/swarm_psschat.go @@ -229,24 +229,6 @@ func (self *Swarm) Start(net *p2p.Server) error { return nil }) log.Info("Pss started ... with 'pssChat' :)") -// pss.RegisterPssProtocol(self.pss, &psschat.ChatTopic, psschat.ChatProtocol, psschat.New(nil, nil, nil, func(ctrl *psschat.ChatCtrl) { -// if ctrl.OutC != nil { -// go func() { -// for { -// select { -// case msg := <-ctrl.OutC: -// err := ctrl.Peer.Send(msg) -// if err != nil { -// // do something with ctrl.ConnC here -// //pp.Drop(err) -// //return -// } -// } -// } -// }() -// } -// -// })) } self.dpa.Start() From 10adb6e9172890edafbd1a01f5117f65b170f8cb Mon Sep 17 00:00:00 2001 From: lash Date: Tue, 20 Jun 2017 21:30:46 +0200 Subject: [PATCH 12/17] swarm, cmd/swarm: Removed websocket opinionation in client constructor PR cleanup --- cmd/swarm/main.go | 19 +- swarm/network/protocol.go | 19 - .../simulations/discovery/discovery_test.go | 8 +- swarm/pss/client/client.go | 226 +++------- swarm/pss/client/client_test.go | 26 +- swarm/pss/client/doc.go | 80 ++++ swarm/swarm.go | 40 +- swarm/swarm_psschat.go | 415 ------------------ 8 files changed, 152 insertions(+), 681 deletions(-) create mode 100644 swarm/pss/client/doc.go delete mode 100644 swarm/swarm_psschat.go diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index fedb413d60..64b26767a9 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -118,18 +118,9 @@ var ( Usage: "force mime type", } PssEnabledFlag = cli.BoolFlag{ - Name: "pss", + Name: "pss", Usage: "Enable pss (message passing over swarm)", } - PssHostFlag = cli.StringFlag{ - Name: "psshost", - Usage: fmt.Sprintf("Websockets host or ip for pss (default '%v')", node.DefaultWSHost), - Value: node.DefaultWSHost, - } - PssPortFlag = cli.IntFlag{ - Name: "pssport", - Usage: fmt.Sprintf("Websockets port for pss (default %d)", node.DefaultWSPort), - } CorsStringFlag = cli.StringFlag{ Name: "corsdomain", Usage: "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')", @@ -312,14 +303,6 @@ func version(ctx *cli.Context) error { func bzzd(ctx *cli.Context) error { cfg := defaultNodeConfig - if ctx.GlobalIsSet(PssEnabledFlag.Name) { - cfg.WSHost = ctx.GlobalString(PssHostFlag.Name) - cfg.WSModules = []string{"eth","pss"} - cfg.WSOrigins = []string{"*"} - if ctx.GlobalIsSet(PssPortFlag.Name) { - cfg.WSPort = ctx.GlobalInt(PssPortFlag.Name) - } - } utils.SetNodeConfig(ctx, &cfg) stack, err := node.New(&cfg) if err != nil { diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index 3d29b9b9ea..bbc65ad765 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -292,25 +292,6 @@ func (self *bzzHandshake) Perform(p *p2p.Peer, rw p2p.MsgReadWriter) (err error) return fmt.Errorf("version mismatch %d (!= %d)", rhs.Version, self.Version) } self.peerAddr = rhs.Addr - //log.Debug("L&R handshake", "local", fmt.Sprintf("%x", self.Addr.Over()), "remote", fmt.Sprintf("%x", rhs.Addr.Over())) - // log.Debug("RHS handshake", "addr", rhs.Addr, "peeraddr", rhs.peerAddr) - // log.Debug("LHS handshake", "addr", self.Addr, "peeraddr", self.peerAddr) - // - // log.Warn( - // strings.Join( - // []string{ - // "handshake ok:", - // fmt.Sprintf("p2p.Peer.ID(): ...... %v", p.ID()), - // fmt.Sprintf("peerAddr.UAddr: ...... %x", self.peerAddr.UAddr[:32]), - // fmt.Sprintf("peerAddr.OAddr: ...... %x", self.peerAddr.OAddr[:32]), - // fmt.Sprintf("selfAddr.UAddr: ...... %x", self.Addr.UAddr[:32]), - // fmt.Sprintf("selfAddr.OAddr: ...... %x", self.Addr.OAddr[:32]), - // fmt.Sprintf("ToOverlayAddr(p2p.Peer.ID): .... %x", ToOverlayAddr(peerid[:])), - // fmt.Sprintf("ToOverlayAddr(peerAddr.UAddr): .. %x", ToOverlayAddr(self.peerAddr.UAddr)), - // }, "\n", - // ), - // ) - return nil } diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go index caeb227d47..5696a23137 100644 --- a/swarm/network/simulations/discovery/discovery_test.go +++ b/swarm/network/simulations/discovery/discovery_test.go @@ -3,8 +3,8 @@ package discovery_test import ( "context" "encoding/json" - "fmt" "flag" + "fmt" "io/ioutil" "os" "testing" @@ -30,8 +30,8 @@ var services = adapters.Services{ var ( snapshotFile = flag.String("snapshot", "", "create snapshot") - nodeCount = flag.Int("nodes", 10, "number of nodes to create (default 10)") - verbose = flag.Bool("verbose", false, "output extra logs") + nodeCount = flag.Int("nodes", 10, "number of nodes to create (default 10)") + verbose = flag.Bool("verbose", false, "output extra logs") ) func init() { @@ -156,7 +156,7 @@ func testDiscoverySimulation(t *testing.T, adapter adapters.NodeAdapter) { t.Fatalf("corrupt json snapshot: %v", err) } log.Info("writing snapshot", "file", *snapshotFile) - err = ioutil.WriteFile(*snapshotFile, jsonsnapshot, os.ModePerm) + err = ioutil.WriteFile(*snapshotFile, jsonsnapshot, 0755) if err != nil { t.Fatal(err) } diff --git a/swarm/pss/client/client.go b/swarm/pss/client/client.go index 48942d912c..7fe57f0dda 100644 --- a/swarm/pss/client/client.go +++ b/swarm/pss/client/client.go @@ -1,82 +1,3 @@ -// simple abstraction for implementing pss functionality -// -// the pss client library aims to simplify usage of the p2p.protocols package over pss -// -// IO is performed using the ordinary p2p.MsgReadWriter interface, which transparently communicates with a pss node via RPC using websockets as transport layer, using methods in the PssAPI class in the swarm/pss package -// -// -// Minimal-ish usage example (requires a running pss node with websocket RPC): -// -// -// import ( -// "context" -// "fmt" -// "os" -// pss "github.com/ethereum/go-ethereum/swarm/pss/client" -// "github.com/ethereum/go-ethereum/p2p/protocols" -// "github.com/ethereum/go-ethereum/p2p" -// "github.com/ethereum/go-ethereum/pot" -// "github.com/ethereum/go-ethereum/log" -// ) -// -// type FooMsg struct { -// Bar int -// } -// -// -// func fooHandler (msg interface{}) error { -// foomsg, ok := msg.(*FooMsg) -// if ok { -// log.Debug("Yay, just got a message", "msg", foomsg) -// } -// return fmt.Errorf("Unknown message") -// } -// -// spec := &protocols.Spec{ -// Name: "foo", -// Version: 1, -// MaxMsgSize: 1024, -// Messages: []interface{}{ -// FooMsg{}, -// }, -// } -// -// proto := &p2p.Protocol{ -// Name: spec.Name, -// Version: spec.Version, -// Length: uint64(len(spec.Messages)), -// Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error { -// pp := protocols.NewPeer(p, rw, spec) -// return pp.Run(fooHandler) -// }, -// } -// -// func implementation() { -// cfg := pss.NewClientConfig() -// psc := pss.NewClient(context.Background(), nil, cfg) -// err := psc.Start() -// if err != nil { -// log.Crit("can't start pss client") -// os.Exit(1) -// } -// -// log.Debug("connected to pss node", "bzz addr", psc.BaseAddr) -// -// err = psc.RunProtocol(proto) -// if err != nil { -// log.Crit("can't start protocol on pss websocket") -// os.Exit(1) -// } -// -// addr := pot.RandomAddress() // should be a real address, of course -// psc.AddPssPeer(addr, spec) -// -// // use the protocol for something -// -// psc.Stop() -// } -// -// BUG(test): TestIncoming test times out due to deadlock issues in the swarm hive package client import ( @@ -86,7 +7,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/protocols" @@ -99,45 +19,29 @@ import ( const ( inboxCapacity = 3000 outboxCapacity = 100 - defaultWSHost = 8546 addrLen = common.HashLength ) -// RemoteHost: hostname of node running websockets proxy to pss (default localhost) -// RemotePort: port of node running websockets proxy to pss (0 = go-ethereum node default) -// SelfHost: local if host to connect from -// Secure: whether or not to use secure connection (not currently in use) -type ClientConfig struct { - RemoteHost string - RemotePort int - SelfHost string - Secure bool -} - -// Generates a pss client configuration with default values -func NewClientConfig() *ClientConfig { - return &ClientConfig{ - SelfHost: node.DefaultWSHost, - RemoteHost: node.DefaultWSHost, - RemotePort: node.DefaultWSPort, - } -} - // After a successful connection with Client.Start, BaseAddr contains the swarm overlay address of the pss node type Client struct { - BaseAddr []byte - localuri string - remoteuri string - ctx context.Context - cancel func() - subscription *rpc.ClientSubscription - topicsC chan []byte - msgC chan pss.APIMsg - quitC chan struct{} - ws *rpc.Client - lock sync.Mutex - peerPool map[pss.Topic]map[pot.Address]*pssRPCRW - protos map[pss.Topic]*p2p.Protocol + BaseAddr []byte + + // peers + peerPool map[pss.Topic]map[pot.Address]*pssRPCRW + protos map[pss.Topic]*p2p.Protocol + + // rpc connections + rpc *rpc.Client + sub *rpc.ClientSubscription + + // channels + topicsC chan []byte + msgC chan pss.APIMsg + quitC chan struct{} + + ctx context.Context + cancel func() + lock sync.Mutex } // implements p2p.MsgReadWriter @@ -181,26 +85,46 @@ func (rw *pssRPCRW) WriteMsg(msg p2p.Msg) error { return err } - return rw.Client.ws.CallContext(rw.Client.ctx, nil, "pss_send", rw.topic, pss.APIMsg{ + return rw.Client.rpc.CallContext(rw.Client.ctx, nil, "pss_send", rw.topic, pss.APIMsg{ Addr: rw.addr.Bytes(), Msg: pmsg, }) } -// Constructor for production-environment clients -// Performs sanity checks on configuration paramters and gets everything ready to connect to pss node -func NewClient(ctx context.Context, cancel func(), config *ClientConfig) (*Client, error) { - prefix := "ws" +func NewClient(ctx context.Context, cancel func(), rpcurl string) (*Client, error) { + rpcclient, err := rpc.Dial(rpcurl) + if err != nil { + return nil, err + } + client, err := NewClientWithRPC(ctx, cancel, rpcclient) + if err != nil { + return nil, err + } + return client, nil +} + +// Constructor for test implementations +// The 'rpcclient' parameter allows passing a in-memory rpc client to act as the remote websocket RPC. +func NewClientWithRPC(ctx context.Context, cancel func(), rpcclient *rpc.Client) (*Client, error) { + client := newClient(ctx, cancel) + client.rpc = rpcclient + err := client.rpc.CallContext(client.ctx, &client.BaseAddr, "pss_baseAddr") + if err != nil { + return nil, fmt.Errorf("cannot get pss node baseaddress: %v", err) + } + return client, nil +} + +func newClient(ctx context.Context, cancel func()) (client *Client) { if ctx == nil { ctx = context.Background() } if cancel == nil { - cancel = func() { return } + cancel = func() {} } - - pssc := &Client{ + client = &Client{ msgC: make(chan pss.APIMsg), quitC: make(chan struct{}), peerPool: make(map[pss.Topic]map[pot.Address]*pssRPCRW), @@ -208,64 +132,13 @@ func NewClient(ctx context.Context, cancel func(), config *ClientConfig) (*Clien ctx: ctx, cancel: cancel, } - - if config.Secure { - prefix = "wss" - } - - pssc.remoteuri = fmt.Sprintf("%s://%s:%d", prefix, config.RemoteHost, config.RemotePort) - pssc.localuri = fmt.Sprintf("%s://%s", prefix, config.SelfHost) - - return pssc, nil -} - -// Constructor for test implementations -// The 'rpcclient' parameter allows passing a in-memory rpc client to act as the remote websocket RPC. -func NewClientWithRPC(ctx context.Context, rpcclient *rpc.Client) (*Client, error) { - var oaddr []byte - err := rpcclient.CallContext(ctx, &oaddr, "pss_baseAddr") - if err != nil { - return nil, fmt.Errorf("cannot get pss node baseaddress: %v", err) - } - return &Client{ - msgC: make(chan pss.APIMsg), - quitC: make(chan struct{}), - peerPool: make(map[pss.Topic]map[pot.Address]*pssRPCRW), - protos: make(map[pss.Topic]*p2p.Protocol), - ws: rpcclient, - ctx: ctx, - BaseAddr: oaddr, - }, nil + return } func (self *Client) shutdown() { self.cancel() } -// Connects to the websockets RPC -// Retrieves the swarm overlay address from the pss node -func (self *Client) Start() error { - if self.ws != nil { - return nil - } - log.Debug("Dialing ws", "src", self.localuri, "dst", self.remoteuri) - ws, err := rpc.DialWebsocket(self.ctx, self.remoteuri, self.localuri) - if err != nil { - return fmt.Errorf("Couldnt dial pss websocket: %v", err) - } - - var oaddr []byte - err = ws.CallContext(self.ctx, &oaddr, "pss_baseAddr") - if err != nil { - return err - } - - self.ws = ws - self.BaseAddr = oaddr - - return nil -} - // Mounts a new devp2p protcool on the pss connection // the protocol is aliased as a "pss topic" // uses normal devp2p Send and incoming message handler routines from the p2p/protocols package @@ -275,12 +148,11 @@ func (self *Client) RunProtocol(proto *p2p.Protocol) error { topic := pss.NewTopic(proto.Name, int(proto.Version)) msgC := make(chan pss.APIMsg) self.peerPool[topic] = make(map[pot.Address]*pssRPCRW) - sub, err := self.ws.Subscribe(self.ctx, "pss", msgC, "receive", topic) + sub, err := self.rpc.Subscribe(self.ctx, "pss", msgC, "receive", topic) if err != nil { return fmt.Errorf("pss event subscription failed: %v", err) } - - self.subscription = sub + self.sub = sub // dispatch incoming messages go func() { diff --git a/swarm/pss/client/client_test.go b/swarm/pss/client/client_test.go index 9f2dea8ef9..b99cd7333a 100644 --- a/swarm/pss/client/client_test.go +++ b/swarm/pss/client/client_test.go @@ -51,7 +51,7 @@ func TestIncoming(t *testing.T) { t.Fatalf(err.Error()) } - client.ws.Call(&addr, "psstest_baseAddr") + client.rpc.Call(&addr, "psstest_baseAddr") code, _ := pss.PingProtocol.GetCode(&pss.PingMsg{}) rlpbundle, err := pss.NewProtocolMsg(code, &pss.PingMsg{ @@ -94,7 +94,7 @@ func TestOutgoing(t *testing.T) { t.Fatalf(err.Error()) } - client.ws.Call(&addr, "psstest_baseAddr") + client.rpc.Call(&addr, "psstest_baseAddr") copy(potaddr[:], addr) msg := &pss.PingMsg{ @@ -118,12 +118,7 @@ func TestOutgoing(t *testing.T) { func baseTester(t *testing.T, proto *p2p.Protocol, ps *pss.Pss, ctx context.Context, cancel func(), quitC chan struct{}) (*Client, error) { var err error - client := newClient(t, ctx, cancel, quitC) - - err = client.Start() - if err != nil { - return nil, err - } + client := newTestclient(t, ctx, cancel, quitC) err = client.RunProtocol(proto) @@ -148,14 +143,7 @@ func newProtocol(ping *pss.Ping) *p2p.Protocol { } } -func newClient(t *testing.T, ctx context.Context, cancel func(), quitC chan struct{}) *Client { - - conf := NewClientConfig() - - pssclient, err := NewClient(ctx, cancel, conf) - if err != nil { - t.Fatalf(err.Error()) - } +func newTestclient(t *testing.T, ctx context.Context, cancel func(), quitC chan struct{}) *Client { ps := pss.NewTestPss(nil) srv := rpc.NewServer() @@ -177,5 +165,11 @@ func newClient(t *testing.T, ctx context.Context, cancel func(), quitC chan stru <-quitC sock.Close() }() + + pssclient, err := NewClient(ctx, cancel, "ws://localhost:8546") + if err != nil { + t.Fatalf(err.Error()) + } + return pssclient } diff --git a/swarm/pss/client/doc.go b/swarm/pss/client/doc.go new file mode 100644 index 0000000000..22aa82d835 --- /dev/null +++ b/swarm/pss/client/doc.go @@ -0,0 +1,80 @@ +// simple abstraction for implementing pss functionality +// +// the pss client library aims to simplify usage of the p2p.protocols package over pss +// +// IO is performed using the ordinary p2p.MsgReadWriter interface, which transparently communicates with a pss node via RPC using websockets as transport layer, using methods in the PssAPI class in the swarm/pss package +// +// +// Minimal-ish usage example (requires a running pss node with websocket RPC): +// +// +// import ( +// "context" +// "fmt" +// "os" +// pss "github.com/ethereum/go-ethereum/swarm/pss/client" +// "github.com/ethereum/go-ethereum/p2p/protocols" +// "github.com/ethereum/go-ethereum/p2p" +// "github.com/ethereum/go-ethereum/pot" +// "github.com/ethereum/go-ethereum/log" +// ) +// +// type FooMsg struct { +// Bar int +// } +// +// +// func fooHandler (msg interface{}) error { +// foomsg, ok := msg.(*FooMsg) +// if ok { +// log.Debug("Yay, just got a message", "msg", foomsg) +// } +// return fmt.Errorf("Unknown message") +// } +// +// spec := &protocols.Spec{ +// Name: "foo", +// Version: 1, +// MaxMsgSize: 1024, +// Messages: []interface{}{ +// FooMsg{}, +// }, +// } +// +// proto := &p2p.Protocol{ +// Name: spec.Name, +// Version: spec.Version, +// Length: uint64(len(spec.Messages)), +// Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error { +// pp := protocols.NewPeer(p, rw, spec) +// return pp.Run(fooHandler) +// }, +// } +// +// func implementation() { +// cfg := pss.NewClientConfig() +// psc := pss.NewClient(context.Background(), nil, cfg) +// err := psc.Start() +// if err != nil { +// log.Crit("can't start pss client") +// os.Exit(1) +// } +// +// log.Debug("connected to pss node", "bzz addr", psc.BaseAddr) +// +// err = psc.RunProtocol(proto) +// if err != nil { +// log.Crit("can't start protocol on pss websocket") +// os.Exit(1) +// } +// +// addr := pot.RandomAddress() // should be a real address, of course +// psc.AddPssPeer(addr, spec) +// +// // use the protocol for something +// +// psc.Stop() +// } +// +// BUG(test): TestIncoming test times out due to deadlock issues in the swarm hive +package client diff --git a/swarm/swarm.go b/swarm/swarm.go index 8dfe124e15..5f2e154696 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -1,5 +1,3 @@ -// +build !psschat - // Copyright 2016 The go-ethereum Authors // This file is part of the go-ethereum library. // @@ -35,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm/api" httpapi "github.com/ethereum/go-ethereum/swarm/api/http" @@ -47,13 +46,12 @@ import ( // the swarm stack type Swarm struct { - config *api.Config // swarm configuration - api *api.Api // high level api layer (fs/manifest) - dns api.Resolver // DNS registrar - storage storage.ChunkStore // internal access to storage, common interface to cloud storage backends - dpa *storage.DPA // distributed preimage archive, the local API to the storage with document level storage/retrieval support - cloud storage.CloudStore // procurement, cloud storage backend (can multi-cloud) - //hive *network.Hive // the logistic manager + config *api.Config // swarm configuration + api *api.Api // high level api layer (fs/manifest) + dns api.Resolver // DNS registrar + storage storage.ChunkStore // internal access to storage, common interface to cloud storage backends + dpa *storage.DPA // distributed preimage archive, the local API to the storage with document level storage/retrieval support + cloud storage.CloudStore // procurement, cloud storage backend (can multi-cloud) bzz *network.Bzz // hive and bzz protocol backend chequebook.Backend // simple blockchain Backend privateKey *ecdsa.PrivateKey @@ -122,14 +120,6 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api. } self.bzz = network.NewBzz(bzzconfig, to, nil) - // set up the kademlia hive - // self.hive = network.NewHive( - // config.HiveParams, // configuration parameters - // self.Kademlia, - // stateStore, - // ) - // log.Debug(fmt.Sprintf("Set up swarm network with Kademlia hive")) - // // setup cloud storage internal access layer self.storage = storage.NewNetStore(hash, self.lstore, nil, config.StoreParams) log.Debug("-> swarm net store shared access layer to Swarm Chunk Store") @@ -213,7 +203,6 @@ func (self *Swarm) Start(net *p2p.Server) error { log.Warn(fmt.Sprintf("Starting Swarm service")) - // self.hive.Start(net) err := self.bzz.Start(net) if err != nil { log.Error("bzz failed", "err", err) @@ -251,7 +240,6 @@ func (self *Swarm) Start(net *p2p.Server) error { // stops all component services. func (self *Swarm) Stop() error { self.dpa.Stop() - //self.hive.Stop() self.bzz.Stop() //if self.pss != nil { // self.pss.Stop() @@ -281,18 +269,6 @@ func (self *Swarm) Protocols() (protos []p2p.Protocol) { protos = append(protos, p) } } - // ct := network.BzzCodeMap() - // ct.Register(2, network.DiscoveryMsgs...) - - // proto := network.Bzz( - // self.hive.Overlay.GetAddr().Over(), - // self.hive.Overlay.GetAddr().Under(), - // ct, - // nil, - // nil, - // nil, - // ) - // return } @@ -380,7 +356,7 @@ func NewLocalSwarm(datadir, port string) (self *Swarm, err error) { return } - config, err := api.NewConfig(datadir, common.Address{}, prvKey, network.NetworkID) + config, err := api.NewConfig(datadir, common.Address{}, prvKey, network.NetworkId) if err != nil { return } diff --git a/swarm/swarm_psschat.go b/swarm/swarm_psschat.go deleted file mode 100644 index 8b3ba10b84..0000000000 --- a/swarm/swarm_psschat.go +++ /dev/null @@ -1,415 +0,0 @@ -// +build psschat - -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package swarm - -import ( - "bytes" - "context" - "crypto/ecdsa" - "fmt" - "io/ioutil" - - "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/contracts/chequebook" - "github.com/ethereum/go-ethereum/contracts/ens" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/p2p/discover" - "github.com/ethereum/go-ethereum/ethclient" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/swarm/api" - httpapi "github.com/ethereum/go-ethereum/swarm/api/http" - "github.com/ethereum/go-ethereum/swarm/fuse" - "github.com/ethereum/go-ethereum/swarm/network" - "github.com/ethereum/go-ethereum/swarm/pss" - "github.com/ethereum/go-ethereum/swarm/storage" - //psschat "github.com/ethereum/go-ethereum/swarm/pss/protocols/chat" -) - -// the swarm stack -type Swarm struct { - config *api.Config // swarm configuration - api *api.Api // high level api layer (fs/manifest) - dns api.Resolver // DNS registrar - storage storage.ChunkStore // internal access to storage, common interface to cloud storage backends - dpa *storage.DPA // distributed preimage archive, the local API to the storage with document level storage/retrieval support - cloud storage.CloudStore // procurement, cloud storage backend (can multi-cloud) - //hive *network.Hive // the logistic manager - bzz *network.Bzz // hive and bzz protocol - backend chequebook.Backend // simple blockchain Backend - privateKey *ecdsa.PrivateKey - corsString string - swapEnabled bool - lstore *storage.LocalStore // local store, needs to store for releasing resources after node stopped - sfs *fuse.SwarmFS // need this to cleanup all the active mounts on node exit - pss *pss.Pss -} - -type SwarmAPI struct { - Api *api.Api - Backend chequebook.Backend - PrvKey *ecdsa.PrivateKey -} - -func (self *Swarm) API() *SwarmAPI { - return &SwarmAPI{ - Api: self.api, - Backend: self.backend, - PrvKey: self.privateKey, - } -} - -// creates a new swarm service instance -// implements node.Service -func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api.Config, swapEnabled, syncEnabled bool, cors string, pssEnabled bool) (self *Swarm, err error) { - if bytes.Equal(common.FromHex(config.PublicKey), storage.ZeroKey) { - return nil, fmt.Errorf("empty public key") - } - if bytes.Equal(common.FromHex(config.BzzKey), storage.ZeroKey) { - return nil, fmt.Errorf("empty bzz key") - } - - self = &Swarm{ - config: config, - swapEnabled: swapEnabled, - backend: backend, - privateKey: config.Swap.PrivateKey(), - corsString: cors, - } - log.Debug(fmt.Sprintf("Setting up Swarm service components")) - - hash := storage.MakeHashFunc(config.ChunkerParams.Hash) - self.lstore, err = storage.NewLocalStore(hash, config.StoreParams) - if err != nil { - return - } - - log.Debug("Set up local db access (iterator/counter)") - - kp := network.NewKadParams() - to := network.NewKademlia( - common.FromHex(config.BzzKey), - kp, - ) - - config.HiveParams.Discovery = true - - nodeid := discover.PubkeyID(crypto.ToECDSAPub(common.FromHex(config.PublicKey))) - addr := network.NewAddrFromNodeID(nodeid) - bzzconfig := &network.BzzConfig{ - OverlayAddr: common.FromHex(config.BzzKey), - UnderlayAddr: addr.UAddr, - HiveParams: config.HiveParams, - } - self.bzz = network.NewBzz(bzzconfig, to, nil) - - // set up the kademlia hive - // self.hive = network.NewHive( - // config.HiveParams, // configuration parameters - // self.Kademlia, - // stateStore, - // ) - // log.Debug(fmt.Sprintf("Set up swarm network with Kademlia hive")) - // - // setup cloud storage internal access layer - self.storage = storage.NewNetStore(hash, self.lstore, nil, config.StoreParams) - log.Debug("-> swarm net store shared access layer to Swarm Chunk Store") - - // set up DPA, the cloud storage local access layer - dpaChunkStore := storage.NewDpaChunkStore(self.lstore, self.storage) - log.Debug(fmt.Sprintf("-> Local Access to Swarm")) - - // Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage - self.dpa = storage.NewDPA(dpaChunkStore, self.config.ChunkerParams) - log.Debug(fmt.Sprintf("-> Content Store API")) - - // netstore is broken, temp workaround to fiddle with pss - cachedir, err := ioutil.TempDir("", "bzz-tmp") - if err != nil { - return nil, err - } - self.dpa, err = storage.NewLocalDPA(cachedir) - if err != nil { - return nil, err - } - - // Pss = postal service over swarm (devp2p over bzz) - if pssEnabled { - pssparams := pss.NewPssParams(false) - self.pss = pss.NewPss(to, self.dpa, pssparams) - } - - // set up high level api - transactOpts := bind.NewKeyedTransactor(self.privateKey) - - if backend == (*ethclient.Client)(nil) { - log.Warn("No ENS, please specify non-empty --ethapi to use domain name resolution") - } else { - self.dns, err = ens.NewENS(transactOpts, config.EnsRoot, self.backend) - if err != nil { - return nil, err - } - } - log.Debug(fmt.Sprintf("-> Swarm Domain Name Registrar @ address %v", config.EnsRoot.Hex())) - - self.api = api.NewApi(self.dpa, self.dns) - - // Manifests for Smart Hosting - log.Debug(fmt.Sprintf("-> Web3 virtual server API")) - - self.sfs = fuse.NewSwarmFS(self.api) - log.Debug("-> Initializing Fuse file system") - - return self, nil -} - -/* -Start is called when the stack is started -* starts the network kademlia hive peer management -* (starts netStore level 0 api) -* starts DPA level 1 api (chunking -> store/retrieve requests) -* (starts level 2 api) -* starts http proxy server -* registers url scheme handlers for bzz, etc -* TODO: start subservices like sword, swear, swarmdns -*/ -// implements the node.Service interface -func (self *Swarm) Start(net *p2p.Server) error { - - // update uaddr to correct enode - newaddr := self.bzz.UpdateLocalAddr([]byte(net.Self().String())) - log.Warn("Updated bzz local addr", "oaddr", fmt.Sprintf("%x", newaddr.OAddr), "uaddr", fmt.Sprintf("%x", newaddr.UAddr)) - - // set chequebook - if self.swapEnabled { - ctx := context.Background() // The initial setup has no deadline. - err := self.SetChequebook(ctx) - if err != nil { - return fmt.Errorf("Unable to set chequebook for SWAP: %v", err) - } - log.Debug(fmt.Sprintf("-> cheque book for SWAP: %v", self.config.Swap.Chequebook())) - } else { - log.Debug(fmt.Sprintf("SWAP disabled: no cheque book set")) - } - - log.Warn(fmt.Sprintf("Starting Swarm service")) - - // self.hive.Start(net) - err := self.bzz.Start(net) - if err != nil { - log.Error("bzz failed", "err", err) - return err - } - log.Info(fmt.Sprintf("Swarm network started on bzz address: %x", self.bzz.Hive.Overlay.BaseAddr())) - - if self.pss != nil { - self.pss.Start(net) - topic := pss.NewTopic("pssChat", 1) - self.pss.Register(&topic, func(msg []byte, p *p2p.Peer, from []byte) error { - log.Trace("placeholder handler for node got psschat proto incoming", "msg", msg, "from", from) - return nil - }) - log.Info("Pss started ... with 'pssChat' :)") - } - - self.dpa.Start() - log.Debug(fmt.Sprintf("Swarm DPA started")) - - // start swarm http proxy server - if self.config.Port != "" { - addr := ":" + self.config.Port - go httpapi.StartHttpServer(self.api, &httpapi.ServerConfig{ - Addr: addr, - CorsString: self.corsString, - }) - } - - log.Debug(fmt.Sprintf("Swarm http proxy started on port: %v", self.config.Port)) - - if self.corsString != "" { - log.Debug(fmt.Sprintf("Swarm http proxy started with corsdomain: %v", self.corsString)) - } - - return nil -} - -// implements the node.Service interface -// stops all component services. -func (self *Swarm) Stop() error { - self.dpa.Stop() - //self.hive.Stop() - self.bzz.Stop() - //if self.pss != nil { - // self.pss.Stop() - //} - if ch := self.config.Swap.Chequebook(); ch != nil { - ch.Stop() - ch.Save() - } - - if self.lstore != nil { - self.lstore.DbStore.Close() - } - self.sfs.Stop() - return self.config.Save() -} - -// implements the node.Service interface -func (self *Swarm) Protocols() (protos []p2p.Protocol) { - - for _, p := range self.bzz.Protocols() { - protos = append(protos, p) - } - - if self.pss != nil { - log.Warn("adding pss protos") - for _, p := range self.pss.Protocols() { - protos = append(protos, p) - } - } - // ct := network.BzzCodeMap() - // ct.Register(2, network.DiscoveryMsgs...) - - // proto := network.Bzz( - // self.hive.Overlay.GetAddr().Over(), - // self.hive.Overlay.GetAddr().Under(), - // ct, - // nil, - // nil, - // nil, - // ) - // - return -} - -// implements node.Service -// Apis returns the RPC Api descriptors the Swarm implementation offers -func (self *Swarm) APIs() []rpc.API { - - apis := []rpc.API{ - // public APIs - { - Namespace: "bzz", - Version: "0.1", - Service: &Info{self.config, chequebook.ContractParams}, - Public: true, - }, - // admin APIs - { - Namespace: "bzz", - Version: "0.1", - Service: api.NewControl(self.api, self.bzz.Hive), - Public: false, - }, - { - Namespace: "chequebook", - Version: chequebook.Version, - Service: chequebook.NewApi(self.config.Swap.Chequebook), - Public: false, - }, - { - Namespace: "swarmfs", - Version: fuse.Swarmfs_Version, - Service: self.sfs, - Public: false, - }, - // storage APIs - // DEPRECATED: Use the HTTP API instead - { - Namespace: "bzz", - Version: "0.1", - Service: api.NewStorage(self.api), - Public: true, - }, - { - Namespace: "bzz", - Version: "0.1", - Service: api.NewFileSystem(self.api), - Public: false, - }, - // {Namespace, Version, api.NewAdmin(self), false}, - } - - for _, api := range self.bzz.APIs() { - apis = append(apis, api) - } - - if self.pss != nil { - for _, api := range self.pss.APIs() { - apis = append(apis, api) - } - } - - return apis -} - -func (self *Swarm) Api() *api.Api { - return self.api -} - -// SetChequebook ensures that the local checquebook is set up on chain. -func (self *Swarm) SetChequebook(ctx context.Context) error { - err := self.config.Swap.SetChequebook(ctx, self.backend, self.config.Path) - if err != nil { - return err - } - log.Info(fmt.Sprintf("new chequebook set (%v): saving config file, resetting all connections in the hive", self.config.Swap.Contract.Hex())) - self.config.Save() - return nil -} - -// Local swarm without netStore -func NewLocalSwarm(datadir, port string) (self *Swarm, err error) { - - prvKey, err := crypto.GenerateKey() - if err != nil { - return - } - - config, err := api.NewConfig(datadir, common.Address{}, prvKey, network.NetworkId) - if err != nil { - return - } - config.Port = port - - dpa, err := storage.NewLocalDPA(datadir) - if err != nil { - return - } - - self = &Swarm{ - api: api.NewApi(dpa, nil), - config: config, - } - - return -} - -// serialisable info about swarm -type Info struct { - *api.Config - *chequebook.Params -} - -func (self *Info) Info() *Info { - return self -} From e94b1e6bfa42b17a7597749911e6ab74460021fa Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 22 Jun 2017 12:30:27 +0200 Subject: [PATCH 13/17] swarm/pss: test panic fix, client cancel rm --- swarm/pss/client/client.go | 19 +++++-------------- swarm/pss/client/client_test.go | 18 +++++++++--------- swarm/pss/pss.go | 2 ++ 3 files changed, 16 insertions(+), 23 deletions(-) diff --git a/swarm/pss/client/client.go b/swarm/pss/client/client.go index 7fe57f0dda..87fed9dcbd 100644 --- a/swarm/pss/client/client.go +++ b/swarm/pss/client/client.go @@ -92,13 +92,13 @@ func (rw *pssRPCRW) WriteMsg(msg p2p.Msg) error { } -func NewClient(ctx context.Context, cancel func(), rpcurl string) (*Client, error) { +func NewClient(ctx context.Context, rpcurl string) (*Client, error) { rpcclient, err := rpc.Dial(rpcurl) if err != nil { return nil, err } - client, err := NewClientWithRPC(ctx, cancel, rpcclient) + client, err := NewClientWithRPC(ctx, rpcclient) if err != nil { return nil, err } @@ -107,8 +107,8 @@ func NewClient(ctx context.Context, cancel func(), rpcurl string) (*Client, erro // Constructor for test implementations // The 'rpcclient' parameter allows passing a in-memory rpc client to act as the remote websocket RPC. -func NewClientWithRPC(ctx context.Context, cancel func(), rpcclient *rpc.Client) (*Client, error) { - client := newClient(ctx, cancel) +func NewClientWithRPC(ctx context.Context, rpcclient *rpc.Client) (*Client, error) { + client := newClient(ctx) client.rpc = rpcclient err := client.rpc.CallContext(client.ctx, &client.BaseAddr, "pss_baseAddr") if err != nil { @@ -117,28 +117,20 @@ func NewClientWithRPC(ctx context.Context, cancel func(), rpcclient *rpc.Client) return client, nil } -func newClient(ctx context.Context, cancel func()) (client *Client) { +func newClient(ctx context.Context) (client *Client) { if ctx == nil { ctx = context.Background() } - if cancel == nil { - cancel = func() {} - } client = &Client{ msgC: make(chan pss.APIMsg), quitC: make(chan struct{}), peerPool: make(map[pss.Topic]map[pot.Address]*pssRPCRW), protos: make(map[pss.Topic]*p2p.Protocol), ctx: ctx, - cancel: cancel, } return } -func (self *Client) shutdown() { - self.cancel() -} - // Mounts a new devp2p protcool on the pss connection // the protocol is aliased as a "pss topic" // uses normal devp2p Send and incoming message handler routines from the p2p/protocols package @@ -171,7 +163,6 @@ func (self *Client) RunProtocol(proto *p2p.Protocol) error { self.peerPool[topic][addr].msgC <- msg.Msg }() case <-self.quitC: - self.shutdown() return } } diff --git a/swarm/pss/client/client_test.go b/swarm/pss/client/client_test.go index b99cd7333a..bcb6c16775 100644 --- a/swarm/pss/client/client_test.go +++ b/swarm/pss/client/client_test.go @@ -30,7 +30,7 @@ func TestRunProtocol(t *testing.T) { C: make(chan struct{}), } proto := newProtocol(ping) - _, err := baseTester(t, proto, ps, nil, nil, quitC) + _, err := baseTester(t, proto, ps, nil, quitC) if err != nil { t.Fatalf(err.Error()) } @@ -40,13 +40,13 @@ func TestRunProtocol(t *testing.T) { func TestIncoming(t *testing.T) { quitC := make(chan struct{}) ps := pss.NewTestPss(nil) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + ctx, _ := context.WithTimeout(context.Background(), time.Second*5) var addr []byte ping := &pss.Ping{ C: make(chan struct{}), } proto := newProtocol(ping) - client, err := baseTester(t, proto, ps, ctx, cancel, quitC) + client, err := baseTester(t, proto, ps, ctx, quitC) if err != nil { t.Fatalf(err.Error()) } @@ -81,7 +81,7 @@ func TestIncoming(t *testing.T) { func TestOutgoing(t *testing.T) { quitC := make(chan struct{}) ps := pss.NewTestPss(nil) - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*250) + ctx, _ := context.WithTimeout(context.Background(), time.Millisecond*250) var addr []byte var potaddr pot.Address @@ -89,7 +89,7 @@ func TestOutgoing(t *testing.T) { C: make(chan struct{}), } proto := newProtocol(ping) - client, err := baseTester(t, proto, ps, ctx, cancel, quitC) + client, err := baseTester(t, proto, ps, ctx, quitC) if err != nil { t.Fatalf(err.Error()) } @@ -115,10 +115,10 @@ func TestOutgoing(t *testing.T) { quitC <- struct{}{} } -func baseTester(t *testing.T, proto *p2p.Protocol, ps *pss.Pss, ctx context.Context, cancel func(), quitC chan struct{}) (*Client, error) { +func baseTester(t *testing.T, proto *p2p.Protocol, ps *pss.Pss, ctx context.Context, quitC chan struct{}) (*Client, error) { var err error - client := newTestclient(t, ctx, cancel, quitC) + client := newTestclient(t, ctx, quitC) err = client.RunProtocol(proto) @@ -143,7 +143,7 @@ func newProtocol(ping *pss.Ping) *p2p.Protocol { } } -func newTestclient(t *testing.T, ctx context.Context, cancel func(), quitC chan struct{}) *Client { +func newTestclient(t *testing.T, ctx context.Context, quitC chan struct{}) *Client { ps := pss.NewTestPss(nil) srv := rpc.NewServer() @@ -166,7 +166,7 @@ func newTestclient(t *testing.T, ctx context.Context, cancel func(), quitC chan sock.Close() }() - pssclient, err := NewClient(ctx, cancel, "ws://localhost:8546") + pssclient, err := NewClient(ctx, "ws://localhost:8546") if err != nil { t.Fatalf(err.Error()) } diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index 1a2b4328a7..9033bd409a 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -131,6 +131,8 @@ func (self *Pss) Protocols() []p2p.Protocol { // Starts the PssMsg protocol func (self *Pss) Run(p *p2p.Peer, rw p2p.MsgReadWriter) error { + self.lock.Lock() + defer self.lock.Unlock() pp := protocols.NewPeer(p, rw, pssSpec) //addr := network.NewAddrFromNodeID(id) //potaddr := pot.NewHashAddressFromBytes(addr.OAddr) From 62c7a8d6ba5ccb1a65d341bfb2e033b30ce07fb0 Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 22 Jun 2017 13:00:28 +0200 Subject: [PATCH 14/17] swarm/pss: removed useless contexts --- swarm/pss/client/client.go | 30 +++++++++++++-------------- swarm/pss/client/client_test.go | 18 +++++----------- swarm/pss/simulations/service_test.go | 4 ++-- 3 files changed, 21 insertions(+), 31 deletions(-) diff --git a/swarm/pss/client/client.go b/swarm/pss/client/client.go index 87fed9dcbd..68ad6cdd89 100644 --- a/swarm/pss/client/client.go +++ b/swarm/pss/client/client.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "sync" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" @@ -17,6 +18,7 @@ import ( ) const ( + rpcTimeout = time.Second inboxCapacity = 3000 outboxCapacity = 100 addrLen = common.HashLength @@ -39,9 +41,7 @@ type Client struct { msgC chan pss.APIMsg quitC chan struct{} - ctx context.Context - cancel func() - lock sync.Mutex + lock sync.Mutex } // implements p2p.MsgReadWriter @@ -85,20 +85,21 @@ func (rw *pssRPCRW) WriteMsg(msg p2p.Msg) error { return err } - return rw.Client.rpc.CallContext(rw.Client.ctx, nil, "pss_send", rw.topic, pss.APIMsg{ + ctx, _ := context.WithTimeout(context.Background(), rpcTimeout) + return rw.Client.rpc.CallContext(ctx, nil, "pss_send", rw.topic, pss.APIMsg{ Addr: rw.addr.Bytes(), Msg: pmsg, }) } -func NewClient(ctx context.Context, rpcurl string) (*Client, error) { +func NewClient(rpcurl string) (*Client, error) { rpcclient, err := rpc.Dial(rpcurl) if err != nil { return nil, err } - client, err := NewClientWithRPC(ctx, rpcclient) + client, err := NewClientWithRPC(rpcclient) if err != nil { return nil, err } @@ -107,26 +108,23 @@ func NewClient(ctx context.Context, rpcurl string) (*Client, error) { // Constructor for test implementations // The 'rpcclient' parameter allows passing a in-memory rpc client to act as the remote websocket RPC. -func NewClientWithRPC(ctx context.Context, rpcclient *rpc.Client) (*Client, error) { - client := newClient(ctx) +func NewClientWithRPC(rpcclient *rpc.Client) (*Client, error) { + client := newClient() client.rpc = rpcclient - err := client.rpc.CallContext(client.ctx, &client.BaseAddr, "pss_baseAddr") + ctx, _ := context.WithTimeout(context.Background(), rpcTimeout) + err := client.rpc.CallContext(ctx, &client.BaseAddr, "pss_baseAddr") if err != nil { return nil, fmt.Errorf("cannot get pss node baseaddress: %v", err) } return client, nil } -func newClient(ctx context.Context) (client *Client) { - if ctx == nil { - ctx = context.Background() - } +func newClient() (client *Client) { client = &Client{ msgC: make(chan pss.APIMsg), quitC: make(chan struct{}), peerPool: make(map[pss.Topic]map[pot.Address]*pssRPCRW), protos: make(map[pss.Topic]*p2p.Protocol), - ctx: ctx, } return } @@ -140,7 +138,8 @@ func (self *Client) RunProtocol(proto *p2p.Protocol) error { topic := pss.NewTopic(proto.Name, int(proto.Version)) msgC := make(chan pss.APIMsg) self.peerPool[topic] = make(map[pot.Address]*pssRPCRW) - sub, err := self.rpc.Subscribe(self.ctx, "pss", msgC, "receive", topic) + ctx, _ := context.WithTimeout(context.Background(), rpcTimeout) + sub, err := self.rpc.Subscribe(ctx, "pss", msgC, "receive", topic) if err != nil { return fmt.Errorf("pss event subscription failed: %v", err) } @@ -174,7 +173,6 @@ func (self *Client) RunProtocol(proto *p2p.Protocol) error { // Always call this to ensure that we exit cleanly func (self *Client) Stop() error { - self.cancel() return nil } diff --git a/swarm/pss/client/client_test.go b/swarm/pss/client/client_test.go index bcb6c16775..b71ac30727 100644 --- a/swarm/pss/client/client_test.go +++ b/swarm/pss/client/client_test.go @@ -69,11 +69,7 @@ func TestIncoming(t *testing.T) { ps.Process(&pssmsg) - select { - case <-client.ctx.Done(): - t.Fatalf("outgoing timed out or canceled") - case <-ping.C: - } + <-ping.C quitC <- struct{}{} } @@ -107,18 +103,14 @@ func TestOutgoing(t *testing.T) { p := p2p.NewPeer(nid, fmt.Sprintf("%v", potaddr), []p2p.Cap{}) pp := protocols.NewPeer(p, client.peerPool[topic][potaddr], pss.PingProtocol) pp.Send(msg) - select { - case <-client.ctx.Done(): - t.Fatalf("outgoing timed out or canceled") - case <-ping.C: - } + <-ping.C quitC <- struct{}{} } func baseTester(t *testing.T, proto *p2p.Protocol, ps *pss.Pss, ctx context.Context, quitC chan struct{}) (*Client, error) { var err error - client := newTestclient(t, ctx, quitC) + client := newTestclient(t, quitC) err = client.RunProtocol(proto) @@ -143,7 +135,7 @@ func newProtocol(ping *pss.Ping) *p2p.Protocol { } } -func newTestclient(t *testing.T, ctx context.Context, quitC chan struct{}) *Client { +func newTestclient(t *testing.T, quitC chan struct{}) *Client { ps := pss.NewTestPss(nil) srv := rpc.NewServer() @@ -166,7 +158,7 @@ func newTestclient(t *testing.T, ctx context.Context, quitC chan struct{}) *Clie sock.Close() }() - pssclient, err := NewClient(ctx, "ws://localhost:8546") + pssclient, err := NewClient("ws://localhost:8546") if err != nil { t.Fatalf(err.Error()) } diff --git a/swarm/pss/simulations/service_test.go b/swarm/pss/simulations/service_test.go index 4ea40bab12..95a21eecae 100644 --- a/swarm/pss/simulations/service_test.go +++ b/swarm/pss/simulations/service_test.go @@ -139,13 +139,13 @@ func (t *testWrapper) newTestService(ctx *adapters.ServiceContext) (node.Service if err != nil { panic(err) } - pssclient, err := client.NewClientWithRPC(context.Background(), rpcClient) + pssclient, err := client.NewClientWithRPC(rpcClient) if err != nil { return nil, err } return &testService{ id: ctx.Config.ID, - pss: pssclient, + pss: pssclient, handshakes: make(chan *testHandshake), }, nil } From 8f141465f27a44d3437a7c41c7221817993a8782 Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 22 Jun 2017 18:02:04 +0200 Subject: [PATCH 15/17] swarm/pss: added context to runprotocol --- swarm/pss/client/client.go | 11 +++-------- swarm/pss/client/client_test.go | 2 +- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/swarm/pss/client/client.go b/swarm/pss/client/client.go index 68ad6cdd89..661978eb11 100644 --- a/swarm/pss/client/client.go +++ b/swarm/pss/client/client.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "sync" - "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" @@ -18,7 +17,6 @@ import ( ) const ( - rpcTimeout = time.Second inboxCapacity = 3000 outboxCapacity = 100 addrLen = common.HashLength @@ -85,8 +83,7 @@ func (rw *pssRPCRW) WriteMsg(msg p2p.Msg) error { return err } - ctx, _ := context.WithTimeout(context.Background(), rpcTimeout) - return rw.Client.rpc.CallContext(ctx, nil, "pss_send", rw.topic, pss.APIMsg{ + return rw.Client.rpc.Call(nil, "pss_send", rw.topic, pss.APIMsg{ Addr: rw.addr.Bytes(), Msg: pmsg, }) @@ -111,8 +108,7 @@ func NewClient(rpcurl string) (*Client, error) { func NewClientWithRPC(rpcclient *rpc.Client) (*Client, error) { client := newClient() client.rpc = rpcclient - ctx, _ := context.WithTimeout(context.Background(), rpcTimeout) - err := client.rpc.CallContext(ctx, &client.BaseAddr, "pss_baseAddr") + err := client.rpc.Call(&client.BaseAddr, "pss_baseAddr") if err != nil { return nil, fmt.Errorf("cannot get pss node baseaddress: %v", err) } @@ -134,11 +130,10 @@ func newClient() (client *Client) { // uses normal devp2p Send and incoming message handler routines from the p2p/protocols package // // when an incoming message is received from a peer that is not yet known to the client, this peer object is instantiated, and the protocol is run on it. -func (self *Client) RunProtocol(proto *p2p.Protocol) error { +func (self *Client) RunProtocol(ctx context.Context, proto *p2p.Protocol) error { topic := pss.NewTopic(proto.Name, int(proto.Version)) msgC := make(chan pss.APIMsg) self.peerPool[topic] = make(map[pot.Address]*pssRPCRW) - ctx, _ := context.WithTimeout(context.Background(), rpcTimeout) sub, err := self.rpc.Subscribe(ctx, "pss", msgC, "receive", topic) if err != nil { return fmt.Errorf("pss event subscription failed: %v", err) diff --git a/swarm/pss/client/client_test.go b/swarm/pss/client/client_test.go index b71ac30727..e758cbb68a 100644 --- a/swarm/pss/client/client_test.go +++ b/swarm/pss/client/client_test.go @@ -112,7 +112,7 @@ func baseTester(t *testing.T, proto *p2p.Protocol, ps *pss.Pss, ctx context.Cont client := newTestclient(t, quitC) - err = client.RunProtocol(proto) + err = client.RunProtocol(context.Background(), proto) if err != nil { return nil, err From 7ab9a617ebeba7c3d7cde7272c63d8e396843432 Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 23 Jun 2017 06:51:30 +0200 Subject: [PATCH 16/17] swarm, cmd/swarm: PR cleanup, rm obsolete flags --- cmd/swarm/main.go | 2 -- swarm/swarm.go | 10 ++++------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 64b26767a9..1818d05194 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -265,9 +265,7 @@ Cleans database of corrupted entries. SwarmUpFromStdinFlag, SwarmUploadMimeType, // pss flags - PssHostFlag, PssEnabledFlag, - PssPortFlag, } app.Flags = append(app.Flags, debug.Flags...) app.Before = func(ctx *cli.Context) error { diff --git a/swarm/swarm.go b/swarm/swarm.go index 5f2e154696..489e8e53ff 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -28,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/contracts/chequebook" "github.com/ethereum/go-ethereum/contracts/ens" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -41,7 +40,6 @@ import ( "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/pss" "github.com/ethereum/go-ethereum/swarm/storage" - //psschat "github.com/ethereum/go-ethereum/swarm/pss/protocols/chat" ) // the swarm stack @@ -241,9 +239,9 @@ func (self *Swarm) Start(net *p2p.Server) error { func (self *Swarm) Stop() error { self.dpa.Stop() self.bzz.Stop() - //if self.pss != nil { - // self.pss.Stop() - //} + if self.pss != nil { + self.pss.Stop() + } if ch := self.config.Swap.Chequebook(); ch != nil { ch.Stop() ch.Save() @@ -356,7 +354,7 @@ func NewLocalSwarm(datadir, port string) (self *Swarm, err error) { return } - config, err := api.NewConfig(datadir, common.Address{}, prvKey, network.NetworkId) + config, err := api.NewConfig(datadir, common.Address{}, prvKey, network.NetworkID) if err != nil { return } From 0628dbaab98a8759ec892dfe352530d83451de0d Mon Sep 17 00:00:00 2001 From: lash Date: Sat, 24 Jun 2017 08:07:57 +0200 Subject: [PATCH 17/17] cmd/swarm: added websocket capability (for pss) --- cmd/swarm/main.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 1818d05194..705e81a389 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -267,6 +267,14 @@ Cleans database of corrupted entries. // pss flags PssEnabledFlag, } + rpcFlags := []cli.Flag{ + utils.WSEnabledFlag, + utils.WSListenAddrFlag, + utils.WSPortFlag, + utils.WSApiFlag, + utils.WSAllowedOriginsFlag, + } + app.Flags = append(app.Flags, rpcFlags...) app.Flags = append(app.Flags, debug.Flags...) app.Before = func(ctx *cli.Context) error { runtime.GOMAXPROCS(runtime.NumCPU()) @@ -301,6 +309,9 @@ func version(ctx *cli.Context) error { func bzzd(ctx *cli.Context) error { cfg := defaultNodeConfig + if ctx.GlobalBool(PssEnabledFlag.Name) && ctx.GlobalBool(utils.WSEnabledFlag.Name) { + cfg.WSModules = append(cfg.WSModules, "pss") + } utils.SetNodeConfig(ctx, &cfg) stack, err := node.New(&cfg) if err != nil {