forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-27 15:16:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

cmd/swarm, swarm/pss: pss snap, docs, pot dup, kad hang

Browse source 
WIP introduce snapshots in pss
cmd can now set websocket host for pss
removed binary log garbage
moved baseaddr get to main pss api
changed pssclient constructor sig to return error
documentation
This commit is contained in:
nolash 2017-06-10 01:12:40 +02:00 committed by Lewis Marshall
parent a63fec8096
commit 1b6f480eba
14 changed files with 955 additions and 376 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
cmd/swarm/main.go
View file

@ -121,6 +121,11 @@ var (
Name: "pss",
Usage: "Enable pss (message passing over swarm)",
}
PssHostFlag = cli.StringFlag{
Name: "psshost",
Usage: fmt.Sprintf("Websockets host or ip for pss (default '%v')", node.DefaultWSHost),
Value: node.DefaultWSHost,
}
PssPortFlag = cli.IntFlag{
Name: "pssport",
Usage: fmt.Sprintf("Websockets port for pss (default %d)", node.DefaultWSPort),
@ -269,6 +274,7 @@ Cleans database of corrupted entries.
SwarmUpFromStdinFlag,
SwarmUploadMimeType,
// pss flags
PssHostFlag,
PssEnabledFlag,
PssPortFlag,
}
@ -307,7 +313,7 @@ func version(ctx *cli.Context) error {
func bzzd(ctx *cli.Context) error {
cfg := defaultNodeConfig
if ctx.GlobalIsSet(PssEnabledFlag.Name) {
cfg.WSHost = "127.0.0.1"
cfg.WSHost = ctx.GlobalString(PssHostFlag.Name)
cfg.WSModules = []string{"eth","pss"}
cfg.WSOrigins = []string{"*"}
if ctx.GlobalIsSet(PssPortFlag.Name) {
@ -366,7 +372,7 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) {
swapEnabled := ctx.GlobalBool(SwarmSwapEnabledFlag.Name)
syncEnabled := ctx.GlobalBoolT(SwarmSyncEnabledFlag.Name)
pssEnabled := ctx.GlobalBool(PssEnabledFlag.Name)
ethapi := ctx.GlobalString(EthAPIFlag.Name)
cors := ctx.GlobalString(CorsStringFlag.Name)

8
swarm/network/simulations/discovery/discovery_test.go
View file

@ -2,6 +2,7 @@ package discovery_test
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"os"
@ -135,6 +136,13 @@ func testDiscoverySimulation(t *testing.T, adapter adapters.NodeAdapter) {
t.Fatalf("simulation failed: %s", result.Error)
}
snap, err := net.Snapshot()
if err != nil {
t.Fatalf("no shapshot dude")
}
jsonsnapshot, err := json.Marshal(snap)
ioutil.WriteFile("jsonsnapshot.txt", jsonsnapshot, os.ModePerm)
t.Log("Simulation Passed:")
t.Logf("Duration: %s", result.FinishedAt.Sub(result.StartedAt))
for _, id := range ids {

185
swarm/pss/README.md Normal file
View file

@ -0,0 +1,185 @@
# Postal Service over Swarm
pss provides devp2p functionality for swarm nodes without the need for a direct tcp connection between them.
It uses swarm kademlia routing to send and receive messages. Routing is deterministic and will seek the shortest route available on the network.
Messages are encapsulated in a devp2p message structure `PssMsg`. These capsules are forwarded from node to node using ordinary tcp devp2p until it reaches it's destination. The destination address is hinted in `PssMsg.To`
The content of a PssMsg can be anything at all, down to a simple, non-descript byte-slices. But convenience methods are made available to implement devp2p protocol functionality on top of it.
In its final implementation, pss is intended to become "shh over bzz," that is; "whisper over swarm." Specifically, this means that the emphemeral encryption envelopes of whisper will be used to obfuscate the correspondance. Ideally, the unencrypted content of the PssMsg will only contain a part of the address of the recipient, where the final recipient is the one who matches this partial address *and* successfully can encrypt the message.
For the current state and roadmap of pss development please see https://github.com/ethersphere/swarm/wiki/swarm-dev-progress.
Please report issues on https://github.com/ethersphere/go-ethereum
Feel free to ask questions in https://gitter.im/ethersphere/pss
## TL;DR IMPLEMENTATION
Most developers will most probably want to use the protocol-wrapping convenience client in swarm/pss/client. Documentation and a minimal code example for the latter is found in the package documentation. The pss API can of course also be used directly. The client implementation provides a clear illustration of its intended usage.
pss implements the node.Service interface. This means that the API methods will be auto-magically exposed to any RPC layer the node activates. In particular, pss provides subscription to incoming messages using the go-ethereum rpc websocket layer.
The important API methods are:
- Receive() - start a subscription to receive new incoming messages matching specific "topics"
- Send() - send content over pss to a specified recipient
## LOWLEVEL IMPLEMENTATION
code speaks louder than words:
import (
"io/ioutil"
"os"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/pss"
"github.com/ethereum/go-ethereum/swarm/network"
"github.com/ethereum/go-ethereum/swarm/storage"
)
var (
righttopic = pss.NewTopic("foo", 4)
wrongtopic = pss.NewTopic("bar", 2)
)
// if you want to see what's going on
func init() {
hs := log.StreamHandler(os.Stderr, log.TerminalFormat(true))
hf := log.LvlFilterHandler(log.LvlTrace, hs)
h := log.CallerFileHandler(hf)
log.Root().SetHandler(h)
}
// Pss.Handler type
func handler(msg []byte, p *p2p.Peer, from []byte) error {
log.Debug("received", "msg", msg, "from", from, "forwarder", p.ID())
return nil
}
func implementation() {
// bogus addresses for illustration purposes
meaddr := network.RandomAddr()
toaddr := network.RandomAddr()
fwdaddr := network.RandomAddr()
// new kademlia for routing
kp := network.NewKadParams()
to := network.NewKademlia(meaddr.Over(), kp)
// new (local) storage for cache
cachedir, err := ioutil.TempDir("", "pss-cache")
if err != nil {
panic("overlay")
}
dpa, err := storage.NewLocalDPA(cachedir)
if err != nil {
panic("storage")
}
// setup pss
psp := pss.NewPssParams(false)
ps := pss.NewPss(to, dpa, psp)
// does nothing but please include it
ps.Start(nil)
dereg := ps.Register(&righttopic, handler)
// in its simplest form a message is just a byteslice
payload := []byte("foobar")
// send a raw message
err = ps.SendRaw(toaddr.Over(), righttopic, payload)
log.Error("Fails. Not connect, so nothing in kademlia. But it illustrates the point.", "err", err)
// forward a full message
envfwd := pss.NewEnvelope(fwdaddr.Over(), righttopic, payload)
msgfwd := &pss.PssMsg{
To: toaddr.Over(),
Payload: envfwd,
}
err = ps.Forward(msgfwd)
log.Error("Also fails, same reason. I wish, I wish, I wish there was somebody out there.", "err", err)
// process an incoming message
// (this is the first step after the devp2p PssMsg message handler)
envme := pss.NewEnvelope(toaddr.Over(), righttopic, payload)
msgme := &pss.PssMsg{
To: meaddr.Over(),
Payload: envme,
}
err = ps.Process(msgme)
if err == nil {
log.Info("this works :)")
}
// if we don't have a registered topic it fails
dereg() // remove the previously registered topic-handler link
ps.Process(msgme)
log.Error("It fails as we expected", "err", err)
// does nothing but please include it
ps.Stop()
}
## MESSAGE STRUCTURE
NOTE! This part is subject to change. In particular the envelope structure will be re-implemented using whisper.
A pss message has the following layers:
- PssMsg
Contains (eventually only part of) recipient address, and (eventually) encrypted Envelope.
- Envelope
Currently rlp-encoded. Contains the Payload, along with sender address, topic and expiry information.
- Payload
Byte-slice of arbitrary data
- ProtocolMsg
An optional convenience structure for implementation of devp2p protocols. Contains Code, Size and Payload analogous to the p2p.Msg structure, where the payload is a rlp-encoded byteslice. For transport, this struct is serialized and used as the "payload" above.
## TOPICS AND PROTOCOLS
Pure pss is protocol agnostic. Instead it uses the notion of Topic. This is NOT the "subject" of a message. Instead this type is used to internally register handlers for messages matching respective Topics.
Topic in this context virtually mean anything; protocols, chatrooms, or social media groups.
When implementing devp2p protocols, topics are direct mappings to protocols name and version. The pss package provides the PssProtocol convenience structure, and a generic Handler that can be passed to Pss.Register. This makes it possible to use the same message handler code for pss that are used for direct connected peers.
## CONNECTIONS
A "connection" in pss is a purely virtual construct. There is no mechanisms in place to ensure that the remote peer actually is there. In fact, "adding" a peer involves merely the node's opinion that the peer is there. It may issue messages to that remote peer to a directly connected peer, which in turn passes it on. But if it is not present on the network - or if there is no route to it - the message will never reach its destination through mere forwarding.
When implementing the devp2p protocol stack, the "adding" of a remote peer is a prerequisite for the side actually initiating the protocol communication. Adding a peer in effect "runs" the protocol on that peer, and adds an internal mapping between a topic and that peer. It also enables sending and receiving messages using the main io-construct in devp2p - the p2p.MsgReadWriter.
Under the hood, pss implements its own MsgReadWriter, which bridges MsgReadWriter.WriteMsg with Pss.SendRaw, and deftly adds an InjectMsg method which pipes incoming messages to appear on the MsgReadWriter.ReadMsg channel.
An incoming connection is nothing more than an actual PssMsg appearing with a certain Topic. If a Handler har been registered to that Topic, the message will be passed to it. This constitutes a "new" connection if:
- The pss node never called AddPeer with this combination of remote peer address and topic, and
- The pss node never received a PssMsg from this remote peer with this specific Topic before.
If it is a "new" connection, the protocol will be "run" on the remote peer, in the same manner as if it was pre-emptively added.
## ROUTING AND CACHING
(please refer to swarm kademlia routing for an explanation of the routing algorithm used for pss)
pss implements a simple caching mechanism, using the swarm DPA for storage of the messages and generation of the digest keys used in the cache table. The caching is intended to alleviate the following:
- save messages so that they can be delivered later if the recipient was not online at the time of sending.
- drop an identical message to the same recipient if received within a given time interval
- prevent backwards routing of messages
the latter may occur if only one entry is in the receiving node's kademlia. In this case the forwarder will be provided as the "nearest node" to the final recipient. The cache keeps the address of who the message was forwarded from, and if the cache lookup matches, the message will be dropped.

83
swarm/pss/client/README.md Normal file
View file

@ -0,0 +1,83 @@
# pss client
simple abstraction for implementing pss functionality
the pss client library aims to simplify usage of the p2p.protocols package over pss
IO is performed using the ordinary p2p.MsgReadWriter interface, which transparently communicates with a pss node via RPC using websockets as transport layer, using methods in the PssAPI class in the swarm/pss package
## USAGE
Minimal-ish usage example. It needs a running pss-enabled swarm node to work. Please refer to the test files for more details.
import (
"context"
"fmt"
"os"
pss "github.com/ethereum/go-ethereum/swarm/pss/client"
"github.com/ethereum/go-ethereum/p2p/protocols"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/pot"
"github.com/ethereum/go-ethereum/log"
)
type FooMsg struct {
Bar int
}
func fooHandler (msg interface{}) error {
foomsg, ok := msg.(*FooMsg)
if ok {
log.Debug("Yay, just got a message", "msg", foomsg)
}
return fmt.Errorf("Unknown message")
}
spec := &protocols.Spec{
Name: "foo",
Version: 1,
MaxMsgSize: 1024,
Messages: []interface{}{
FooMsg{},
},
}
proto := &p2p.Protocol{
Name: spec.Name,
Version: spec.Version,
Length: uint64(len(spec.Messages)),
Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error {
pp := protocols.NewPeer(p, rw, spec)
return pp.Run(fooHandler)
},
}
func implementation() {
cfg := pss.NewClientConfig()
psc := pss.NewClient(context.Background(), nil, cfg)
err := psc.Start()
if err != nil {
log.Crit("can't start pss client")
os.Exit(1)
}
log.Debug("connected to pss node", "bzz addr", psc.BaseAddr)
err = psc.RunProtocol(proto)
if err != nil {
log.Crit("can't start protocol on pss websocket")
os.Exit(1)
}
addr := pot.RandomAddress() // should be a real address, of course
psc.AddPssPeer(addr, spec)
// use the protocol for something
psc.Stop()
}
BUG(test): TestIncoming test times out due to deadlock issues in the swarm hive

160
swarm/pss/client/client.go
View file

@ -1,3 +1,82 @@
// simple abstraction for implementing pss functionality
//
// the pss client library aims to simplify usage of the p2p.protocols package over pss
//
// IO is performed using the ordinary p2p.MsgReadWriter interface, which transparently communicates with a pss node via RPC using websockets as transport layer, using methods in the PssAPI class in the swarm/pss package
//
//
// Minimal-ish usage example (requires a running pss node with websocket RPC):
//
//
// import (
// "context"
// "fmt"
// "os"
// pss "github.com/ethereum/go-ethereum/swarm/pss/client"
// "github.com/ethereum/go-ethereum/p2p/protocols"
// "github.com/ethereum/go-ethereum/p2p"
// "github.com/ethereum/go-ethereum/pot"
// "github.com/ethereum/go-ethereum/log"
// )
//
// type FooMsg struct {
// Bar int
// }
//
//
// func fooHandler (msg interface{}) error {
// foomsg, ok := msg.(*FooMsg)
// if ok {
// log.Debug("Yay, just got a message", "msg", foomsg)
// }
// return fmt.Errorf("Unknown message")
// }
//
// spec := &protocols.Spec{
// Name: "foo",
// Version: 1,
// MaxMsgSize: 1024,
// Messages: []interface{}{
// FooMsg{},
// },
// }
//
// proto := &p2p.Protocol{
// Name: spec.Name,
// Version: spec.Version,
// Length: uint64(len(spec.Messages)),
// Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error {
// pp := protocols.NewPeer(p, rw, spec)
// return pp.Run(fooHandler)
// },
// }
//
// func implementation() {
// cfg := pss.NewClientConfig()
// psc := pss.NewClient(context.Background(), nil, cfg)
// err := psc.Start()
// if err != nil {
// log.Crit("can't start pss client")
// os.Exit(1)
// }
//
// log.Debug("connected to pss node", "bzz addr", psc.BaseAddr)
//
// err = psc.RunProtocol(proto)
// if err != nil {
// log.Crit("can't start protocol on pss websocket")
// os.Exit(1)
// }
//
// addr := pot.RandomAddress() // should be a real address, of course
// psc.AddPssPeer(addr, spec)
//
// // use the protocol for something
//
// psc.Stop()
// }
//
// BUG(test): TestIncoming test times out due to deadlock issues in the swarm hive
package client
import (
@ -6,12 +85,12 @@ import (
"sync"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/discover"
"github.com/ethereum/go-ethereum/p2p/protocols"
"github.com/ethereum/go-ethereum/pot"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/swarm/pss"
@ -26,24 +105,27 @@ const (
// RemoteHost: hostname of node running websockets proxy to pss (default localhost)
// RemotePort: port of node running websockets proxy to pss (0 = go-ethereum node default)
// Secure: whether or not to use secure connection
// SelfHost: local if host to connect from
// Secure: whether or not to use secure connection (not currently in use)
type ClientConfig struct {
SelfHost string
RemoteHost string
RemotePort int
SelfHost string
Secure bool
}
// Generates a pss client configuration with default values
func NewClientConfig() *ClientConfig {
return &ClientConfig{
SelfHost: "localhost",
RemoteHost: "localhost",
RemotePort: 8546,
SelfHost: node.DefaultWSHost,
RemoteHost: node.DefaultWSHost,
RemotePort: node.DefaultWSPort,
}
}
// After a successful connection with Client.Start, BaseAddr contains the swarm overlay address of the pss node
type Client struct {
BaseAddr []byte
localuri string
remoteuri string
ctx context.Context
@ -58,6 +140,7 @@ type Client struct {
protos map[pss.Topic]*p2p.Protocol
}
// implements p2p.MsgReadWriter
type pssRPCRW struct {
*Client
topic *pss.Topic
@ -97,13 +180,17 @@ func (rw *pssRPCRW) WriteMsg(msg p2p.Msg) error {
if err != nil {
return err
}
return rw.Client.ws.CallContext(rw.Client.ctx, nil, "pss_send", rw.topic, pss.APIMsg{
Addr: rw.addr.Bytes(),
Msg: pmsg,
})
}
func NewClient(ctx context.Context, cancel func(), config *ClientConfig) *Client {
// Constructor for production-environment clients
// Performs sanity checks on configuration paramters and gets everything ready to connect to pss node
func NewClient(ctx context.Context, cancel func(), config *ClientConfig) (*Client, error) {
prefix := "ws"
if ctx == nil {
@ -129,24 +216,34 @@ func NewClient(ctx context.Context, cancel func(), config *ClientConfig) *Client
pssc.remoteuri = fmt.Sprintf("%s://%s:%d", prefix, config.RemoteHost, config.RemotePort)
pssc.localuri = fmt.Sprintf("%s://%s", prefix, config.SelfHost)
return pssc
return pssc, nil
}
func NewClientWithRPC(ctx context.Context, client *rpc.Client) *Client {
// Constructor for test implementations
// The 'rpcclient' parameter allows passing a in-memory rpc client to act as the remote websocket RPC.
func NewClientWithRPC(ctx context.Context, rpcclient *rpc.Client) (*Client, error) {
var oaddr []byte
err := rpcclient.CallContext(ctx, &oaddr, "pss_baseAddr")
if err != nil {
return nil, fmt.Errorf("cannot get pss node baseaddress: %v", err)
}
return &Client{
msgC: make(chan pss.APIMsg),
quitC: make(chan struct{}),
peerPool: make(map[pss.Topic]map[pot.Address]*pssRPCRW),
protos: make(map[pss.Topic]*p2p.Protocol),
ws: client,
ws: rpcclient,
ctx: ctx,
}
BaseAddr: oaddr,
}, nil
}
func (self *Client) shutdown() {
self.cancel()
}
// Connects to the websockets RPC
// Retrieves the swarm overlay address from the pss node
func (self *Client) Start() error {
if self.ws != nil {
return nil
@ -157,11 +254,23 @@ func (self *Client) Start() error {
return fmt.Errorf("Couldnt dial pss websocket: %v", err)
}
var oaddr []byte
err = ws.CallContext(self.ctx, &oaddr, "pss_baseAddr")
if err != nil {
return err
}
self.ws = ws
self.BaseAddr = oaddr
return nil
}
// Mounts a new devp2p protcool on the pss connection
// the protocol is aliased as a "pss topic"
// uses normal devp2p Send and incoming message handler routines from the p2p/protocols package
//
// when an incoming message is received from a peer that is not yet known to the client, this peer object is instantiated, and the protocol is run on it.
func (self *Client) RunProtocol(proto *p2p.Protocol) error {
topic := pss.NewTopic(proto.Name, int(proto.Version))
msgC := make(chan pss.APIMsg)
@ -200,11 +309,13 @@ func (self *Client) RunProtocol(proto *p2p.Protocol) error {
return nil
}
// Always call this to ensure that we exit cleanly
func (self *Client) Stop() error {
self.cancel()
return nil
}
// Preemptively add a remote pss peer
func (self *Client) AddPssPeer(addr pot.Address, spec *protocols.Spec) {
topic := pss.NewTopic(spec.Name, int(spec.Version))
if self.peerPool[topic] == nil {
@ -219,31 +330,10 @@ func (self *Client) AddPssPeer(addr pot.Address, spec *protocols.Spec) {
}
}
// Remove a remote pss peer
//
// Note this doesn't actually currently drop the peer, but only remmoves the reference from the client's peer lookup table
func (self *Client) RemovePssPeer(addr pot.Address, spec *protocols.Spec) {
topic := pss.NewTopic(spec.Name, int(spec.Version))
delete(self.peerPool[topic], addr)
}
func (self *Client) SubscribeEvents(ch chan *p2p.PeerEvent) event.Subscription {
log.Error("PSS client handles events internally, use the read functions instead")
return nil
}
func (self *Client) PeerCount() int {
return len(self.peerPool)
}
func (self *Client) NodeInfo() *p2p.NodeInfo {
return nil
}
func (self *Client) PeersInfo() []*p2p.PeerInfo {
return nil
}
func (self *Client) AddPeer(node *discover.Node) {
log.Error("Cannot add peer in PSS with discover.Node, need swarm overlay address")
}
func (self *Client) RemovePeer(node *discover.Node) {
log.Error("Cannot remove peer in PSS with discover.Node, need swarm overlay address")
}

5
swarm/pss/client/client_test.go
View file

@ -152,7 +152,10 @@ func newClient(t *testing.T, ctx context.Context, cancel func(), quitC chan stru
conf := NewClientConfig()
pssclient := NewClient(ctx, cancel, conf)
pssclient, err := NewClient(ctx, cancel, conf)
if err != nil {
t.Fatalf(err.Error())
}
ps := pss.NewTestPss(nil)
srv := rpc.NewServer()

1
swarm/pss/common.go → swarm/pss/ping.go
View file

@ -27,6 +27,7 @@ func (self *Ping) PingHandler(msg interface{}) error {
return nil
}
// Sample protocol used for tests
var PingProtocol = &protocols.Spec{
Name: "psstest",
Version: 1,

98
swarm/pss/protocols/chat/protocol.go
View file

@ -1,98 +0,0 @@
package chat
import (
"time"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/protocols"
"github.com/ethereum/go-ethereum/swarm/pss"
"github.com/ethereum/go-ethereum/swarm/network"
)
const (
ESendFail = iota
)
var (
chatConnString = map[int]string{
ESendFail: "Send error",
}
)
type ChatMsg struct {
Serial uint64
Content []byte
Source string
}
type chatPing struct {
Created time.Time
Pong bool
}
type chatAck struct {
Seen time.Time
Serial uint64
}
var ChatProtocol = &protocols.Spec{
Name: "pssChat",
Version: 1,
MaxMsgSize: 1024,
Messages: []interface{}{
ChatMsg{}, chatPing{}, chatAck{},
},
}
var ChatTopic = pss.NewTopic(ChatProtocol.Name, int(ChatProtocol.Version))
type ChatConn struct {
Addr []byte
E int
}
func (c* ChatConn) Error() string {
return chatConnString[c.E]
}
type ChatCtrl struct {
Peer *protocols.Peer
OutC chan interface{}
ConnC chan ChatConn
inC chan *ChatMsg
oAddr []byte
pingTX int
pingRX int
pingLast int
}
func (self *ChatCtrl) chatHandler(msg interface{}) error {
Chatmsg, ok := msg.(*ChatMsg)
if ok {
if self.inC != nil {
self.inC <- Chatmsg
}
}
return nil
}
func New(inC chan *ChatMsg, connC chan ChatConn, injectfunc func(*ChatCtrl)) *p2p.Protocol {
//func New(inC chan *ChatMsg, outC chan interface{}, connC chan ChatConn) *p2p.Protocol {
chatctrl := &ChatCtrl{
inC: inC,
ConnC: connC,
}
return &p2p.Protocol{
Name: ChatProtocol.Name,
Version: ChatProtocol.Version,
Length: 3,
Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error {
peerid := p.ID()
pp := protocols.NewPeer(p, rw, ChatProtocol)
chatctrl.Peer = pp
chatctrl.oAddr = network.ToOverlayAddr(peerid[:])
injectfunc(chatctrl)
pp.Run(chatctrl.chatHandler)
return nil
},
}
}

285
swarm/pss/pss.go
View file

@ -1,3 +1,185 @@
// pss provides devp2p functionality for swarm nodes without the need for a direct tcp connection between them.
//
// It uses swarm kademlia routing to send and receive messages. Routing is deterministic and will seek the shortest route available on the network.
//
// Messages are encapsulated in a devp2p message structure `PssMsg`. These capsules are forwarded from node to node using ordinary tcp devp2p until it reaches it's destination. The destination address is hinted in `PssMsg.To`
//
// The content of a PssMsg can be anything at all, down to a simple, non-descript byte-slices. But convenience methods are made available to implement devp2p protocol functionality on top of it.
//
// In its final implementation, pss is intended to become "shh over bzz," that is; "whisper over swarm." Specifically, this means that the emphemeral encryption envelopes of whisper will be used to obfuscate the correspondance. Ideally, the unencrypted content of the PssMsg will only contain a part of the address of the recipient, where the final recipient is the one who matches this partial address *and* successfully can encrypt the message.
//
// For the current state and roadmap of pss development please see https://github.com/ethersphere/swarm/wiki/swarm-dev-progress.
//
// Please report issues on https://github.com/ethersphere/go-ethereum
//
// Feel free to ask questions in https://gitter.im/ethersphere/pss
//
// TLDR IMPLEMENTATION
//
// Most developers will most probably want to use the protocol-wrapping convenience client in swarm/pss/client. Documentation and a minimal code example for the latter is found in the package documentation. The pss API can of course also be used directly. The client implementation provides a clear illustration of its intended usage.
//
// pss implements the node.Service interface. This means that the API methods will be auto-magically exposed to any RPC layer the node activates. In particular, pss provides subscription to incoming messages using the go-ethereum rpc websocket layer.
//
// The important API methods are:
// - Receive() - start a subscription to receive new incoming messages matching specific "topics"
// - Send() - send content over pss to a specified recipient
//
//
// LOWLEVEL IMPLEMENTATION
//
// code speaks louder than words:
//
// import (
// "io/ioutil"
// "os"
// "github.com/ethereum/go-ethereum/p2p"
// "github.com/ethereum/go-ethereum/log"
// "github.com/ethereum/go-ethereum/swarm/pss"
// "github.com/ethereum/go-ethereum/swarm/network"
// "github.com/ethereum/go-ethereum/swarm/storage"
// )
//
// var (
// righttopic = pss.NewTopic("foo", 4)
// wrongtopic = pss.NewTopic("bar", 2)
// )
//
// func init() {
// hs := log.StreamHandler(os.Stderr, log.TerminalFormat(true))
// hf := log.LvlFilterHandler(log.LvlTrace, hs)
// h := log.CallerFileHandler(hf)
// log.Root().SetHandler(h)
// }
//
//
// // Pss.Handler type
// func handler(msg []byte, p *p2p.Peer, from []byte) error {
// log.Debug("received", "msg", msg, "from", from, "forwarder", p.ID())
// return nil
// }
//
// func implementation() {
//
// // bogus addresses for illustration purposes
// meaddr := network.RandomAddr()
// toaddr := network.RandomAddr()
// fwdaddr := network.RandomAddr()
//
// // new kademlia for routing
// kp := network.NewKadParams()
// to := network.NewKademlia(meaddr.Over(), kp)
//
// // new (local) storage for cache
// cachedir, err := ioutil.TempDir("", "pss-cache")
// if err != nil {
// panic("overlay")
// }
// dpa, err := storage.NewLocalDPA(cachedir)
// if err != nil {
// panic("storage")
// }
//
// // setup pss
// psp := pss.NewPssParams(false)
// ps := pss.NewPss(to, dpa, psp)
//
// // does nothing but please include it
// ps.Start(nil)
//
// dereg := ps.Register(&righttopic, handler)
//
// // in its simplest form a message is just a byteslice
// payload := []byte("foobar")
//
// // send a raw message
// err = ps.SendRaw(toaddr.Over(), righttopic, payload)
// log.Error("Fails. Not connect, so nothing in kademlia. But it illustrates the point.", "err", err)
//
// // forward a full message
// envfwd := pss.NewEnvelope(fwdaddr.Over(), righttopic, payload)
// msgfwd := &pss.PssMsg{
// To: toaddr.Over(),
// Payload: envfwd,
// }
// err = ps.Forward(msgfwd)
// log.Error("Also fails, same reason. I wish, I wish, I wish there was somebody out there.", "err", err)
//
// // process an incoming message
// // (this is the first step after the devp2p PssMsg message handler)
// envme := pss.NewEnvelope(toaddr.Over(), righttopic, payload)
// msgme := &pss.PssMsg{
// To: meaddr.Over(),
// Payload: envme,
// }
// err = ps.Process(msgme)
// if err == nil {
// log.Info("this works :)")
// }
//
// // if we don't have a registered topic it fails
// dereg() // remove the previously registered topic-handler link
// ps.Process(msgme)
// log.Error("It fails as we expected", "err", err)
//
// // does nothing but please include it
// ps.Stop()
// }
//
// MESSAGE STRUCTURE
//
// NOTE! This part is subject to change. In particular the envelope structure will be re-implemented using whisper.
//
// A pss message has the following layers:
//
// PssMsg
// Contains (eventually only part of) recipient address, and (eventually) encrypted Envelope.
//
// Envelope
// Currently rlp-encoded. Contains the Payload, along with sender address, topic and expiry information.
//
// Payload
// Byte-slice of arbitrary data
//
// ProtocolMsg
// An optional convenience structure for implementation of devp2p protocols. Contains Code, Size and Payload analogous to the p2p.Msg structure, where the payload is a rlp-encoded byteslice. For transport, this struct is serialized and used as the "payload" above.
//
// TOPICS AND PROTOCOLS
//
// Pure pss is protocol agnostic. Instead it uses the notion of Topic. This is NOT the "subject" of a message. Instead this type is used to internally register handlers for messages matching respective Topics.
//
// Topic in this context virtually mean anything; protocols, chatrooms, or social media groups.
//
// When implementing devp2p protocols, topics are direct mappings to protocols name and version. The pss package provides the PssProtocol convenience structure, and a generic Handler that can be passed to Pss.Register. This makes it possible to use the same message handler code for pss that are used for direct connected peers.
//
// CONNECTIONS
//
// A "connection" in pss is a purely virtual construct. There is no mechanisms in place to ensure that the remote peer actually is there. In fact, "adding" a peer involves merely the node's opinion that the peer is there. It may issue messages to that remote peer to a directly connected peer, which in turn passes it on. But if it is not present on the network - or if there is no route to it - the message will never reach its destination through mere forwarding.
//
// When implementing the devp2p protocol stack, the "adding" of a remote peer is a prerequisite for the side actually initiating the protocol communication. Adding a peer in effect "runs" the protocol on that peer, and adds an internal mapping between a topic and that peer. It also enables sending and receiving messages using the main io-construct in devp2p - the p2p.MsgReadWriter.
//
// Under the hood, pss implements its own MsgReadWriter, which bridges MsgReadWriter.WriteMsg with Pss.SendRaw, and deftly adds an InjectMsg method which pipes incoming messages to appear on the MsgReadWriter.ReadMsg channel.
//
// An incoming connection is nothing more than an actual PssMsg appearing with a certain Topic. If a Handler har been registered to that Topic, the message will be passed to it. This constitutes a "new" connection if:
//
// - The pss node never called AddPeer with this combination of remote peer address and topic, and
//
// - The pss node never received a PssMsg from this remote peer with this specific Topic before.
//
// If it is a "new" connection, the protocol will be "run" on the remote peer, in the same manner as if it was pre-emptively added.
//
// ROUTING AND CACHING
//
// (please refer to swarm kademlia routing for an explanation of the routing algorithm used for pss)
//
// pss implements a simple caching mechanism, using the swarm DPA for storage of the messages and generation of the digest keys used in the cache table. The caching is intended to alleviate the following:
//
// - save messages so that they can be delivered later if the recipient was not online at the time of sending.
//
// - drop an identical message to the same recipient if received within a given time interval
//
// - prevent backwards routing of messages
//
// the latter may occur if only one entry is in the receiving node's kademlia. In this case the forwarder will be provided as the "nearest node" to the final recipient. The cache keeps the address of who the message was forwarded from, and if the cache lookup matches, the message will be dropped.
package pss
import (
@ -20,23 +202,24 @@ import (
)
const (
TopicResolverLength = 8
PssPeerCapacity = 256
PssPeerTopicDefaultCapacity = 8
digestLength = 32
digestCapacity = 256
PssPeerCapacity = 256 // limit of peers kept in cache. (not implemented)
PssPeerTopicDefaultCapacity = 8 // limit of topics kept per peer. (not implemented)
digestLength = 32 // byte length of digest used for pss cache (currently same as swarm chunk hash)
digestCapacity = 256 // cache entry limit (not implement)
)
var (
errorForwardToSelf = errors.New("forward to self")
)
// abstraction to enable access to p2p.protocols.Peer.Send
type senderPeer interface {
ID() discover.NodeID
Address() []byte
Send(interface{}) error
}
// protocol specification of the pss capsule
var pssSpec = &protocols.Spec{
Name: "pss",
Version: 1,
@ -53,24 +236,12 @@ type pssCacheEntry struct {
type pssDigest [digestLength]byte
// implements node.Service
// Toplevel pss object, taking care of message sending and receiving, message handler dispatchers and message forwarding.
//
// pss provides sending messages to nodes without having to be directly connected to them.
//
// The messages are wrapped in a PssMsg structure and routed using the swarm kademlia routing.
//
// The top-level Pss object provides:
//
// - access to the swarm overlay and routing (kademlia)
// - a collection of remote overlay addresses mapped to MsgReadWriters, representing the virtually connected peers
// - a collection of remote underlay address, mapped to the overlay addresses above
// - a method to send a message to specific overlayaddr
// - a dispatcher lookup, mapping protocols to topics
// - a message cache to spot messages that previously have been forwarded
// Implements node.Service
type Pss struct {
network.Overlay // we can get the overlayaddress from this
peerPool map[pot.Address]map[Topic]p2p.MsgReadWriter // keep track of all virtual p2p.Peers we are currently speaking to
//fwdPool map[pot.Address]*protocols.Peer // keep track of all peers sitting on the pssmsg routing layer
fwdPool map[discover.NodeID]*protocols.Peer // keep track of all peers sitting on the pssmsg routing layer
handlers map[Topic]map[*Handler]bool // topic and version based pss payload handlers
fwdcache map[pssDigest]pssCacheEntry // checksum of unique fields from pssmsg mapped to expiry, cache to determine whether to drop msg
@ -83,7 +254,7 @@ type Pss struct {
func (self *Pss) storeMsg(msg *PssMsg) (pssDigest, error) {
swg := &sync.WaitGroup{}
wwg := &sync.WaitGroup{}
buf := bytes.NewReader(msg.Serialize())
buf := bytes.NewReader(msg.serialize())
key, err := self.dpa.Store(buf, int64(buf.Len()), swg, wwg)
if err != nil {
log.Warn("Could not store in swarm", "err", err)
@ -95,12 +266,13 @@ func (self *Pss) storeMsg(msg *PssMsg) (pssDigest, error) {
return digest, nil
}
// Creates a new Pss instance. A node should only need one of these
// Creates a new Pss instance.
//
// Needs a swarm network overlay, a DPA storage for message cache storage.
func NewPss(k network.Overlay, dpa *storage.DPA, params *PssParams) *Pss {
return &Pss{
Overlay: k,
peerPool: make(map[pot.Address]map[Topic]p2p.MsgReadWriter, PssPeerCapacity),
//fwdPool: make(map[pot.Address]*protocols.Peer),
fwdPool: make(map[discover.NodeID]*protocols.Peer),
handlers: make(map[Topic]map[*Handler]bool),
fwdcache: make(map[pssDigest]pssCacheEntry),
@ -110,18 +282,25 @@ func NewPss(k network.Overlay, dpa *storage.DPA, params *PssParams) *Pss {
}
}
// Convenience accessor to the swarm overlay address of the pss node
func (self *Pss) BaseAddr() []byte {
return self.Overlay.BaseAddr()
}
// For node.Service implementation. Does nothing for now, but should be included in the code for backwards compatibility.
func (self *Pss) Start(srv *p2p.Server) error {
return nil
}
// For node.Service implementation. Does nothing for now, but should be included in the code for backwards compatibility.
func (self *Pss) Stop() error {
return nil
}
// devp2p protocol object for the PssMsg struct.
//
// This represents the PssMsg capsule, and is the entry point for processing, receiving and sending pss messages between directly connected peers.
func (self *Pss) Protocols() []p2p.Protocol {
return []p2p.Protocol{
p2p.Protocol{
@ -133,16 +312,19 @@ func (self *Pss) Protocols() []p2p.Protocol {
}
}
// Starts the PssMsg protocol
func (self *Pss) Run(p *p2p.Peer, rw p2p.MsgReadWriter) error {
pp := protocols.NewPeer(p, rw, pssSpec)
//addr := network.NewAddrFromNodeID(id)
//potaddr := pot.NewHashAddressFromBytes(addr.OAddr)
//self.fwdPool[potaddr.Address] = pp
self.fwdPool[p.ID()] = pp
return pp.Run(self.handlePssMsg)
}
// Exposes the API methods
//
// If the debug-parameter was given to the top Pss object, the TestAPI methods will also be included
func (self *Pss) APIs() []rpc.API {
apis := []rpc.API{
rpc.API{
@ -163,12 +345,11 @@ func (self *Pss) APIs() []rpc.API {
return apis
}
// Takes the generated Topic of a protocol/chatroom etc, and links a handler function to it
// This allows the implementer to retrieve the right handler functions (invoke the right protocol)
// for an incoming message by inspecting the topic on it.
// a topic allows for multiple handlers
// returns a deregister function which needs to be called to deregister the handler
// (similar to event.Subscription.Unsubscribe())
// Links a handler function to a Topic
//
// After calling this, all incoming messages with an envelope Topic matching the Topic specified will be passed to the given Handler function.
//
// Returns a deregister function which needs to be called to deregister the handler,
func (self *Pss) Register(topic *Topic, handler Handler) func() {
self.lock.Lock()
defer self.lock.Unlock()
@ -192,10 +373,7 @@ func (self *Pss) deregister(topic *Topic, h *Handler) {
delete(handlers, h)
}
// enables to set address of node, to avoid backwards forwarding
//
// currently not in use as forwarder address is not known in the handler function hooked to the pss dispatcher.
// it is included as a courtesy to custom transport layers that may want to implement this
// Adds an address/message pair to the cache
func (self *Pss) AddToCache(addr []byte, msg *PssMsg) error {
digest, err := self.storeMsg(msg)
if err != nil {
@ -263,13 +441,13 @@ func (self *Pss) handlePssMsg(msg interface{}) error {
return self.Process(pssmsg)
}
// processes a message with self as recipient
// Entry point to processing a message for which the current node is the intended recipient.
func (self *Pss) Process(pssmsg *PssMsg) error {
env := pssmsg.Payload
payload := env.Payload
handlers := self.getHandlers(env.Topic)
if len(handlers) == 0 {
return fmt.Errorf("No registered handler for topic '%s'", env.Topic)
return fmt.Errorf("No registered handler for topic '%x'", env.Topic)
}
nid, _ := discover.HexID("0x00")
p := p2p.NewPeer(nid, fmt.Sprintf("%x", env.From), []p2p.Cap{})
@ -282,9 +460,11 @@ func (self *Pss) Process(pssmsg *PssMsg) error {
return nil
}
// Sends a message using The message could be anything at all, and will be handled by whichever handler function is mapped to Topic using *Pss.Register()
// Sends a message using Pss.
//
// The to address is a swarm overlay address
// This method is payload agnostic, and will accept any arbitrary byte slice as the payload for a message.
//
// It generates an envelope for the specified recipient and topic, and wraps the message payload in it.
func (self *Pss) SendRaw(to []byte, topic Topic, msg []byte) error {
sender := self.Overlay.BaseAddr()
pssenv := NewEnvelope(sender, topic, msg)
@ -297,18 +477,20 @@ func (self *Pss) SendRaw(to []byte, topic Topic, msg []byte) error {
// Forwards a pss message to the peer(s) closest to the to address
//
// Handlers that want to pass on a message should call this directly
// Handlers that are merely passing on the PssMsg to its final recipient should call this directly
func (self *Pss) Forward(msg *PssMsg) error {
if self.isSelfRecipient(msg) {
return errorForwardToSelf
}
// cache it
digest, err := self.storeMsg(msg)
if err != nil {
log.Warn(fmt.Sprintf("could not store message %v to cache: %v", msg, err))
}
// flood guard
if self.checkFwdCache(nil, digest) {
log.Trace(fmt.Sprintf("pss relay block-cache match: FROM %x TO %x", common.ByteLabel(self.Overlay.BaseAddr()), common.ByteLabel(msg.To)))
return nil
@ -326,15 +508,12 @@ func (self *Pss) Forward(msg *PssMsg) error {
return false
}
sendMsg := fmt.Sprintf("TO %x FROM %x VIA %x", common.ByteLabel(msg.To), common.ByteLabel(self.BaseAddr()), common.ByteLabel(op.Address()))
//h := pot.NewHashAddressFromBytes(op.Address())
//pp := self.fwdPool[h.Address]
pp := self.fwdPool[sp.ID()]
if self.checkFwdCache(op.Address(), digest) {
log.Info("%v: peer already forwarded to", sendMsg)
return true
}
err := pp.Send(msg)
//err := sp.Send(msg)
if err != nil {
log.Warn(fmt.Sprintf("%v: failed forwarding: %v", sendMsg, err))
return true
@ -351,14 +530,16 @@ func (self *Pss) Forward(msg *PssMsg) error {
if sent == 0 {
log.Error("PSS: unable to forward to any peers")
return nil
return fmt.Errorf("unable to forward to any peers")
}
self.addFwdCacheExpire(digest)
return nil
}
// Links a pss peer address and topic to a dedicated p2p.MsgReadWriter in the pss peerpool, and runs the specificed protocol on this p2p.MsgReadWriter and the specified peer
// For devp2p protocol integration only. Analogous to an outgoing devp2p connection.
//
// Links a remote peer and Topic to a dedicated p2p.MsgReadWriter in the pss peerpool, and runs the specificed protocol using these resources.
//
// The effect is that now we have a "virtual" protocol running on an artificial p2p.Peer, which can be looked up and piped to through Pss using swarm overlay address and topic
func (self *Pss) AddPeer(p *p2p.Peer, addr pot.Address, run func(*p2p.Peer, p2p.MsgReadWriter) error, topic Topic, rw p2p.MsgReadWriter) error {
@ -403,10 +584,9 @@ func (self *Pss) isActive(id pot.Address, topic Topic) bool {
return self.peerPool[id][topic] != nil
}
// Convenience object that:
// For devp2p protocol integration only.
//
// - allows passing of the unwrapped PssMsg payload to the p2p level message handlers
// - interprets outgoing p2p.Msg from the p2p level to pass in to *Pss.Send()
// Bridges pss send/receive with devp2p protocol send/receive
//
// Implements p2p.MsgReadWriter
type PssReadWriter struct {
@ -448,6 +628,9 @@ func (prw PssReadWriter) injectMsg(msg p2p.Msg) error {
return nil
}
// For devp2p protocol integration only.
//
// Convenience object for passing messages in and out of the p2p layer
type PssProtocol struct {
*Pss
@ -456,7 +639,9 @@ type PssProtocol struct {
spec *protocols.Spec
}
// Constructor
// For devp2p protocol integration only.
//
// Maps a Topic to a devp2p protocol.
func RegisterPssProtocol(ps *Pss, topic *Topic, spec *protocols.Spec, targetprotocol *p2p.Protocol) *PssProtocol {
pp := &PssProtocol{
Pss: ps,
@ -467,8 +652,12 @@ func RegisterPssProtocol(ps *Pss, topic *Topic, spec *protocols.Spec, targetprot
return pp
}
// For devp2p protocol integration only.
//
// Generic handler for initiating devp2p-like protocol connections
//
// This handler should be passed to Pss.Register with the associated ropic.
func (self *PssProtocol) Handle(msg []byte, p *p2p.Peer, senderAddr []byte) error {
//hashoaddr := pot.NewHashAddressFromBytes(senderAddr).Address
hashoaddr := pot.NewAddressFromBytes(senderAddr)
if !self.isActive(hashoaddr, *self.topic) {
rw := &PssReadWriter{

340
swarm/pss/pss_test.go
View file

@ -4,14 +4,17 @@ import (
"bytes"
"context"
"encoding/hex"
"encoding/json"
"fmt"
"io/ioutil"
"math/rand"
"os"
"sync"
"testing"
"time"
"flag"
"github.com/ethereum/go-ethereum/common"
// "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/p2p"
@ -20,7 +23,6 @@ import (
"github.com/ethereum/go-ethereum/p2p/simulations"
"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
p2ptest "github.com/ethereum/go-ethereum/p2p/testing"
"github.com/ethereum/go-ethereum/pot"
"github.com/ethereum/go-ethereum/swarm/network"
"github.com/ethereum/go-ethereum/swarm/storage"
)
@ -30,6 +32,10 @@ const (
bzzServiceName = "bzz"
)
var (
snapshotfile string
)
var services = newServices()
func init() {
@ -38,6 +44,8 @@ func init() {
hf := log.LvlFilterHandler(log.LvlTrace, hs)
h := log.CallerFileHandler(hf)
log.Root().SetHandler(h)
flag.StringVar(&snapshotfile, "file", "snapsnot.json", "snapshot file")
}
func TestCache(t *testing.T) {
@ -199,8 +207,9 @@ func TestSimpleLinear(t *testing.T) {
Peer: pp,
addr: network.ToOverlayAddr(id[:]),
}
a := pot.NewAddressFromBytes(bp.addr)
ps.fwdPool[a] = pp
//a := pot.NewAddressFromBytes(bp.addr)
//ps.fwdPool[a] = pp
ps.fwdPool[id] = pp
ps.Overlay.On(bp)
defer ps.Overlay.Off(bp)
log.Debug(fmt.Sprintf("%v", ps.Overlay))
@ -236,32 +245,56 @@ func TestSimpleLinear(t *testing.T) {
func TestFullRandom50n(t *testing.T) {
adapter := adapters.NewSimAdapter(services)
testFullRandom(t, adapter, 50, 50, 50)
testFullRandom(t, adapter, 50, 50)
}
func TestFullRandom25n(t *testing.T) {
adapter := adapters.NewSimAdapter(services)
testFullRandom(t, adapter, 25, 25, 25)
testFullRandom(t, adapter, 25, 25)
}
func TestFullRandom10n(t *testing.T) {
adapter := adapters.NewSimAdapter(services)
testFullRandom(t, adapter, 10, 10, 10)
testFullRandom(t, adapter, 10, 10)
}
func TestFullRandom5n(t *testing.T) {
adapter := adapters.NewSimAdapter(services)
testFullRandom(t, adapter, 5, 5, 5)
baseDir, err := ioutil.TempDir("", "swarm-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(baseDir)
adapter := adapters.NewExecAdapter(baseDir)
testFullRandom(t, adapter, 5, 5)
}
func testFullRandom(t *testing.T, adapter adapters.NodeAdapter, nodecount int, fullnodecount int, msgcount int) {
var i int
var msgfromids []discover.NodeID
func testFullRandom(t *testing.T, adapter adapters.NodeAdapter, nodecount int, msgcount int) {
}
func TestFullRandomSnapshot50(t *testing.T) {
testFullRandomSnapshot(t, true, 5)
}
func testFullRandomSnapshot(t *testing.T, sim bool, msgcount int) {
var msgtoids []discover.NodeID
var msgreceived []discover.NodeID
var cancelmain func()
var triggerptr *chan discover.NodeID
//var triggerptr *chan discover.NodeID
msgtoids := make([]discover.NodeID, msgcount)
baseDir, err := ioutil.TempDir("", "swarm-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(baseDir)
var adapter adapters.NodeAdapter
if sim {
adapter = adapters.NewSimAdapter(services)
} else {
adapter = adapters.NewExecAdapter(baseDir)
}
wg := sync.WaitGroup{}
wg.Add(msgcount)
@ -269,141 +302,187 @@ func testFullRandom(t *testing.T, adapter adapters.NodeAdapter, nodecount int, f
psslog := make(map[discover.NodeID]log.Logger)
psslogmain := log.New("psslog", "*")
jsonsnapshot, err := ioutil.ReadFile(snapshotfile)
if err != nil {
t.Fatalf("cant read snapshot: %s", snapshotfile)
}
snapshot := &simulations.Snapshot{}
err = json.Unmarshal(jsonsnapshot, snapshot)
net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{
ID: "0",
})
defer net.Shutdown()
err = net.Load(snapshot)
if err != nil {
t.Fatalf("invalid snapshot: %v", err)
}
//msgtoids := make([]discover.NodeID, msgcount)
timeout := 15 * time.Second
ctx, cancelmain := context.WithTimeout(context.Background(), timeout)
defer cancelmain()
trigger := make(chan discover.NodeID)
triggerptr = &trigger
//triggerptr = &trigger
ids := make([]discover.NodeID, nodecount)
fullids := ids[0:fullnodecount]
fullpeers := make(map[discover.NodeID][]byte)
recvaddrs := make(map[discover.NodeID][]byte)
for i = 0; i < nodecount; i++ {
nodeconfig := adapters.RandomNodeConfig()
nodeconfig.Services = []string{"bzz", "pss"}
node, err := net.NewNodeWithConfig(nodeconfig)
if err != nil {
t.Fatalf("error starting node: %s", err)
}
if err := net.Start(node.ID()); err != nil {
t.Fatalf("error starting node %s: %s", node.ID().TerminalString(), err)
}
if err := triggerChecks(ctx, &wg, triggerptr, net, node.ID()); err != nil {
t.Fatal("error triggering checks for node %s: %s", node.ID().TerminalString(), err)
}
ids[i] = node.ID()
if i < fullnodecount {
fullpeers[ids[i]] = network.ToOverlayAddr(node.ID().Bytes())
psslog[ids[i]] = log.New("psslog", fmt.Sprintf("%x", fullpeers[ids[i]]))
}
log.Debug("psslog starting node", "id", nodeconfig.ID)
}
for i, id := range fullids {
msgfromids = append(msgfromids, id)
msgtoids[i] = fullids[(i+(len(fullids)/2)+1)%len(fullids)]
}
// for i = 0; i < nodecount; i++ {
// nodeconfig := adapters.RandomNodeConfig()
// nodeconfig.Services = []string{"bzz", "pss"}
// node, err := net.NewNodeWithConfig(nodeconfig)
// if err != nil {
// t.Fatalf("error starting node: %s", err)
// }
//
// if err := net.Start(node.ID()); err != nil {
// t.Fatalf("error starting node %s: %s", node.ID().TerminalString(), err)
// }
//
// if err := triggerChecks(ctx, &wg, triggerptr, net, node.ID()); err != nil {
// t.Fatal("error triggering checks for node %s: %s", node.ID().TerminalString(), err)
// }
// ids[i] = node.ID()
// if i < fullnodecount {
// fullpeers[ids[i]] = network.ToOverlayAddr(node.ID().Bytes())
// psslog[ids[i]] = log.New("psslog", fmt.Sprintf("%x", fullpeers[ids[i]]))
// }
// log.Debug("psslog starting node", "id", nodeconfig.ID)
// }
//
// for i, id := range fullids {
// msgfromids = append(msgfromids, id)
// msgtoids[i] = fullids[(i+(len(fullids)/2)+1)%len(fullids)]
// }
// run a simulation which connects the 10 nodes in a ring and waits
// for full peer discovery
// action := func(ctx context.Context) error {
// for i, id := range ids {
// peerID := ids[(i+1)%len(ids)]
// if net.GetConn(id, peerID) != nil {
// continue
// }
// if err := net.Connect(id, peerID); err != nil {
// return err
// }
// psslog[id].Debug("conn ok", "one", id, "other", peerID)
// }
// return nil
// }
// check := func(ctx context.Context, id discover.NodeID) (bool, error) {
// select {
// case <-ctx.Done():
// wg.Done()
// psslog[id].Error("conn failed!", "id", id)
// return false, ctx.Err()
// default:
// }
// var tgt []byte
// var fwd struct {
// Addr []byte
// Count int
// }
//
// for i, fid := range msgfromids {
// if id == fid {
// tgt = network.ToOverlayAddr(msgtoids[(i+(len(msgtoids)/2)+1)%len(msgtoids)].Bytes())
// break
// }
// }
// p := net.GetNode(id)
// if p == nil {
// return false, fmt.Errorf("Unknown node: %v", id)
// }
// c, err := p.Client()
// if err != nil {
// return false, err
// }
// for fwd.Count < 2 {
// c.CallContext(context.Background(), &fwd, "pss_getForwarder", tgt)
// time.Sleep(time.Microsecond * 250)
// }
// psslog[id].Debug("fwd check ok", "topaddr", fmt.Sprintf("%x", common.ByteLabel(fwd.Addr)), "kadcount", fwd.Count)
// return true, nil
// }
//
// result := simulations.NewSimulation(net).Run(ctx, &simulations.Step{
// Action: action,
// Trigger: trigger,
// Expect: &simulations.Expectation{
// Nodes: ids,
// Check: check,
// },
// })
// if result.Error != nil {
// t.Fatalf("simulation failed: %s", result.Error)
// cancelmain()
// }
//
// trigger = make(chan discover.NodeID)
// triggerptr = &trigger
var ids []discover.NodeID
action := func(ctx context.Context) error {
for i, id := range ids {
peerID := ids[(i+1)%len(ids)]
if net.GetConn(id, peerID) != nil {
continue
var rpcerr error
var rpcbyte []byte
//for ii, id := range msgfromids {
for _, simnode := range net.Nodes {
ids = append(ids, simnode.ID())
//node := net.GetNode(id)
if simnode == nil {
return fmt.Errorf("unknown node: %s", simnode.ID())
}
if err := net.Connect(id, peerID); err != nil {
return err
client, err := simnode.Client()
if err != nil {
return fmt.Errorf("error getting recp node client: %s", err)
}
err = client.Call(&rpcbyte, "pss_baseAddr")
if err != nil {
t.Fatalf("cant get overlayaddr: %v", err)
}
recvaddrs[simnode.ID()] = rpcbyte
err = client.Call(&rpcbyte, "pss_baseAddr")
if err != nil {
t.Fatalf("cant get overlayaddr: %v", err)
}
}
for i := 0; i < msgcount; i++ {
idx := rand.Intn(len(net.Nodes))
sendernode := net.Nodes[idx]
toidx := rand.Intn(len(net.Nodes)-1)
if idx >= toidx {
toidx++
}
recvnode := net.Nodes[toidx]
msg := PingMsg{Created: time.Now()}
code, _ := PingProtocol.GetCode(&PingMsg{})
pmsg, _ := NewProtocolMsg(code, msg)
client, err := sendernode.Client()
if err != nil {
return fmt.Errorf("error getting sendernode client: %s", err)
}
client.CallContext(ctx, &rpcerr, "pss_send", PingTopic, APIMsg{
//Addr: fullpeers[msgtoids[ii]],
Addr: recvaddrs[recvnode.ID()],
Msg: pmsg,
})
if rpcerr != nil {
return fmt.Errorf("error rpc send id %x: %v", sendernode.ID(), rpcerr)
}
psslog[id].Debug("conn ok", "one", id, "other", peerID)
}
return nil
}
check := func(ctx context.Context, id discover.NodeID) (bool, error) {
var tgt []byte
var fwd struct {
Addr []byte
Count int
}
select {
case <-ctx.Done():
wg.Done()
psslog[id].Error("conn failed!", "id", id)
return false, ctx.Err()
default:
}
for i, fid := range msgfromids {
if id == fid {
tgt = network.ToOverlayAddr(msgtoids[(i+(len(msgtoids)/2)+1)%len(msgtoids)].Bytes())
break
}
}
p := net.GetNode(id)
if p == nil {
return false, fmt.Errorf("Unknown node: %v", id)
}
c, err := p.Client()
if err != nil {
return false, err
}
for fwd.Count < 2 {
c.CallContext(context.Background(), &fwd, "pss_getForwarder", tgt)
time.Sleep(time.Microsecond * 250)
}
psslog[id].Debug("fwd check ok", "topaddr", fmt.Sprintf("%x", common.ByteLabel(fwd.Addr)), "kadcount", fwd.Count)
return true, nil
}
result := simulations.NewSimulation(net).Run(ctx, &simulations.Step{
Action: action,
Trigger: trigger,
Expect: &simulations.Expectation{
Nodes: ids,
Check: check,
},
})
if result.Error != nil {
t.Fatalf("simulation failed: %s", result.Error)
cancelmain()
}
trigger = make(chan discover.NodeID)
triggerptr = &trigger
action = func(ctx context.Context) error {
var rpcerr error
for ii, id := range msgfromids {
node := net.GetNode(id)
if node == nil {
return fmt.Errorf("unknown node: %s", id)
}
client, err := node.Client()
if err != nil {
return fmt.Errorf("error getting node client: %s", err)
}
msg := PingMsg{Created: time.Now()}
code, _ := PingProtocol.GetCode(&PingMsg{})
pmsg, _ := NewProtocolMsg(code, msg)
client.CallContext(ctx, &rpcerr, "pss_send", PingTopic, APIMsg{
Addr: fullpeers[msgtoids[ii]],
Msg: pmsg,
})
if rpcerr != nil {
return fmt.Errorf("error rpc send id %x: %v", id, rpcerr)
}
}
return nil
}
check = func(ctx context.Context, id discover.NodeID) (bool, error) {
select {
case <-ctx.Done():
@ -417,11 +496,12 @@ func testFullRandom(t *testing.T, adapter adapters.NodeAdapter, nodecount int, f
return true, nil
}
result = simulations.NewSimulation(net).Run(ctx, &simulations.Step{
result := simulations.NewSimulation(net).Run(ctx, &simulations.Step{
Action: action,
Trigger: trigger,
Expect: &simulations.Expectation{
Nodes: msgtoids,
//Nodes: msgtoids,
Nodes: ids,
Check: check,
},
})
@ -555,7 +635,7 @@ func newServices() adapters.Services {
"bzz": func(ctx *adapters.ServiceContext) (node.Service, error) {
addr := network.NewAddrFromNodeID(ctx.Config.ID)
hp := network.NewHiveParams()
hp.Discovery = true
hp.Discovery = false
config := &network.BzzConfig{
OverlayAddr: addr.Over(),
UnderlayAddr: addr.Under(),

41
swarm/pss/pssapi.go
View file

@ -11,17 +11,20 @@ import (
"github.com/ethereum/go-ethereum/swarm/network"
)
// API is the RPC API module for Pss
// Pss API services
type API struct {
*Pss
}
// NewAPI constructs a PssAPI instance
func NewAPI(ps *Pss) *API {
return &API{Pss: ps}
}
// NewMsg API endpoint creates an RPC subscription
// Creates a new subscription for the caller. Enables external handling of incoming messages.
//
// A new handler is registered in pss for the supplied topic
//
// All incoming messages to the node matching this topic will be encapsulated in the APIMsg struct and sent to the subscriber
func (pssapi *API) Receive(ctx context.Context, topic Topic) (*rpc.Subscription, error) {
notifier, supported := rpc.NotifierFromContext(ctx)
if !supported {
@ -54,31 +57,47 @@ func (pssapi *API) Receive(ctx context.Context, topic Topic) (*rpc.Subscription,
return psssub, nil
}
// SendRaw sends the message (serialized into byte slice) to a peer with topic
// Sends the message wrapped in APIMsg through pss
//
// Wrapper method for the pss.SendRaw function.
//
// The method will pass on the error received from pss.
//
// Note that normally pss will report an error if an attempt is made to send a pss to oneself. However, if the debug flag has been set, and the address specified in APIMsg is the node's own, this method implements a short-circuit which injects the message as an incoming message (using Pss.Process). This can be useful for testing purposes, when only operating with one node.
func (pssapi *API) Send(topic Topic, msg APIMsg) error {
if pssapi.debug && bytes.Equal(msg.Addr, pssapi.BaseAddr()) {
if pssapi.debug && bytes.Equal(msg.Addr, pssapi.Pss.BaseAddr()) {
log.Warn("Pss debug enabled; send to self shortcircuit", "apimsg", msg, "topic", topic)
env := NewEnvelope(msg.Addr, topic, msg.Msg)
return pssapi.Process(&PssMsg{
To: pssapi.BaseAddr(),
To: pssapi.Pss.BaseAddr(),
Payload: env,
})
}
return pssapi.SendRaw(msg.Addr, topic, msg.Msg)
}
// BaseAddr returns the pss node's swarm overlay address
//
// Note that the overlay address is NOT inferable. To really know the node's overlay address it must reveal it itself.
func (pssapi *API) BaseAddr() ([]byte, error) {
return pssapi.Pss.BaseAddr(), nil
}
// PssAPITest are temporary API calls for development use only
// These symbols should not be included in production environment
//
// These symbols should NOT be included in production environment
type APITest struct {
*Pss
}
// NewAPI constructs a API instance
// Include these methods to the node.Service if test symbols should be used
func NewAPITest(ps *Pss) *APITest {
return &APITest{Pss: ps}
}
// temporary for access to overlay while faking kademlia healthy routines
// Get the current nearest swarm node to the specified address
//
// (Can be used for diagnosing kademlia state)
func (pssapitest *APITest) GetForwarder(addr []byte) (fwd struct {
Addr []byte
Count int
@ -93,7 +112,3 @@ func (pssapitest *APITest) GetForwarder(addr []byte) (fwd struct {
return
}
// BaseAddr gets our own overlayaddress
func (pssapitest *APITest) BaseAddr() ([]byte, error) {
return pssapitest.Pss.BaseAddr(), nil
}

6
swarm/pss/simulations/service_test.go
View file

@ -139,9 +139,13 @@ func (t *testWrapper) newTestService(ctx *adapters.ServiceContext) (node.Service
if err != nil {
panic(err)
}
pssclient, err := client.NewClientWithRPC(context.Background(), rpcClient)
if err != nil {
return nil, err
}
return &testService{
id: ctx.Config.ID,
pss: client.NewClientWithRPC(context.Background(), rpcClient),
pss: pssclient,
handshakes: make(chan *testHandshake),
}, nil
}

62
swarm/pss/types.go
View file

@ -18,13 +18,13 @@ const (
defaultDigestCacheTTL = time.Second
)
// Defines params for Pss
// Pss configuration parameters
type PssParams struct {
Cachettl time.Duration
Debug bool
}
// Initializes default params for Pss
// Sane defaults for Pss
func NewPssParams(debug bool) *PssParams {
return &PssParams{
Cachettl: defaultDigestCacheTTL,
@ -32,13 +32,14 @@ func NewPssParams(debug bool) *PssParams {
}
}
// Encapsulates the message transported over pss.
// Encapsulates messages transported over pss.
type PssMsg struct {
To []byte
Payload *Envelope
}
func (msg *PssMsg) Serialize() []byte {
// serializes the message for use in cache
func (msg *PssMsg) serialize() []byte {
rlpdata, _ := rlp.EncodeToBytes(msg)
return rlpdata
}
@ -48,16 +49,8 @@ func (self *PssMsg) String() string {
return fmt.Sprintf("PssMsg: Recipient: %x", common.ByteLabel(self.To))
}
// Topic defines the context of a message being transported over pss
// It is used by pss to determine what action is to be taken on an incoming message
// Typically, one can map protocol handlers for the message payloads by mapping topic to them; see *Pss.Register()
type Topic [TopicLength]byte
func (self *Topic) String() string {
return fmt.Sprintf("%x", self)
}
// Pre-Whisper placeholder, payload of PssMsg
// Pre-Whisper placeholder, payload of PssMsg, sender address, Topic
type Envelope struct {
Topic Topic
TTL uint16
@ -65,7 +58,7 @@ type Envelope struct {
From []byte
}
// creates Pss envelope from sender address, topic and raw payload
// Creates A Pss envelope from sender address, topic and raw payload
func NewEnvelope(addr []byte, topic Topic, payload []byte) *Envelope {
return &Envelope{
From: addr,
@ -75,7 +68,7 @@ func NewEnvelope(addr []byte, topic Topic, payload []byte) *Envelope {
}
}
// encapsulates a protocol msg as PssEnvelope data
// Convenience wrapper for devp2p protocol messages for transport over pss
type ProtocolMsg struct {
Code uint64
Size uint32
@ -83,13 +76,7 @@ type ProtocolMsg struct {
ReceivedAt time.Time
}
// PssAPIMsg is the type for messages, it extends the rlp encoded protocol Msg
// with the Sender's overlay address
type APIMsg struct {
Msg []byte
Addr []byte
}
// Creates a ProtocolMsg
func NewProtocolMsg(code uint64, msg interface{}) ([]byte, error) {
rlpdata, err := rlp.EncodeToBytes(msg)
@ -97,8 +84,6 @@ func NewProtocolMsg(code uint64, msg interface{}) ([]byte, error) {
return nil, err
}
// previous attempts corrupted nested structs in the payload iself upon deserializing
// therefore we use two separate []byte fields instead of peerAddr
// TODO verify that nested structs cannot be used in rlp
smsg := &ProtocolMsg{
Code: code,
@ -109,12 +94,30 @@ func NewProtocolMsg(code uint64, msg interface{}) ([]byte, error) {
return rlp.EncodeToBytes(smsg)
}
// Message handler func for a topic
// Convenience wrapper for sending and receiving pss messages when using the pss API
type APIMsg struct {
Msg []byte
Addr []byte
}
// Signature for a message handler function for a PssMsg
//
// Implementations of this type are passed to Pss.Register together with a topic,
type Handler func(msg []byte, p *p2p.Peer, from []byte) error
// constructs a new PssTopic from a given name and version.
// Topic defines the context of a message being transported over pss
// It is used by pss to determine what action is to be taken on an incoming message
// Typically, one can map protocol handlers for the message payloads by mapping topic to them; see Pss.Register
type Topic [TopicLength]byte
// String representation of Topic
func (self *Topic) String() string {
return fmt.Sprintf("%x", self)
}
// Constructs a new PssTopic from a given name and version.
//
// Analogous to the name and version members of p2p.Protocol
// Analogous to the name and version members of p2p.Protocol.
func NewTopic(s string, v int) (topic Topic) {
h := sha3.NewKeccak256()
h.Write([]byte(s))
@ -125,6 +128,11 @@ func NewTopic(s string, v int) (topic Topic) {
return topic
}
// For devp2p protocol integration only
//
// Creates a serialized (non-buffered) version of a p2p.Msg, used in the specialized p2p.MsgReadwriter implementations used internally by pss
//
// Should not normally be called outside the pss package hierarchy
func ToP2pMsg(msg []byte) (p2p.Msg, error) {
payload := &ProtocolMsg{}
if err := rlp.DecodeBytes(msg, payload); err != nil {

47
swarm/swarm_psschat.go
View file

@ -42,7 +42,7 @@ import (
"github.com/ethereum/go-ethereum/swarm/network"
"github.com/ethereum/go-ethereum/swarm/pss"
"github.com/ethereum/go-ethereum/swarm/storage"
psschat "github.com/ethereum/go-ethereum/swarm/pss/protocols/chat"
//psschat "github.com/ethereum/go-ethereum/swarm/pss/protocols/chat"
)
// the swarm stack
@ -197,7 +197,7 @@ func (self *Swarm) Start(net *p2p.Server) error {
// update uaddr to correct enode
newaddr := self.bzz.UpdateLocalAddr([]byte(net.Self().String()))
log.Warn("Updated bzz local addr", "oaddr", fmt.Sprintf("%s", newaddr.OAddr), "uaddr", fmt.Sprintf("%x", newaddr.UAddr))
log.Warn("Updated bzz local addr", "oaddr", fmt.Sprintf("%x", newaddr.OAddr), "uaddr", fmt.Sprintf("%x", newaddr.UAddr))
// set chequebook
if self.swapEnabled {
@ -223,25 +223,30 @@ func (self *Swarm) Start(net *p2p.Server) error {
if self.pss != nil {
self.pss.Start(net)
log.Info("Pss started")
pss.RegisterPssProtocol(self.pss, &psschat.ChatTopic, psschat.ChatProtocol, psschat.New(nil, nil, func(ctrl *psschat.ChatCtrl) {
if ctrl.OutC != nil {
go func() {
for {
select {
case msg := <-ctrl.OutC:
err := ctrl.Peer.Send(msg)
if err != nil {
// do something with ctrl.ConnC here
//pp.Drop(err)
//return
}
}
}
}()
}
}))
topic := pss.NewTopic("pssChat", 1)
self.pss.Register(&topic, func(msg []byte, p *p2p.Peer, from []byte) error {
log.Trace("placeholder handler for node got psschat proto incoming", "msg", msg, "from", from)
return nil
})
log.Info("Pss started ... with 'pssChat' :)")
// pss.RegisterPssProtocol(self.pss, &psschat.ChatTopic, psschat.ChatProtocol, psschat.New(nil, nil, nil, func(ctrl *psschat.ChatCtrl) {
// if ctrl.OutC != nil {
// go func() {
// for {
// select {
// case msg := <-ctrl.OutC:
// err := ctrl.Peer.Send(msg)
// if err != nil {
// // do something with ctrl.ConnC here
// //pp.Drop(err)
// //return
// }
// }
// }
// }()
// }
//
// }))
}
self.dpa.Start()