network test using ethutil/swarm

* migrate ethutil scripts under swarm/cmd/swarm * add basic peer connection bootstrap test * hive keepAlive launches with alarm in case no discover and no kaddb * fix IP address formatting issue [::1] -> became ::1 which refused to dial, now use discover.NewNode#String * database Print removed * add another syncer test for aborted sync * forwarding: only send if syncer is setup on the peer (not blocking) * add control api for network (block read) * syncing test for stalled/aborted connection
2026-07-24 13:46:43 +00:00 · 2015-12-18 23:38:11 +00:00 · 2015-12-18 23:38:11 +00:00 · 18cac67202
commit 18cac67202
parent 0e26569539
19 changed files with 640 additions and 96 deletions
--- a/rpc/api/bzz_js.go
+++ b/rpc/api/bzz_js.go
@ -51,6 +51,12 @@ web3._extend({
 			params: 3,
 			inputFormatter: [null, null, null]
 		}),
+		new web3._extend.Method({
+			name: 'blockNetworkRead',
+			call: 'bzz_blockNetworkRead',
+			params: 1,
+			inputFormatter: [null]
+		}),
 		new web3._extend.Method({
 			name: 'resolve',
 			call: 'bzz_resolve',
--- a/rpc/api/utils.go
+++ b/rpc/api/utils.go
@ -73,6 +73,7 @@ var (
 			"get",
 			"put",
 			"modify",
+			"blockNetworkRead",
 		},
 		"chequebook": []string{
 			"version",
--- a/swarm/api/testapi.go
+++ b/swarm/api/testapi.go
@ -0,0 +1,20 @@
+package api
+
+import (
+	// "fmt"
+
+	"github.com/ethereum/go-ethereum/swarm/network"
+)
+
+type Control struct {
+	api  *Api
+	hive *network.Hive
+}
+
+func NewControl(api *Api, hive *network.Hive) *Control {
+	return &Control{api, hive}
+}
+
+func (self *Control) BlockNetworkRead(on bool) {
+	self.hive.BlockNetworkRead(true)
+}
--- a/swarm/cmd/swarm/gethup.sh
+++ b/swarm/cmd/swarm/gethup.sh
@ -0,0 +1,137 @@
+#!/bin/bash
+# Usage:
+# bash /path/to/eth-utils/gethup.sh <datadir> <instance_name> <ip_addr>
+
+root=$1  # base directory to use for datadir and logs
+shift
+id=$1  # double digit instance id like 00 01 02
+shift
+ip_addr=$1 # ip address to substitute
+shift
+
+# logs are output to a date-tagged file for each run , while a link is
+# created to the latest, so that monitoring be easier with the same filename
+# TODO: use this if GETH not set
+# GETH=geth
+
+# geth CLI params       e.g., (dd=04, run=09)
+datetag=`date "+%c%y%m%d-%H%M%S"|cut -d ' ' -f 5`
+datadir=$root/data/$id        # /tmp/eth/04
+log=$root/log/$id.$datetag.log     # /tmp/eth/04.09.log
+linklog=$root/log/$id.current.log     # /tmp/eth/04.09.log
+stablelog=$root/log/$id.log     # /tmp/eth/04.09.log
+password=$id            # 04
+port=345$id              # 34504
+bzzport=622$id              # 62204
+rpcport=62$id            # 6204
+
+mkdir -p $root/data
+mkdir -p $root/log
+ln -sf "$log" "$linklog"
+# if we do not have an account, create one
+# will not prompt for password, we use the double digit instance id as passwd
+# NEVER EVER USE THESE ACCOUNTS FOR INTERACTING WITH A LIVE CHAIN
+if [ ! -d "$root/keystore/$id" ]; then
+  # echo "create an account with password $id"
+  # echo "create an account with password $id [DO NOT EVER USE THIS ON LIVE]"
+  mkdir -p $root/keystore/$id
+  $GETH --datadir $datadir --password <(echo -n $id) account new >/dev/null 2>&1
+# create account with password 00, 01, ...
+  # note that the account key will be stored also separately outside
+  # datadir
+  # this way you can safely clear the data directory and still keep your key
+  # under `<rootdir>/keystore/dd
+
+  cp -R "$datadir/keystore" $root/keystore/$id
+fi
+
+# echo "copying keys $root/keystore/$id $datadir/keystore"
+# ls $root/keystore/$id/keystore/ $datadir/keystore
+
+# mkdir -p $datadir/keystore
+# if [ ! -d "$datadir/keystore" ]; then
+# echo "copying keys $root/keystore/$id $datadir/keystore"
+cp -R $root/keystore/$id/keystore/ $datadir/keystore/
+# fi
+
+
+# query node's enode url
+if [ $ip_addr="" ]; then
+  pattern='\d+\.\d+\.\d+\.\d+'
+  ip_addr="[::]"
+else
+  pattern='\[\:\:\]'
+fi
+
+# echo -n "enode for instance $id...  "
+
+if [ ! "$GETH" = "" ] && [ ! -f $root/enodes/$id.enode ]; then
+  geth="$GETH --datadir $root/data/$id --port $port"
+  cmd="$geth js <(echo 'console.log(admin.nodeInfo.enode); exit();') "
+  # echo $cmd '2>/dev/null |grep enode | perl -pe "s/'$pattern'/'$ip_addr'/g" | perl -pe "s/^/\"/; s/\s*$/\"/;" > '$root/enodes/$id.enode
+  eval $cmd 2>/dev/null |grep enode | perl -pe "s/$pattern/$ip_addr/g" | perl -pe "s/^/\"/; s/\s*\$/\"/;" > $root/enodes/$id.enode
+fi
+# cat  $root/enodes/$id.enode
+echo
+
+# copy cluster enodes list to node's static node list
+# echo "copy cluster enodes list to node's static node list"
+if [  -f $root/enodes.all ]; then
+  cp $root/enodes.all $datadir/static-nodes.json
+fi
+
+if [ ! -f $root/pids/$id.pid ]; then
+  # bring up node `dd` (double digit)
+  # - using <rootdir>/<dd>
+  # - listening on port 303dd, (like 30300, 30301, ...)
+  # - with the account unlocked
+  # - launching json-rpc server on port 81dd (like 8100, 8101, 8102, ...)
+  BZZKEY=`$GETH --datadir=$datadir account list|head -n1|perl -ne '/([a-f0-9]{40})/ && print $1'`
+  echo -n "starting instance $id ($BZZKEY @ $datadir )..."
+  # echo "$GETH --datadir=$datadir \
+  #   --identity=$id \
+  #   --bzzaccount=$BZZKEY --bzzport=$bzzport \
+  #   --port=$port \
+  #   --unlock=$BZZKEY \
+  #   --password=<(echo -n $id) \
+  #   --rpc --rpcport=$rpcport --rpccorsdomain='*' $* \
+  #   2>&1 | tee "$stablelog" > "$log" &  # comment out if you pipe it to a tty etc.
+  # " >&2
+
+  if [ -f $log ] && [ -f $root/stablelog ]; then
+    cp $stablelog `cat $root/prevlog`
+  fi
+  echo $log > $root/prevlog
+
+  $GETH --datadir=$datadir \
+    --identity=$id \
+    --bzzaccount=$BZZKEY --bzzport=$bzzport \
+    --port=$port \
+    --unlock=$BZZKEY \
+    --password=<(echo -n $id) \
+    --rpc --rpcport=$rpcport --rpccorsdomain='*' $* \
+     > "$stablelog" 2>&1 &  # comment out if you pipe it to a tty etc.
+
+  # pid=`ps auxwww|grep geth|grep "ty=$id"|grep -v grep|awk '{print $2}'`
+  # echo "pid: $pid"
+  # ps auxwww|grep geth|grep "ty=$id"|grep -v grep
+  # echo $pid > $root/pids/$id.pid
+  #echo $! > $root/pids/$id.pid
+  while true; do
+    $GETH --ipcpath=$datadir/geth.ipc --exec="net" attach > /dev/null 2>&1 && break
+    sleep 1
+    echo -n "."
+    if ((i++>10)); then
+      echo "instance $id failed to start"
+      exit 1
+    fi
+  done
+  echo -n "started - "
+  pid=`ps auxwww|grep geth|grep "ty=$id"|grep -v grep|awk '{print $2}'`
+  echo "pid: $pid"
+  # ps auxwww|grep geth|grep "ty=$id"|grep -v grep
+  echo $pid > $root/pids/$id.pid
+fi
+
+# to bring up logs, uncomment
+# tail -f $log
--- a/swarm/cmd/swarm/netstatconf.sh
+++ b/swarm/cmd/swarm/netstatconf.sh
@ -0,0 +1,35 @@
+# !/bin/bash
+# bash intelligence <destination_app_json_path> <number_of_nodes> <name_prefix> <ws_server> <ws_secret>
+
+# sets up a eth-net-intelligence app.json for a local ethereum network cluster of nodes
+# - <number_of_clusters> is the number of clusters
+# - <name_prefix> is a prefix for the node names as will appear in the listing
+# - <ws_server> is the eth-netstats server
+# - <ws_secret> is the eth-netstats secret
+#
+
+# open http://localhost:3301
+
+N=$1
+shift
+name_prefix=$1
+shift
+ws_server=$1
+shift
+ws_secret=$1
+shift
+
+echo -e "["
+
+for ((i=0;i<N;++i)); do
+    id=`printf "%02d" $i`
+    single_template="  {\n    \"name\"        : \"$name_prefix-$i\",\n    \"cwd\"         : \".\",\n    \"script\"      : \"app.js\",\n    \"log_date_format\"   : \"YYYY-MM-DD HH:mm Z\",\n    \"merge_logs\"    : false,\n    \"watch\"       : false,\n    \"exec_interpreter\"  : \"node\",\n    \"exec_mode\"     : \"fork_mode\",\n    \"env\":\n    {\n      \"NODE_ENV\"    : \"production\",\n      \"RPC_HOST\"    : \"localhost\",\n      \"RPC_PORT\"    : \"622$id\",\n      \"INSTANCE_NAME\"   : \"$name_prefix-$i\",\n      \"WS_SERVER\"     : \"$ws_server\",\n      \"WS_SECRET\"     : \"$ws_secret\",\n    }\n  }"
+
+    endline=""
+    if [ "$i" -ne "$N" ]; then
+        endline=","
+    fi
+    echo -e "$single_template$endline"
+done
+
+echo "]"
--- a/swarm/cmd/swarm/swarm.sh
+++ b/swarm/cmd/swarm/swarm.sh
@ -0,0 +1,267 @@
+# !/bin/bash
+# bash cluster <root> <network_id> <number_of_nodes>  <runid> <local_IP> [[params]...]
+# https://github.com/ethereum/go-ethereum/wiki/Setting-up-monitoring-on-local-cluster
+
+# sets up a local ethereum network cluster of nodes
+# - <number_of_nodes> is the number of nodes in cluster
+# - <root> is the root directory for the cluster, the nodes are set up
+#   with datadir `<root>/<network_id>/00`, `<root>/ <network_id>/01`, ...
+# - new accounts are created for each node
+# - they launch on port 30300, 30301, ...
+# - they star rpc on port 8100, 8101, ...
+# - by collecting the nodes nodeUrl, they get connected to each other
+# - if enode has no IP, `<local_IP>` is substituted
+# - if `<network_id>` is not 0, they will not connect to a default client,
+#   resulting in a private isolated network
+# - the nodes log into `<root>/<network_id>/00.<runid>.log`, `<root>/<network_id>/01.<runid>.log`, ...
+# - The nodes launch in mining mode
+# - the cluster can be killed with `killall geth` (FIXME: should record PIDs)
+#   and restarted from the same state
+# - if you want to interact with the nodes, use rpc
+# - you can supply additional params on the command line which will be passed
+#   to each node, for instance `-mine`
+
+if [ "$GETH" = "" ]; then
+  echo "env var GETH not set "
+  exit 1
+fi
+
+srcdir=`dirname $0`
+
+root=$1
+shift
+network_id=$1
+shift
+cmd=$1
+shift
+# ip_addr=`curl ipecho.net/plain 2>/dev/null;echo `
+
+# echo "external IP: $ip_addr"
+swarmoptions='--vmdebug=false --maxpeers=20 --dev --shh=false --nodiscover --ipcexp'
+
+
+function needs {
+  id=$1
+  keyfile=$2
+  target=$3
+  dir=`dirname $3`
+  dest=$dir/down
+  mkdir -p $dest
+  file=$dest/`basename $target`
+  rm -f $file
+  echo -n "download to '$file'...waiting for root hash in '$keyfile'..."
+  while true; do
+   if [ -f $keyfile ] && [ ! -z $keyfile ]; then
+    break
+   fi
+   sleep 1
+   echo -n "."
+  done
+  key=`cat $keyfile`
+  echo " => $key"
+  download $id $key $dest && cmp --silent $file $target && echo "PASS" || echo "FAIL"
+  # && ls -l $keyfile $file $target
+}
+
+function attach {
+  id=$1
+  shift
+  echo "attaching console to instance $id"
+  cmd="$GETH --ipcpath $root/$network_id/data/$id/geth.ipc $* attach"
+  # echo $cmd
+  eval $cmd
+}
+
+function log {
+  id=$1
+  shift
+  echo "streaming logs for instance $id"
+  cmd="tail -f $root/$network_id/log/$id.log"
+  echo $cmd
+  eval $cmd
+}
+
+function less {
+  id=$1
+  shift
+  echo "viewing logs for instance $id"
+  cmd="less $root/$network_id/log/$id.log"
+  echo $cmd
+  eval $cmd
+}
+
+function start {
+  id=$1
+  shift
+  # echo -n "starting instance $id - "
+  cmd="bash $srcdir/gethup.sh $root/$network_id/ $id '$ip_addr' --networkid=$network_id $swarmoptions $*"
+  # echo "pid="`cat $root/$network_id/pids/$id.pid`
+  # echo $cmd
+  eval $cmd
+}
+
+function stop {
+  id=$1
+  shift
+  if [ $id = "all" ]; then
+    procs=`cat $root/$network_id/pids/*.pid 2>/dev/null |perl -pe 's/^\s+//;s/\s+\\$//;s/\s+/\n/g'`
+    # echo "stopping processes $procs"
+    for p in $procs; do
+      shutdown $p
+    done
+    rm -rf $root/$network_id/pids/*
+  else
+    pid=$root/$network_id/pids/$id.pid
+    if [ -f $pid ]; then
+      echo "stopping instance $id, pid="`cat $pid`
+      shutdown `cat $pid`
+      rm $pid
+    fi
+  fi
+  # ps auxwww|grep geth|grep bzz|grep -v grep
+}
+
+function shutdown {
+  echo -n "stopping $1..."
+  kill -2 $1
+  while true   ;do
+    ps auxwww|grep geth|grep -v grep|awk '{print $2}'|grep -ql $1 || break
+    sleep 1
+    # ps auxwww|grep geth|grep -v grep |grep  $1 #|awk '{print $2}'
+  done
+  echo "stopped"
+}
+
+function restart {
+  id=$1
+  shift
+  stop $id
+  start $id $*
+}
+
+function init {
+  stop all
+  killall geth
+  reset all
+  cluster $*
+  stop all
+  sleep 5
+  cluster $*
+}
+
+function reset {
+  id=$1
+  shift
+  if [ $id = "all" ]; then
+    rm -rf $root/$network_id
+  else
+    rm -rf$root/$network_id/*/$id*
+  fi
+
+}
+
+function cluster {
+  N=$1
+  shift
+# --vmodule=netstore=6,depo=6,forwarding=6,hive=5,dpa=6,dpa=6,http=6,syncb=6,syncer=6,protocol=6,swap=6,chequebook=6
+  echo "launching cluster of $N instances"
+  # cmd="bash $srcdir/gethcluster.sh $root $network_id $N '' $swarmoptions $*"
+  # echo $cmd
+  # eval $cmd
+  dir=$root/$network_id
+  mkdir -p $dir/data
+  mkdir -p $dir/enodes
+  mkdir -p $dir/pids
+  mkdir -p $dir/log
+
+  enodes=$dir/enodes.all
+  rm -f $enodes
+  # build a static nodes(-like) list of all enodes of the local cluster
+  echo "[" >> $enodes
+  for ((i=0;i<N;++i)); do
+    id=`printf "%02d" $i`
+    enode=$dir/enodes/$id.enode
+    if [ -f "$enode" ]; then
+      cat "$enode" >> $enodes
+      echo "," >> $enodes
+    fi
+  done
+  echo "\"\"]" >> $enodes
+
+  for ((i=0;i<N;++i)); do
+    id=`printf "%02d" $i`
+    mkdir -p $dir/data/$id
+    echo "launching node $i/$N ---> tail-f $dir/log/$id.log"
+    start $id $*
+  done
+}
+
+
+function up { #port, file
+  echo "Upload file '$2' to node $1... " 1>&2
+  file=`basename $2`
+  key=`attach $1 "--exec 'bzz.upload(\"$2\", \"$file\")'"|tail -n1`
+  # key=`bash swarm/cmd/bzzup.sh $2 86$1`
+  echo "-> $key"
+  echo -n `eval echo $key`
+}
+
+function download {
+  echo "download to '$3'"
+  attach $1 "--exec 'bzz.download(\"$2\", \"$3\")'" > /dev/null
+}
+
+
+function down {
+  echo -n "Download hash '$2' from node $1... "
+  # echo "wget -O- http://localhost:86$1/$2 > /dev/null 2>&1 && echo 'got it' || echo 'not found'"
+  # wget -O- http://localhost:86$1/$2 > /dev/null 2>&1 && echo "got it" || echo "not found"
+  while true; do
+    attach $1 "--exec 'bzz.get(\"$2\")'" 2> /dev/null |grep -qil "status" && break
+    sleep 1
+    echo -n "."
+    if ((i++>10)); then
+      echo "not found"
+      return
+    fi
+  done
+  echo "found OK"
+}
+
+function clean { #index
+  echo "Clean up for $1"
+  rm -rf $root/$network_id/data/$1/{bzz/*/chunks,bzz/*/requests/,bzz/*/bzz-peers.json,chaindata,nodes}
+}
+
+function info {
+  echo "ROOTDIR: $root"
+  echo "DATADIR: $root/data/$1"
+  echo "LOGFILE: $root/log/$1.log"
+}
+
+case $cmd in
+  "info" )
+    info $*;;
+  "needs" )
+    needs $*;;
+  "up" )
+    up $*;;
+  "down" )
+    down $*;;
+  "init" )
+    init $*;;
+  "start" )
+    start $*;;
+  "stop" )
+    stop $* ;;
+  "restart" )
+    restart $*;;
+  "reset" )
+    reset $*;;
+  "cluster" )
+    cluster $*;;
+  "attach" )
+    attach $*;;
+  "log" )
+    log $*;;
+esac
--- a/swarm/cmd/swarm/test.sh
+++ b/swarm/cmd/swarm/test.sh
@ -0,0 +1,26 @@
+#!/bin/bash
+
+TEST_DIR=`dirname $0`
+TEST_NAME=`basename $0 .sh`
+
+export SWARM_BIN=$TEST_DIR/../../cmd/swarm
+export GETH=$SWARM_BIN/../../../geth
+export NETWORKID=622$TEST_NAME
+export TMPDIR=/tmp/swarm-test
+export DATA_ROOT=$TMPDIR/$NETWORKID
+# alias swarm='bash $SWARM_BIN/swarm.sh $DATA_ROOT $NETWORKID'
+EXTRA_ARGS=$*
+
+rm -rf $DATA_ROOT
+
+wait=1
+
+function swarm {
+  # echo bash $SWARM_BIN/swarm.sh $TMPDIR $NETWORKID  $* $EXTRA_ARGS
+  bash $SWARM_BIN/swarm.sh $TMPDIR $NETWORKID $* $EXTRA_ARGS
+}
+
+
+function randomfile {
+  dd if=/dev/urandom of=/dev/stdout bs=1024 count=$1 2>/dev/null
+}
--- a/swarm/network/forwarding.go
+++ b/swarm/network/forwarding.go
@ -4,9 +4,9 @@ import (
 	"math/rand"
 	"time"

-	"github.com/ethereum/go-ethereum/swarm/storage"
 	"github.com/ethereum/go-ethereum/logger"
 	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/swarm/storage"
 )

 const requesterCount = 3
@ -81,12 +81,12 @@ func (self *forwarder) Store(chunk *storage.Chunk) {
 	for _, p := range self.hive.getPeers(chunk.Key, 0) {
 		glog.V(logger.Detail).Infof("[BZZ] %v %v", p, chunk)

-		if source == nil || p.Addr() != source.Addr() {
+		if p.syncer != nil && (source == nil || p.Addr() != source.Addr()) {
 			n++
 			Deliver(p, msg, PropagateReq)
 		}
 	}
-	glog.V(logger.Detail).Infof("[BZZ] forwarder.Store: sent to %v ps (chunk = %v)", n, chunk)
+	glog.V(logger.Detail).Infof("[BZZ] forwarder.Store: sent to %v peers (chunk = %v)", n, chunk)
 }

 // once a chunk is found deliver it to its requesters unless timed out
--- a/swarm/network/hive.go
+++ b/swarm/network/hive.go
@ -32,6 +32,8 @@ type Hive struct {
 	path         string
 	toggle       chan bool
 	more         chan bool
+	blockRead    bool
+	blockWrite   bool
 }

 const (
@ -70,6 +72,14 @@ func NewHive(addr common.Hash, params *HiveParams) *Hive {
 	}
 }

+func (self *Hive) BlockNetworkRead(on bool) {
+	self.blockRead = on
+}
+
+func (self *Hive) BlockNetworkWrite(on bool) {
+	self.blockWrite = on
+}
+
 // public accessor to the hive base address
 func (self *Hive) Addr() kademlia.Address {
 	return self.addr
@ -134,7 +144,7 @@ func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPee
 // wake state is toggled by writing to self.toggle
 // it restarts if the table becomes non-full again due to disconnections
 func (self *Hive) keepAlive() {
-	var alarm <-chan time.Time
+	alarm := time.NewTicker(time.Duration(self.callInterval)).C
 	for {
 		select {
 		case <-alarm:
--- a/swarm/network/messages.go
+++ b/swarm/network/messages.go
@ -5,10 +5,11 @@ import (
 	"net"
 	"time"

-	"github.com/ethereum/go-ethereum/swarm/services/swap"
-	"github.com/ethereum/go-ethereum/swarm/storage"
 	"github.com/ethereum/go-ethereum/common/chequebook"
 	"github.com/ethereum/go-ethereum/common/kademlia"
+	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/swarm/services/swap"
+	"github.com/ethereum/go-ethereum/swarm/storage"
 )

 /*
@ -160,7 +161,9 @@ type peerAddr struct {

 // peerAddr pretty prints as enode
 func (self peerAddr) String() string {
-	return fmt.Sprintf("enode://%x@%v:%d", self.ID, self.IP, self.Port)
+	var nodeid discover.NodeID
+	copy(nodeid[:], self.ID)
+	return discover.NewNode(nodeid, self.IP, 0, self.Port).String()
 }

 /*
--- a/swarm/network/protocol.go
+++ b/swarm/network/protocol.go
@ -18,9 +18,8 @@ import (
 	"fmt"
 	"net"
 	"strconv"
+	"time"

-	bzzswap "github.com/ethereum/go-ethereum/swarm/services/swap"
-	"github.com/ethereum/go-ethereum/swarm/storage"
 	"github.com/ethereum/go-ethereum/common/chequebook"
 	"github.com/ethereum/go-ethereum/common/swap"
 	"github.com/ethereum/go-ethereum/errs"
@ -28,6 +27,8 @@ import (
 	"github.com/ethereum/go-ethereum/logger/glog"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/p2p/discover"
+	bzzswap "github.com/ethereum/go-ethereum/swarm/services/swap"
+	"github.com/ethereum/go-ethereum/swarm/storage"
 )

 const (
@ -174,6 +175,10 @@ func run(requestDb *storage.LDBDatabase, depo StorageHandler, hive *Hive, dbacce

 	// the main forever loop that handles incoming requests
 	for {
+		if self.hive.blockRead {
+			time.Sleep(1 * time.Second)
+			continue
+		}
 		err = self.handle()
 		if err != nil {
 			return
@ -496,6 +501,9 @@ func (self *bzz) protoErrorDisconnect(err *errs.Error) {
 }

 func (self *bzz) send(msg uint64, data interface{}) error {
+	if self.hive.blockWrite {
+		return fmt.Errorf("network write blocked")
+	}
 	glog.V(logger.Debug).Infof("[BZZ] -> %v: %v (%T) to %v", msg, data, data, self)
 	err := p2p.Send(self.rw, msg, data)
 	if err != nil {
--- a/swarm/network/syncer.go
+++ b/swarm/network/syncer.go
@ -478,6 +478,7 @@ LOOP:
 				glog.V(logger.Warn).Infof("[BZZ] syncer[%v]: unable to send unsynced keys: %v", err)
 			}
 			unsynced = nil
+			keys = nil
 		}

 		// process item and add it to the batch
@ -503,11 +504,11 @@ LOOP:
 			deliveryRequest = nil

 		case <-newUnsyncedKeys:
-			glog.V(logger.Detail).Infof("[BZZ] syncer[%v]: new unsynked keys available", self.key.Log())
+			glog.V(logger.Detail).Infof("[BZZ] syncer[%v]: new unsynced keys available", self.key.Log())
 			// this 1 cap channel can wake up the loop
 			// signals that data is available to send if peer is ready to receive
 			newUnsyncedKeys = nil
-			// this can only happen if keys is
+			keys = self.keys[High]

 		case state, more = <-syncStates:
 			// this resets the state
@ -620,7 +621,8 @@ func (self *syncer) syncDeliveries() {
 If sync mode is off then, requests are directly sent to deliveries
 */
 func (self *syncer) addRequest(req interface{}, ty int) {
-	// retrieve priority for request type
+	// retrieve priority for request type name int8
+
 	priority := self.SyncPriorities[ty]
 	// sync mode for this type ON
 	if self.SyncModes[ty] {
--- a/swarm/storage/chunker.go
+++ b/swarm/storage/chunker.go
@ -97,7 +97,7 @@ func (self *TreeChunker) KeySize() int64 {

 // String() for pretty printing
 func (self *Chunk) String() string {
-	return fmt.Sprintf("Key: %v TreeSize: %v Chunksize: %v\n", self.Key.Log(), self.Size, len(self.SData))
+	return fmt.Sprintf("Key: %v TreeSize: %v Chunksize: %v", self.Key.Log(), self.Size, len(self.SData))
 }

 // The treeChunkers own Hash hashes together
--- a/swarm/storage/database.go
+++ b/swarm/storage/database.go
@ -6,7 +6,6 @@ package storage
 import (
 	"fmt"

-	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/compression/rle"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/iterator"
@ -82,15 +81,3 @@ func (self *LDBDatabase) Close() {
 	// Close the leveldb database
 	self.db.Close()
 }
-
-func (self *LDBDatabase) Print() {
-	iter := self.db.NewIterator(nil, nil)
-	for iter.Next() {
-		key := iter.Key()
-		value := iter.Value()
-
-		fmt.Printf("%x(%d): ", key, len(key))
-		node := common.NewValueFromBytes(value)
-		fmt.Printf("%v\n", node)
-	}
-}
--- a/swarm/swarm.go
+++ b/swarm/swarm.go
@ -203,7 +203,7 @@ func (self *Swarm) APIs() []rpc.API {
 		rpc.API{Namespace, Version, &Info{self.config, chequebook.ContractParams}, true},
 		// admin APIs
 		rpc.API{Namespace, Version, api.NewFileSystem(self.api), false},
-		// rpc.API{Namespace, Version, test.New(self), false},
+		rpc.API{Namespace, Version, api.NewControl(self.api, self.hive), false},
 		// rpc.API{Namespace, Version, api.NewAdmin(self), false},
 		// TODO: external apis exposed
 		rpc.API{"chequebook", chequebook.Version, chequebook.NewApi(self.config.Swap.Chequebook()), true},
--- a/swarm/test/connections/00.sh
+++ b/swarm/test/connections/00.sh
@ -0,0 +1,33 @@
+#!/bin/bash
+
+dir=`dirname $0`
+source $dir/../../cmd/swarm/test.sh
+
+swarm init 4
+echo "expect each node to have 3 peers"
+cmd="'net.peerCount'"
+sleep 5
+swarm attach 00 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
+swarm attach 01 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
+swarm attach 02 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
+swarm attach 03 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
+
+swarm stop all
+
+echo "after static nodes is deleted, connections are recovered from kaddb in bzz-peers.json"
+# echo rm -rf $DATA_ROOT/enodes\*
+# echo rm -rf $DATA_ROOT/data/\*/static-nodes.json
+rm -rf $DATA_ROOT/enodes*
+rm -rf $DATA_ROOT/data/*/static-nodes.json
+
+swarm cluster 4
+echo "expect each node to have 3 peers"
+cmd="'net.peerCount'"
+sleep 10
+swarm attach 00 --exec "$cmd" |tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
+swarm attach 01 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
+swarm attach 02 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
+swarm attach 03 --exec "$cmd"|tail -n1|grep -ql 3&& echo "PASS"||echo "FAIL"
+
+swarm stop all
+
--- a/swarm/test/syncing/00.sh
+++ b/swarm/test/syncing/00.sh
@ -1,83 +1,48 @@
 #!/bin/bash

-function up { #file port
-  echo "Upload file '$1' to node $2 on port 85$2" 1>&2
-  key=`bash swarm/cmd/bzzup.sh $1 85$2`
-  echo -n $key
-}
+dir=`dirname $0`
+source $dir/../../cmd/swarm/test.sh

-function down { #key port
-  echo "Download hash '$1' from node $2 on port 85$2"
-  wget -O- http://localhost:85$2/$1 > /dev/null && echo "got it" || echo "not found"
-}
+mkdir -p /tmp/swarm-test-files
+FILE_00=/tmp/swarm-test-files/00
+FILE_01=/tmp/swarm-test-files/01
+FILE_02=/tmp/swarm-test-files/02
+FILE_03=/tmp/swarm-test-files/03
+FILE_04=/tmp/swarm-test-files/04

-function clean { #index
-  echo "Clean up for $1"
-  rm -rf ~/tmp/sync/$1/{bzz/*/chunks,bzz/*/requests/,bzz/*/bzz-peers.json,chaindata,nodes}
-}
+for f in $FILE_00 $FILE_01 $FILE_02 $FILE_03 $FILE_04; do
+  randomfile 20 > $f
+done

-function gethup { #index account
-  cp geth geth$1
-  echo "start node $1"
-  echo "./geth$1 --datadir ~/tmp/sync/$1 --bzzaccount $2 --unlock 0 --password <(echo bzz) --networkid 323 --port 303$1 --vmodule netstore=6,depo=6,forwarding=6,hive=5,dpa=6,dpa=6,http=6,syncb=6,syncer=6,protocol=6,swap=6,chequebook=6 --shh=false --nodiscover --maxpeers 20 --dev --vmdebug=false --verbosity 4 2> sync$1.log &"
-  ./geth$1 --datadir ~/tmp/sync/$1 --bzzaccount "$2" --unlock 0 --password <(echo bzz) --networkid 323 --port 303$1 --vmodule netstore=6,depo=6,forwarding=6,hive=5,dpa=6,dpa=6,http=6,syncb=6,syncer=6,protocol=6,swap=6,chequebook=6 --shh=false --nodiscover --maxpeers 20 --dev --vmdebug=false --verbosity 4 2> sync$1.log
-}
+key=/tmp/key
+swarm init 2
+swarm up 00 $FILE_00|tail -n1 > $key
+swarm needs 00 $key $FILE_00

-function gethdown { #index
-  killall -INT geth$1
-}
+swarm start 01
+swarm needs 01 $key $FILE_00
+swarm stop 01

-wait=5
-gethdown 00
-gethdown 01
+swarm up 00 $FILE_01|tail -n1 > $key
+swarm needs 00 $key $FILE_01
+swarm start 01
+swarm needs 01 $key $FILE_01

-# ./geth --datadir ~/tmp/sync/00 --password <(echo bzz) account new
-BZZKEY00=5c6332e46a095feb9da1ed9803af2fa425f96aa6
-BZZKEY01=7dafa7436cba4b5b94a0452557b20b01d23bdc05
-echo "Fresh start, wipe datadir"
-clean 00
-clean 01
+swarm up 00 $FILE_02|tail -n1 > $key
+swarm needs 00 $key $FILE_02
+swarm needs 01 $key $FILE_02

-gethup 00 $BZZKEY00 &
-sleep 2
-echo "beyond\n"
-key=$(up COPYING.LESSER 00)
-echo "beyond\n"
-down $key 00
+swarm up 01 $FILE_03|tail -n1 > $key
+swarm needs 01 $key $FILE_03
+swarm needs 00 $key $FILE_03

-echo "beyond\n"
-gethup 01 $BZZKEY01 &
-sleep $wait
-down $key 01
-gethdown 01
+swarm stop 00
+swarm up 01 $FILE_04|tail -n1 > $key
+swarm needs 01 $key $FILE_04
+swarm start 00
+swarm needs 00 $key $FILE_04

-
-key=$(up AUTHORS 00)
-down $key 00
-gethup 01 $BZZKEY01 &
-sleep $wait
-down $key 01
-
-key=$(up COPYING 00)
-down $key 00
-down $key 01
-
-key=$(up README.md 01)
-down $key 01
-sleep $wait
-down $key 00
-
-exit 0
-
-gethdown 00
-key=$(up README.md 01)
-down $key 01
-gethup 00 $BZZKEY00 &
-sleep $wait
-down $key 00
-
-gethdown 00
-gethdown 01
+swarm stop all



--- a/swarm/test/syncing/01.sh
+++ b/swarm/test/syncing/01.sh
@ -0,0 +1,19 @@
+#!/bin/bash
+
+dir=`dirname $0`
+source $dir/../../cmd/swarm/test.sh
+
+long=/tmp/10M
+randomfile 10000 > $long
+ls -l $long
+
+swarm init 2
+sleep $wait
+swarm up 00 $long |tail -n1 > key &
+sleep $wait
+swarm stop 01
+
+swarm start 01
+swarm needs 01 key $long
+
+swarm stop all
--- a/swarm/test/syncing/02.sh
+++ b/swarm/test/syncing/02.sh
@ -0,0 +1,25 @@
+#!/bin/bash
+
+dir=`dirname $0`
+source $dir/../../cmd/swarm/test.sh
+
+long=/tmp/10M
+randomfile 10000 > $long
+ls -l $long
+
+swarm init 2
+sleep $wait
+swarm up 00 $long |tail -n1 > key &
+sleep $wait
+swarm attach 01 -exec "'bzz.blockNetworkRead(true)'"
+sleep $wait
+swarm attach 01 -exec "'bzz.blockNetworkRead(false)'"
+sleep $wait
+swarm attach 01 -exec "'bzz.blockNetworkRead(true)'"
+sleep $wait
+swarm stop 01
+
+swarm start 01
+swarm needs 01 key $long
+
+swarm stop all