swarm/cmd: fix directory checksum-ing for correctness tests; add bootstrap-test.sh script

swarm/network: config tweaks and log improved
swarm/storage: cloud delivery in own go routine, dpa worker settings

swarm/cmd/bootstrap-testnet.sh

@@ -0,0 +1,28 @@
+#
+swarm remote-run nodes.lst 'swarm netstatkill; swarm stop all;  rm -rf bin bzz .ethash /tmp/.eth* /tmp/swarm*; sudo reboot'
+sleep 60
+# update the control scripts on each host
+swarm remote-update-scripts nodes.lst
+# update the binary on each host node
+swarm remote-update-bin nodes.lst
+
+# spawn a 2-instance local cluster on each of our 10 host nodes
+swarm remote-run nodes.lst 'swarm init  2; swarm netstatconf sworm; swarm netstatrun'
+
+#collect enodes from all instances on all hosts
+swarm remote-run nodes.lst 'swarm enode all' | tr -d '"' |grep -v running > enodes.lst
+
+# swarm remote-run nodes.lst swarm addpeers <(enode)'
+
+# copy enodes file to each host node
+for node in `cat nodes.lst`; do scp enodes.lst $node:; done
+
+# inject all enodes as peers to
+
+swarm remote-run nodes.lst swarm addpeers all enodes.lst
+
+# to bootstrap with  i
+
+# restart an instance on the first two  host nodes to get the blockchain rolling
+swarm remote-run <(head -2 nodes.lst) swarm restart 00 --mine
+

swarm/cmd/swarm/swarm

@@ -287,6 +287,7 @@ function stop {
 function shutdown {
   echo -n "stopping $1..."
   kill -2 $1
+  ((i=0))
   while true; do
     ps auxwww|grep geth|grep -v grep|awk '{print $2}'|grep -ql $1 || break
     if ((i++>5)); then
@@ -434,7 +435,7 @@ function up {
   id=$1
   path=$2
   file=$3
-  if [ $file = "" ]; then
+  if [ "$file" = "" ]; then
     file=`basename $path`
   fi
   echo "Upload file '$path' to node $id... " 1>&2
@@ -590,20 +591,26 @@ function update {
 }
 
 function checksum {
-  tar -cf - $1 | md5sum|awk '{print $1}'
+  if [ -f $1 ]; then
+    cat $1 | md5sum | awk '{print $1}'
+  else
+    cd $1 && md5deep -r -l . | sort | md5sum| awk '{print $1}'
+  fi
 }
 
 function checkaccess {
   nodes=$1
   path=$2
   target=`basename $2`
-  chsum=`md5sum $2|cut -f1 -d' '`
+  chsum=`checksum $2`
   shift
   shift
   master=`head  -1 $nodes`
-  echo "uploading target on $master (md5sum $chsum, size: `du -b -d0 $2|cut -f1`)"
+  echo "uploading target on $master (md5sum $chsum, size: `du -b -d0 $path|cut -f1`)"
-  scp $path $master:$target
+  remote-run $master rm -rf $target
+  scp -r $path $master:$target
   hash=`swarm remote-run $master "swarm up 00 $target $*"|tr -d '"'`
+  echo $hash
   remote-run $nodes swarm checkdownload all $hash $chsum
 }
 
@@ -618,25 +625,27 @@ function checkdownload {
     done
   else
     hash=$2
-    target=$3
     datetag=`date "+%Y-%m-%d-%H:%M:%S"`
     file="swarm-$datetag"
-    rm -rf $tmpdir/$file
+    dest=$tmpdir/$file
-    /usr/bin/time -o $tmpdir/$file.log -f "%e" swarm download $id $hash $tmpdir/$file > /dev/null
+    target=$3
-    echo
+    rm -rf $dest
+    mkdir -p $dest
+    /usr/bin/time -o $dest.log -f "%e" swarm download $id $hash $dest > /dev/null
+    echo "instance $id download $hash to $dest"
     if [ -f $target ]; then
-      cmp --silent  $target $tmpdir/$file/* && echo PASS || echo FAIL
+      cmp --silent  $target $dest/* && echo PASS || echo FAIL
     elif [ -r $target ]; then
-      diff -r $target $tmpdir/$file/ >/dev/null && echo PASS || echo FAIL
+      diff -r $target $dest >/dev/null && echo PASS || echo FAIL
     else
-      exp=`md5sum $tmpdir/$file/*|cut -f1 -d' '`
+      exp=`checksum $dest/*`
       if [ "$exp" = "$target" ]; then
         echo -n  PASS
       else
         echo FAIL "$exp = $target"
       fi
     fi
-    echo " latency: " `cat $tmpdir/$file.log`
+    echo " latency: " `cat $dest.log`
   fi
 }
 
@@ -676,6 +685,8 @@ case $cmd in
     info $*;;
   "enode" )
     enode $*;;
+  "checksum" )
+    checksum $*;;
   "status" )
     status $*;;
   "peers" )

swarm/network/hive.go

@@ -128,7 +128,7 @@ func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPee
 			node, need, proxLimit := self.kad.Suggest()
 
 			if node != nil && len(node.Url) > 0 {
-				glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: call known bee %v", node.Url)
+				glog.V(logger.Detail).Infof("[BZZ] KΛÐΞMLIΛ hive: call known bee %v", node.Url)
 				// enode or any lower level connection address is unnecessary in future
 				// discovery table is used to look it up.
 				connectPeer(node.Url)
@@ -142,7 +142,7 @@ func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPee
 					req := &retrieveRequestMsgData{
 						Key: storage.Key(randAddr[:]),
 					}
-					glog.V(logger.Debug).Infof("[BZZ] KΛÐΞMLIΛ hive: call any bee near %v (PO%03d) - messenger bee: %v", randAddr, proxLimit, peers[0])
+					glog.V(logger.Detail).Infof("[BZZ] KΛÐΞMLIΛ hive: call any bee near %v (PO%03d) - messenger bee: %v", randAddr, proxLimit, peers[0])
 					peers[0].(*peer).retrieve(req)
 				} else {
 					glog.V(logger.Warn).Infof("[BZZ] KΛÐΞMLIΛ hive: no peer")

swarm/network/kademlia/kademlia.go

@@ -22,7 +22,8 @@ const (
 var (
 	purgeInterval        = 42 * time.Hour
 	initialRetryInterval = 42 * time.Millisecond
-	maxIdleInterval      = 42 * 100 * time.Millisecond
+	maxIdleInterval      = 42 * 1000 * time.Millisecond
+	// maxIdleInterval      = 42 * 10	0 * time.Millisecond
 )
 
 type KadParams struct {

swarm/network/protocol.go

@@ -272,6 +272,7 @@ func (self *bzz) handle() error {
 			return self.protoError(ErrDecode, "<- %v: %v", msg, err)
 		}
 		glog.V(logger.Debug).Infof("[BZZ] <- sync request: %v", req)
+		self.lastActive = time.Now()
 		self.sync(req.SyncState)
 
 	case unsyncedKeysMsg:
@@ -282,6 +283,7 @@ func (self *bzz) handle() error {
 		}
 		glog.V(logger.Debug).Infof("[BZZ] <- unsynced keys : %s", req.String())
 		err := self.storage.HandleUnsyncedKeysMsg(&req, &peer{bzz: self})
+		self.lastActive = time.Now()
 		if err != nil {
 			return self.protoError(ErrDecode, "<- %v: %v", msg, err)
 		}
@@ -295,6 +297,7 @@ func (self *bzz) handle() error {
 		}
 		glog.V(logger.Debug).Infof("[BZZ] <- delivery request: %s", req.String())
 		err := self.storage.HandleDeliveryRequestMsg(&req, &peer{bzz: self})
+		self.lastActive = time.Now()
 		if err != nil {
 			return self.protoError(ErrDecode, "<- %v: %v", msg, err)
 		}

swarm/network/syncer.go

@@ -426,7 +426,7 @@ LOOP:
 					keys = self.keys[priority]
 					break PRIORITIES
 				}
-				glog.V(logger.Debug).Infof("[BZZ] syncer[%v/%v]: queue: [%v, %v, %v]", self.key.Log(), priority, len(self.keys[High]), len(self.keys[Medium]), len(self.keys[Low]))
+				glog.V(logger.Detail).Infof("[BZZ] syncer[%v/%v]: queue: [%v, %v, %v]", self.key.Log(), priority, len(self.keys[High]), len(self.keys[Medium]), len(self.keys[Low]))
 				// if the input queue is empty on this level, resort to history if there is any
 				if uint(priority) == histPrior && history != nil {
 					glog.V(logger.Detail).Infof("[BZZ] syncer[%v]: reading history for %v", self.key.Log(), self.key)

swarm/storage/dpa.go

@@ -23,12 +23,12 @@ implementation for storage or retrieval.
 */
 
 const (
-	storeChanCapacity           = 1000
+	storeChanCapacity           = 100
-	retrieveChanCapacity        = 1000
+	retrieveChanCapacity        = 100
 	singletonSwarmDbCapacity    = 50000
 	singletonSwarmCacheCapacity = 500
-	maxStoreProcesses           = 100
+	maxStoreProcesses           = 8
-	maxRetrieveProcesses        = 100
+	maxRetrieveProcesses        = 8
 )
 
 var (

swarm/storage/netstore.go

@@ -87,12 +87,12 @@ func (self *NetStore) Put(entry *Chunk) {
 		// that the chunk is has been retrieved
 		close(entry.Req.C)
 		// deliver the chunk to requesters upstream
-		self.cloud.Deliver(entry)
+		go self.cloud.Deliver(entry)
 	} else {
 		glog.V(logger.Detail).Infof("[BZZ] NetStore.Put: localStore.Put %v stored locally", entry.Key.Log())
 		// handle propagating store requests
 		// go self.cloud.Store(entry)
-		self.cloud.Store(entry)
+		go self.cloud.Store(entry)
 	}
 }