Merge pull request #16 from ethereum/master

merge

cmd/swarm/config.go

@@ -132,7 +132,7 @@ func initSwarmNode(config *bzzapi.Config, stack *node.Node, ctx *cli.Context) {
 	log.Debug(printConfig(config))
 }
 
-//override the current config with whatever is in the config file, if a config file has been provided
+//configFileOverride overrides the current config with the config file, if a config file has been provided
 func configFileOverride(config *bzzapi.Config, ctx *cli.Context) (*bzzapi.Config, error) {
 	var err error
 
@@ -142,7 +142,8 @@ func configFileOverride(config *bzzapi.Config, ctx *cli.Context) (*bzzapi.Config
 		if filepath = ctx.GlobalString(SwarmTomlConfigPathFlag.Name); filepath == "" {
 			utils.Fatalf("Config file flag provided with invalid file path")
 		}
-		f, err := os.Open(filepath)
+		var f *os.File
+		f, err = os.Open(filepath)
 		if err != nil {
 			return nil, err
 		}

cmd/swarm/config_test.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net"
 	"os"
 	"os/exec"
 	"testing"
@@ -559,3 +560,16 @@ func TestValidateConfig(t *testing.T) {
 		}
 	}
 }
+
+func assignTCPPort() (string, error) {
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		return "", err
+	}
+	l.Close()
+	_, port, err := net.SplitHostPort(l.Addr().String())
+	if err != nil {
+		return "", err
+	}
+	return port, nil
+}

cmd/swarm/fs_test.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
 
-// +build linux darwin freebsd
+// +build linux freebsd
 
 package main
 
@@ -43,12 +43,12 @@ type testFile struct {
 }
 
 // TestCLISwarmFs is a high-level test of swarmfs
+//
+// This test fails on travis for macOS as this executable exits with code 1
+// and without any log messages in the log:
+// /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse.
+// This is the reason for this file not being built on darwin architecture.
 func TestCLISwarmFs(t *testing.T) {
-	// This test fails on travis as this executable exits with code 1
-	// and without any log messages in the log.
-	// /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse
-	t.Skip()
-
 	cluster := newTestCluster(t, 3)
 	defer cluster.Shutdown()
 

cmd/swarm/run_test.go

@@ -17,12 +17,15 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 	"net"
 	"os"
 	"path/filepath"
 	"runtime"
+	"sync"
+	"syscall"
 	"testing"
 	"time"
 
@@ -218,14 +221,12 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode {
 	}
 
 	// assign ports
-	httpPort, err := assignTCPPort()
-	if err != nil {
-		t.Fatal(err)
-	}
-	p2pPort, err := assignTCPPort()
+	ports, err := getAvailableTCPPorts(2)
 	if err != nil {
 		t.Fatal(err)
 	}
+	p2pPort := ports[0]
+	httpPort := ports[1]
 
 	// start the node
 	node.Cmd = runSwarm(t,
@@ -246,6 +247,17 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode {
 		}
 	}()
 
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	// ensure that all ports have active listeners
+	// so that the next node will not get the same
+	// when calling getAvailableTCPPorts
+	err = waitTCPPorts(ctx, ports...)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	// wait for the node to start
 	for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(50 * time.Millisecond) {
 		node.Client, err = rpc.Dial(conf.IPCEndpoint())
@@ -280,14 +292,12 @@ func newTestNode(t *testing.T, dir string) *testNode {
 	node := &testNode{Dir: dir}
 
 	// assign ports
-	httpPort, err := assignTCPPort()
-	if err != nil {
-		t.Fatal(err)
-	}
-	p2pPort, err := assignTCPPort()
+	ports, err := getAvailableTCPPorts(2)
 	if err != nil {
 		t.Fatal(err)
 	}
+	p2pPort := ports[0]
+	httpPort := ports[1]
 
 	// start the node
 	node.Cmd = runSwarm(t,
@@ -308,6 +318,17 @@ func newTestNode(t *testing.T, dir string) *testNode {
 		}
 	}()
 
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	// ensure that all ports have active listeners
+	// so that the next node will not get the same
+	// when calling getAvailableTCPPorts
+	err = waitTCPPorts(ctx, ports...)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	// wait for the node to start
 	for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(50 * time.Millisecond) {
 		node.Client, err = rpc.Dial(conf.IPCEndpoint())
@@ -343,15 +364,92 @@ func (n *testNode) Shutdown() {
 	}
 }
 
-func assignTCPPort() (string, error) {
-	l, err := net.Listen("tcp", "127.0.0.1:0")
-	if err != nil {
-		return "", err
+// getAvailableTCPPorts returns a set of ports that
+// nothing is listening on at the time.
+//
+// Function assignTCPPort cannot be called in sequence
+// and guardantee that the same port will be returned in
+// different calls as the listener is closed within the function,
+// not after all listeners are started and selected unique
+// available ports.
+func getAvailableTCPPorts(count int) (ports []string, err error) {
+	for i := 0; i < count; i++ {
+		l, err := net.Listen("tcp", "127.0.0.1:0")
+		if err != nil {
+			return nil, err
+		}
+		// defer close in the loop to be sure the same port will not
+		// be selected in the next iteration
+		defer l.Close()
+
+		_, port, err := net.SplitHostPort(l.Addr().String())
+		if err != nil {
+			return nil, err
+		}
+		ports = append(ports, port)
+	}
+	return ports, nil
+}
+
+// waitTCPPorts blocks until tcp connections can be
+// established on all provided ports. It runs all
+// ports dialers in parallel, and returns the first
+// encountered error.
+// See waitTCPPort also.
+func waitTCPPorts(ctx context.Context, ports ...string) error {
+	var err error
+	// mu locks err variable that is assigned in
+	// other goroutines
+	var mu sync.Mutex
+
+	// cancel is canceling all goroutines
+	// when the firs error is returned
+	// to prevent unnecessary waiting
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	var wg sync.WaitGroup
+	for _, port := range ports {
+		wg.Add(1)
+		go func(port string) {
+			defer wg.Done()
+
+			e := waitTCPPort(ctx, port)
+
+			mu.Lock()
+			defer mu.Unlock()
+			if e != nil && err == nil {
+				err = e
+				cancel()
+			}
+		}(port)
+	}
+	wg.Wait()
+
+	return err
+}
+
+// waitTCPPort blocks until tcp connection can be established
+// ona provided port. It has a 3 minute timeout as maximum,
+// to prevent long waiting, but it can be shortened with
+// a provided context instance. Dialer has a 10 second timeout
+// in every iteration, and connection refused error will be
+// retried in 100 milliseconds periods.
+func waitTCPPort(ctx context.Context, port string) error {
+	ctx, cancel := context.WithTimeout(ctx, 3*time.Minute)
+	defer cancel()
+
+	for {
+		c, err := (&net.Dialer{Timeout: 10 * time.Second}).DialContext(ctx, "tcp", "127.0.0.1:"+port)
+		if err != nil {
+			if operr, ok := err.(*net.OpError); ok {
+				if syserr, ok := operr.Err.(*os.SyscallError); ok && syserr.Err == syscall.ECONNREFUSED {
+					time.Sleep(100 * time.Millisecond)
+					continue
+				}
+			}
+			return err
+		}
+		return c.Close()
 	}
-	l.Close()
-	_, port, err := net.SplitHostPort(l.Addr().String())
-	if err != nil {
-		return "", err
-	}
-	return port, nil
 }

eth/api_tracer.go

@@ -297,7 +297,9 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
 				database.TrieDB().Reference(root, common.Hash{})
 			}
 			// Dereference all past tries we ourselves are done working with
-			database.TrieDB().Dereference(proot)
+			if proot != (common.Hash{}) {
+				database.TrieDB().Dereference(proot)
+			}
 			proot = root
 
 			// TODO(karalabe): Do we need the preimages? Won't they accumulate too much?
@@ -526,7 +528,9 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (*
 			return nil, err
 		}
 		database.TrieDB().Reference(root, common.Hash{})
-		database.TrieDB().Dereference(proot)
+		if proot != (common.Hash{}) {
+			database.TrieDB().Dereference(proot)
+		}
 		proot = root
 	}
 	nodes, imgs := database.TrieDB().Size()

p2p/discv5/net.go

@@ -1228,7 +1228,7 @@ func (net *Network) checkTopicRegister(data *topicRegister) (*pong, error) {
 	if rlpHash(data.Topics) != pongpkt.data.(*pong).TopicHash {
 		return nil, errors.New("topic hash mismatch")
 	}
-	if data.Idx < 0 || int(data.Idx) >= len(data.Topics) {
+	if int(data.Idx) < 0 || int(data.Idx) >= len(data.Topics) {
 		return nil, errors.New("topic index out of range")
 	}
 	return pongpkt.data.(*pong), nil

rpc/client_example_test.go

@@ -66,7 +66,7 @@ func subscribeBlocks(client *rpc.Client, subch chan Block) {
 	defer cancel()
 
 	// Subscribe to new blocks.
-	sub, err := client.EthSubscribe(ctx, subch, "newBlocks")
+	sub, err := client.EthSubscribe(ctx, subch, "newHeads")
 	if err != nil {
 		fmt.Println("subscribe error:", err)
 		return

swarm/api/api.go

@@ -525,6 +525,10 @@ func (a *API) GetDirectoryTar(ctx context.Context, uri *URI) (io.ReadCloser, err
 
 			return nil
 		})
+		// close tar writer before closing pipew
+		// to flush remaining data to pipew
+		// regardless of error value
+		tw.Close()
 		if err != nil {
 			apiGetTarFail.Inc(1)
 			pipew.CloseWithError(err)

trie/database.go

@@ -431,6 +431,11 @@ func (db *Database) reference(child common.Hash, parent common.Hash) {
 
 // Dereference removes an existing reference from a root node.
 func (db *Database) Dereference(root common.Hash) {
+	// Sanity check to ensure that the meta-root is not removed
+	if root == (common.Hash{}) {
+		log.Error("Attempted to dereference the trie cache meta root")
+		return
+	}
 	db.lock.Lock()
 	defer db.lock.Unlock()