all: fix tests on 32-bit and windows + minor rpc fixes #21871 (#1055)

cmd/XDC/accountcmd_test.go

@@ -202,9 +202,9 @@ func TestUnlockFlag(t *testing.T) {
 	datadir := tmpDatadirWithKeystore(t)
 	defer os.RemoveAll(datadir)
 	XDC := runXDC(t,
-		"js", "--datadir", datadir, "--nat", "none", "--nodiscover", "--maxpeers", "0",
+		"--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0", "--nousb", "--cache", "256", "--ipcdisable",
-		"--port", "0", "--unlock", "f466859ead1932d743d622cb74fc058882e8648a",
+		"--datadir", datadir, "--unlock", "f466859ead1932d743d622cb74fc058882e8648a",
-		"testdata/empty.js")
+		"js", "testdata/empty.js")
 	XDC.Expect(`
 Unlocking account f466859ead1932d743d622cb74fc058882e8648a | Attempt 1/3
 !! Unsupported terminal, password will be echoed.
@@ -227,8 +227,8 @@ func TestUnlockFlagWrongPassword(t *testing.T) {
 	datadir := tmpDatadirWithKeystore(t)
 	defer os.RemoveAll(datadir)
 	XDC := runXDC(t,
-		"--datadir", datadir, "--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0",
+		"--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0", "--nousb", "--cache", "128", "--ipcdisable",
-		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a")
+		"--datadir", datadir, "--unlock", "f466859ead1932d743d622cb74fc058882e8648a")
 	defer XDC.ExpectExit()
 	XDC.Expect(`
 Unlocking account f466859ead1932d743d622cb74fc058882e8648a | Attempt 1/3
@@ -247,9 +247,8 @@ func TestUnlockFlagMultiIndex(t *testing.T) {
 	datadir := tmpDatadirWithKeystore(t)
 	defer os.RemoveAll(datadir)
 	XDC := runXDC(t,
-		"js", "--datadir", datadir, "--nat", "none", "--nodiscover",
+		"--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0", "--nousb", "--cache", "128", "--ipcdisable",
-		"--maxpeers", "0", "--port", "0", "--unlock", "0,2",
+		"--datadir", datadir, "--unlock", "0,2", "js", "testdata/empty.js")
-		"testdata/empty.js")
 	XDC.Expect(`
 Unlocking account 0 | Attempt 1/3
 !! Unsupported terminal, password will be echoed.
@@ -275,9 +274,9 @@ func TestUnlockFlagPasswordFile(t *testing.T) {
 	datadir := tmpDatadirWithKeystore(t)
 	defer os.RemoveAll(datadir)
 	XDC := runXDC(t,
-		"js", "--datadir", datadir, "--nat", "none", "--nodiscover", "--maxpeers", "0",
+		"--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0", "--nousb", "--cache", "128", "--ipcdisable",
-		"--port", "0", "--password", "testdata/passwords.txt", "--unlock", "0,2",
+		"--datadir", datadir, "--password", "testdata/passwords.txt", "--unlock", "0,2",
-		"testdata/empty.js")
+		"js", "testdata/empty.js")
 	XDC.ExpectExit()
 
 	wantMessages := []string{
@@ -296,8 +295,8 @@ func TestUnlockFlagPasswordFileWrongPassword(t *testing.T) {
 	datadir := tmpDatadirWithKeystore(t)
 	defer os.RemoveAll(datadir)
 	XDC := runXDC(t,
-		"--datadir", datadir, "--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0",
+		"--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0", "--nousb", "--cache", "128", "--ipcdisable",
-		"--password", "testdata/wrong-passwords.txt", "--unlock", "0,2")
+		"--datadir", datadir, "--password", "testdata/wrong-passwords.txt", "--unlock", "0,2")
 	defer XDC.ExpectExit()
 	XDC.Expect(`
 Fatal: Failed to unlock account 0 (could not decrypt key with given password)
@@ -307,9 +306,9 @@ Fatal: Failed to unlock account 0 (could not decrypt key with given password)
 func TestUnlockFlagAmbiguous(t *testing.T) {
 	store := filepath.Join("..", "..", "accounts", "keystore", "testdata", "dupes")
 	XDC := runXDC(t,
-		"js", "--keystore", store, "--nat", "none", "--nodiscover", "--maxpeers", "0",
+		"--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0", "--nousb", "--cache", "128", "--ipcdisable",
-		"--port", "0", "--unlock", "f466859ead1932d743d622cb74fc058882e8648a",
+		"--keystore", store, "--unlock", "f466859ead1932d743d622cb74fc058882e8648a",
-		"testdata/empty.js")
+		"js", "testdata/empty.js")
 	defer XDC.ExpectExit()
 
 	// Helper for the expect template, returns absolute keystore path.
@@ -345,8 +344,8 @@ In order to avoid this warning, you need to remove the following duplicate key f
 func TestUnlockFlagAmbiguousWrongPassword(t *testing.T) {
 	store := filepath.Join("..", "..", "accounts", "keystore", "testdata", "dupes")
 	XDC := runXDC(t,
-		"--keystore", store, "--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0",
+		"--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0", "--nousb", "--cache", "128", "--ipcdisable",
-		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a")
+		"--keystore", store, "--unlock", "f466859ead1932d743d622cb74fc058882e8648a")
 	defer XDC.ExpectExit()
 
 	// Helper for the expect template, returns absolute keystore path.

internal/cmdtest/test_cmd.go

@@ -26,6 +26,7 @@ import (
 	"regexp"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"syscall"
 	"testing"
 	"text/template"
@@ -54,10 +55,13 @@ type TestCmd struct {
 	Err error
 }
 
+var id int32
+
 // Run exec's the current binary using name as argv[0] which will trigger the
 // reexec init function for that name (e.g. "geth-test" in cmd/geth/run_test.go)
 func (tt *TestCmd) Run(name string, args ...string) {
-	tt.stderr = &testlogger{t: tt.T}
+	id := atomic.AddInt32(&id, 1)
+	tt.stderr = &testlogger{t: tt.T, name: fmt.Sprintf("%d", id)}
 	tt.cmd = &exec.Cmd{
 		Path:   reexec.Self(),
 		Args:   append([]string{name}, args...),
@@ -237,16 +241,17 @@ func (tt *TestCmd) withKillTimeout(fn func()) {
 // testlogger logs all written lines via t.Log and also
 // collects them for later inspection.
 type testlogger struct {
-	t   *testing.T
+	t    *testing.T
-	mu  sync.Mutex
+	mu   sync.Mutex
-	buf bytes.Buffer
+	buf  bytes.Buffer
+	name string
 }
 
 func (tl *testlogger) Write(b []byte) (n int, err error) {
 	lines := bytes.Split(b, []byte("\n"))
 	for _, line := range lines {
 		if len(line) > 0 {
-			tl.t.Logf("(stderr) %s", line)
+			tl.t.Logf("(stderr:%v) %s", tl.name, line)
 		}
 	}
 	tl.mu.Lock()

node/rpcstack.go

@@ -593,6 +593,7 @@ func (is *ipcServer) start(apis []rpc.API) error {
 	}
 	listener, srv, err := rpc.StartIPCEndpoint(is.endpoint, apis)
 	if err != nil {
+		is.log.Warn("IPC opening failed", "url", is.endpoint, "error", err)
 		return err
 	}
 	is.log.Info("IPC endpoint opened", "url", is.endpoint)

trie/trie_test.go

@@ -468,6 +468,24 @@ type account struct {
 	Code    []byte
 }
 
+// Benchmarks the trie Commit following a Hash. Since the trie caches the result of any operation,
+// we cannot use b.N as the number of hashing rouns, since all rounds apart from
+// the first one will be NOOP. As such, we'll use b.N as the number of account to
+// insert into the trie before measuring the hashing.
+func BenchmarkCommitAfterHash(b *testing.B) {
+	b.Run("no-onleaf", func(b *testing.B) {
+		benchmarkCommitAfterHash(b, nil)
+	})
+	var a account
+	onleaf := func(path []byte, leaf []byte, parent common.Hash) error {
+		rlp.DecodeBytes(leaf, &a)
+		return nil
+	}
+	b.Run("with-onleaf", func(b *testing.B) {
+		benchmarkCommitAfterHash(b, onleaf)
+	})
+}
+
 func TestCommitAfterHash(t *testing.T) {
 	// Create a realistic account trie to hash
 	addresses, accounts := makeAccounts(1000)
@@ -479,7 +497,7 @@ func TestCommitAfterHash(t *testing.T) {
 	trie.Hash()
 	trie.Commit(nil)
 	root := trie.Hash()
-	exp := common.HexToHash("e5e9c29bb50446a4081e6d1d748d2892c6101c1e883a1f77cf21d4094b697822")
+	exp := common.HexToHash("72f9d3f3fe1e1dd7b8936442e7642aef76371472d94319900790053c493f3fe6")
 	if exp != root {
 		t.Errorf("got %x, exp %x", root, exp)
 	}
@@ -489,25 +507,73 @@ func TestCommitAfterHash(t *testing.T) {
 	}
 }
 
+func TestTinyTrie(t *testing.T) {
+	// Create a realistic account trie to hash
+	_, accounts := makeAccounts(5)
+	trie := newEmpty()
+	trie.Update(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001337"), accounts[3])
+	if exp, root := common.HexToHash("8c6a85a4d9fda98feff88450299e574e5378e32391f75a055d470ac0653f1005"), trie.Hash(); exp != root {
+		t.Errorf("1: got %x, exp %x", root, exp)
+	}
+	trie.Update(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001338"), accounts[4])
+	if exp, root := common.HexToHash("ec63b967e98a5720e7f720482151963982890d82c9093c0d486b7eb8883a66b1"), trie.Hash(); exp != root {
+		t.Errorf("2: got %x, exp %x", root, exp)
+	}
+	trie.Update(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001339"), accounts[4])
+	if exp, root := common.HexToHash("0608c1d1dc3905fa22204c7a0e43644831c3b6d3def0f274be623a948197e64a"), trie.Hash(); exp != root {
+		t.Errorf("3: got %x, exp %x", root, exp)
+	}
+	checktr, _ := New(common.Hash{}, trie.Db)
+	it := NewIterator(trie.NodeIterator(nil))
+	for it.Next() {
+		checktr.Update(it.Key, it.Value)
+	}
+	if troot, itroot := trie.Hash(), checktr.Hash(); troot != itroot {
+		t.Fatalf("hash mismatch in opItercheckhash, trie: %x, check: %x", troot, itroot)
+	}
+}
+
+func benchmarkCommitAfterHash(b *testing.B, onleaf LeafCallback) {
+	// Make the random benchmark deterministic
+	addresses, accounts := makeAccounts(b.N)
+	trie := newEmpty()
+	for i := 0; i < len(addresses); i++ {
+		trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i])
+	}
+	// Insert the accounts into the trie and hash it
+	trie.Hash()
+	b.ResetTimer()
+	b.ReportAllocs()
+	trie.Commit(onleaf)
+}
+
 func makeAccounts(size int) (addresses [][20]byte, accounts [][]byte) {
 	// Make the random benchmark deterministic
 	random := rand.New(rand.NewSource(0))
 	// Create a realistic account trie to hash
 	addresses = make([][20]byte, size)
 	for i := 0; i < len(addresses); i++ {
-		for j := 0; j < len(addresses[i]); j++ {
+		data := make([]byte, 20)
-			addresses[i][j] = byte(random.Intn(256))
+		random.Read(data)
-		}
+		copy(addresses[i][:], data)
 	}
 	accounts = make([][]byte, len(addresses))
 	for i := 0; i < len(accounts); i++ {
 		var (
-			nonce   = uint64(random.Int63())
+			nonce = uint64(random.Int63())
-			balance = new(big.Int).Rand(random, new(big.Int).Exp(common.Big2, common.Big256, nil))
+			root  = types.EmptyRootHash
-			root    = types.EmptyRootHash
+			code  = crypto.Keccak256(nil)
-			code    = crypto.Keccak256(nil)
 		)
-		accounts[i], _ = rlp.EncodeToBytes(&account{nonce, balance, root, code})
+		// The big.Rand function is not deterministic with regards to 64 vs 32 bit systems,
+		// and will consume different amount of data from the rand source.
+		//balance = new(big.Int).Rand(random, new(big.Int).Exp(common.Big2, common.Big256, nil))
+		// Therefore, we instead just read via byte buffer
+		numBytes := random.Uint32() % 33 // [0, 32] bytes
+		balanceBytes := make([]byte, numBytes)
+		random.Read(balanceBytes)
+		balance := new(big.Int).SetBytes(balanceBytes)
+		data, _ := rlp.EncodeToBytes(&account{nonce, balance, root, code})
+		accounts[i] = data
 	}
 	return addresses, accounts
 }