trie: store nodes as pointers

This avoids memory copies when unwrapping node interface values.

name      old time/op  new time/op  delta
Get        388ns ± 8%   215ns ± 2%  -44.56%  (p=0.000 n=15+15)
GetDB      363ns ± 3%   202ns ± 2%  -44.21%  (p=0.000 n=15+15)
UpdateBE  1.57µs ± 2%  1.29µs ± 3%  -17.80%  (p=0.000 n=13+15)
UpdateLE  1.92µs ± 2%  1.61µs ± 2%  -16.25%  (p=0.000 n=14+14)
HashBE    2.16µs ± 6%  2.18µs ± 6%     ~     (p=0.436 n=15+15)
HashLE    7.43µs ± 3%  7.21µs ± 3%   -2.96%  (p=0.000 n=15+13)

trie/hasher.go

@@ -62,16 +62,18 @@ func (h *hasher) hash(n node, db DatabaseWriter, force bool) (node, node, error)
 	if err != nil {
 		return hashNode{}, n, err
 	}
-	// Cache the hash and RLP blob of the ndoe for later reuse
+	// Cache the hash of the ndoe for later reuse.
 	if hash, ok := hashed.(hashNode); ok && !force {
 		switch cached := cached.(type) {
-		case shortNode:
+		case *shortNode:
+			cached = cached.copy()
 			cached.hash = hash
 			if db != nil {
 				cached.dirty = false
 			}
 			return hashed, cached, nil
-		case fullNode:
+		case *fullNode:
+			cached = cached.copy()
 			cached.hash = hash
 			if db != nil {
 				cached.dirty = false
@@ -89,40 +91,43 @@ func (h *hasher) hashChildren(original node, db DatabaseWriter) (node, node, err
 	var err error
 
 	switch n := original.(type) {
-	case shortNode:
+	case *shortNode:
 		// Hash the short node's child, caching the newly hashed subtree
-		cached := n
+		collapsed, cached := n.copy(), n.copy()
-		cached.Key = common.CopyBytes(cached.Key)
+		collapsed.Key = compactEncode(n.Key)
+		cached.Key = common.CopyBytes(n.Key)
 
-		n.Key = compactEncode(n.Key)
 		if _, ok := n.Val.(valueNode); !ok {
-			if n.Val, cached.Val, err = h.hash(n.Val, db, false); err != nil {
+			collapsed.Val, cached.Val, err = h.hash(n.Val, db, false)
-				return n, original, err
+			if err != nil {
+				return original, original, err
 			}
 		}
-		if n.Val == nil {
+		if collapsed.Val == nil {
-			n.Val = valueNode(nil) // Ensure that nil children are encoded as empty strings.
+			collapsed.Val = valueNode(nil) // Ensure that nil children are encoded as empty strings.
 		}
-		return n, cached, nil
+		return collapsed, cached, nil
 
-	case fullNode:
+	case *fullNode:
 		// Hash the full node's children, caching the newly hashed subtrees
-		cached := fullNode{dirty: n.dirty}
+		collapsed := n.copy()
+		cached := &fullNode{dirty: n.dirty}
 
 		for i := 0; i < 16; i++ {
 			if n.Children[i] != nil {
-				if n.Children[i], cached.Children[i], err = h.hash(n.Children[i], db, false); err != nil {
+				collapsed.Children[i], cached.Children[i], err = h.hash(n.Children[i], db, false)
-					return n, original, err
+				if err != nil {
+					return original, original, err
 				}
 			} else {
-				n.Children[i] = valueNode(nil) // Ensure that nil children are encoded as empty strings.
+				collapsed.Children[i] = valueNode(nil) // Ensure that nil children are encoded as empty strings.
 			}
 		}
 		cached.Children[16] = n.Children[16]
-		if n.Children[16] == nil {
+		if collapsed.Children[16] == nil {
-			n.Children[16] = valueNode(nil)
+			collapsed.Children[16] = valueNode(nil)
 		}
-		return n, cached, nil
+		return collapsed, cached, nil
 
 	default:
 		// Value and hash nodes don't have children so they're left as were
@@ -140,6 +145,7 @@ func (h *hasher) store(n node, db DatabaseWriter, force bool) (node, error) {
 	if err := rlp.Encode(h.tmp, n); err != nil {
 		panic("encode error: " + err.Error())
 	}
+
 	if h.tmp.Len() < 32 && !force {
 		return n, nil // Nodes smaller than 32 bytes are stored inside their parent
 	}

trie/iterator.go

@@ -56,11 +56,11 @@ func (it *Iterator) makeKey() []byte {
 	key := it.keyBuf[:0]
 	for _, se := range it.nodeIt.stack {
 		switch node := se.node.(type) {
-		case fullNode:
+		case *fullNode:
 			if se.child <= 16 {
 				key = append(key, byte(se.child))
 			}
-		case shortNode:
+		case *shortNode:
 			if hasTerm(node.Key) {
 				key = append(key, node.Key[:len(node.Key)-1]...)
 			} else {
@@ -148,7 +148,7 @@ func (it *NodeIterator) step() error {
 		if (ancestor == common.Hash{}) {
 			ancestor = parent.parent
 		}
-		if node, ok := parent.node.(fullNode); ok {
+		if node, ok := parent.node.(*fullNode); ok {
 			// Full node, traverse all children, then the node itself
 			if parent.child >= len(node.Children) {
 				break
@@ -164,7 +164,7 @@ func (it *NodeIterator) step() error {
 					break
 				}
 			}
-		} else if node, ok := parent.node.(shortNode); ok {
+		} else if node, ok := parent.node.(*shortNode); ok {
 			// Short node, traverse the pointer singleton child, then the node itself
 			if parent.child >= 0 {
 				break

trie/node.go

@@ -49,23 +49,26 @@ type (
 )
 
 // EncodeRLP encodes a full node into the consensus RLP format.
-func (n fullNode) EncodeRLP(w io.Writer) error {
+func (n *fullNode) EncodeRLP(w io.Writer) error {
 	return rlp.Encode(w, n.Children)
 }
 
+func (n *fullNode) copy() *fullNode   { copy := *n; return &copy }
+func (n *shortNode) copy() *shortNode { copy := *n; return &copy }
+
 // Cache accessors to retrieve precalculated values (avoid lengthy type switches).
-func (n fullNode) cache() (hashNode, bool)  { return n.hash, n.dirty }
+func (n *fullNode) cache() (hashNode, bool)  { return n.hash, n.dirty }
-func (n shortNode) cache() (hashNode, bool) { return n.hash, n.dirty }
+func (n *shortNode) cache() (hashNode, bool) { return n.hash, n.dirty }
-func (n hashNode) cache() (hashNode, bool)  { return nil, true }
+func (n hashNode) cache() (hashNode, bool)   { return nil, true }
-func (n valueNode) cache() (hashNode, bool) { return nil, true }
+func (n valueNode) cache() (hashNode, bool)  { return nil, true }
 
 // Pretty printing.
-func (n fullNode) String() string  { return n.fstring("") }
+func (n *fullNode) String() string  { return n.fstring("") }
-func (n shortNode) String() string { return n.fstring("") }
+func (n *shortNode) String() string { return n.fstring("") }
-func (n hashNode) String() string  { return n.fstring("") }
+func (n hashNode) String() string   { return n.fstring("") }
-func (n valueNode) String() string { return n.fstring("") }
+func (n valueNode) String() string  { return n.fstring("") }
 
-func (n fullNode) fstring(ind string) string {
+func (n *fullNode) fstring(ind string) string {
 	resp := fmt.Sprintf("[\n%s  ", ind)
 	for i, node := range n.Children {
 		if node == nil {
@@ -76,7 +79,7 @@ func (n fullNode) fstring(ind string) string {
 	}
 	return resp + fmt.Sprintf("\n%s] ", ind)
 }
-func (n shortNode) fstring(ind string) string {
+func (n *shortNode) fstring(ind string) string {
 	return fmt.Sprintf("{%x: %v} ", n.Key, n.Val.fstring(ind+"  "))
 }
 func (n hashNode) fstring(ind string) string {
@@ -127,17 +130,17 @@ func decodeShort(hash, buf, elems []byte) (node, error) {
 		if err != nil {
 			return nil, fmt.Errorf("invalid value node: %v", err)
 		}
-		return shortNode{key, valueNode(val), hash, false}, nil
+		return &shortNode{key, valueNode(val), hash, false}, nil
 	}
 	r, _, err := decodeRef(rest)
 	if err != nil {
 		return nil, wrapError(err, "val")
 	}
-	return shortNode{key, r, hash, false}, nil
+	return &shortNode{key, r, hash, false}, nil
 }
 
-func decodeFull(hash, buf, elems []byte) (fullNode, error) {
+func decodeFull(hash, buf, elems []byte) (*fullNode, error) {
-	n := fullNode{hash: hash}
+	n := &fullNode{hash: hash}
 	for i := 0; i < 16; i++ {
 		cld, rest, err := decodeRef(elems)
 		if err != nil {

trie/proof.go

@@ -44,7 +44,7 @@ func (t *Trie) Prove(key []byte) []rlp.RawValue {
 	tn := t.root
 	for len(key) > 0 && tn != nil {
 		switch n := tn.(type) {
-		case shortNode:
+		case *shortNode:
 			if len(key) < len(n.Key) || !bytes.Equal(n.Key, key[:len(n.Key)]) {
 				// The trie doesn't contain the key.
 				tn = nil
@@ -53,7 +53,7 @@ func (t *Trie) Prove(key []byte) []rlp.RawValue {
 				key = key[len(n.Key):]
 			}
 			nodes = append(nodes, n)
-		case fullNode:
+		case *fullNode:
 			tn = n.Children[key[0]]
 			key = key[1:]
 			nodes = append(nodes, n)
@@ -130,13 +130,13 @@ func VerifyProof(rootHash common.Hash, key []byte, proof []rlp.RawValue) (value
 func get(tn node, key []byte) ([]byte, node) {
 	for len(key) > 0 {
 		switch n := tn.(type) {
-		case shortNode:
+		case *shortNode:
 			if len(key) < len(n.Key) || !bytes.Equal(n.Key, key[:len(n.Key)]) {
 				return nil, nil
 			}
 			tn = n.Val
 			key = key[len(n.Key):]
-		case fullNode:
+		case *fullNode:
 			tn = n.Children[key[0]]
 			key = key[1:]
 		case hashNode:

trie/sync.go

@@ -212,12 +212,12 @@ func (s *TrieSync) children(req *request) ([]*request, error) {
 	children := []child{}
 
 	switch node := (*req.object).(type) {
-	case shortNode:
+	case *shortNode:
 		children = []child{{
 			node:  &node.Val,
 			depth: req.depth + len(node.Key),
 		}}
-	case fullNode:
+	case *fullNode:
 		for i := 0; i < 17; i++ {
 			if node.Children[i] != nil {
 				children = append(children, child{

trie/trie.go

@@ -120,27 +120,25 @@ func (t *Trie) tryGet(origNode node, key []byte, pos int) (value []byte, newnode
 		return nil, nil, false, nil
 	case valueNode:
 		return n, n, false, nil
-	case shortNode:
+	case *shortNode:
 		if len(key)-pos < len(n.Key) || !bytes.Equal(n.Key, key[pos:pos+len(n.Key)]) {
 			// key not found in trie
 			return nil, n, false, nil
 		}
 		value, newnode, didResolve, err = t.tryGet(n.Val, key, pos+len(n.Key))
 		if err == nil && didResolve {
+			n = n.copy()
 			n.Val = newnode
-			return value, n, didResolve, err
-		} else {
-			return value, origNode, didResolve, err
 		}
-	case fullNode:
+		return value, n, didResolve, err
-		child := n.Children[key[pos]]
+	case *fullNode:
-		value, newnode, didResolve, err = t.tryGet(child, key, pos+1)
+		value, newnode, didResolve, err = t.tryGet(n.Children[key[pos]], key, pos+1)
 		if err == nil && didResolve {
+			n = n.copy()
 			n.Children[key[pos]] = newnode
-			return value, n, didResolve, err
+
-		} else {
-			return value, origNode, didResolve, err
 		}
+		return value, n, didResolve, err
 	case hashNode:
 		child, err := t.resolveHash(n, key[:pos], key[pos:])
 		if err != nil {
@@ -199,22 +197,19 @@ func (t *Trie) insert(n node, prefix, key []byte, value node) (bool, node, error
 		return true, value, nil
 	}
 	switch n := n.(type) {
-	case shortNode:
+	case *shortNode:
 		matchlen := prefixLen(key, n.Key)
 		// If the whole key matches, keep this short node as is
 		// and only update the value.
 		if matchlen == len(n.Key) {
 			dirty, nn, err := t.insert(n.Val, append(prefix, key[:matchlen]...), key[matchlen:], value)
-			if err != nil {
+			if !dirty || err != nil {
-				return false, nil, err
+				return false, n, err
 			}
-			if !dirty {
+			return true, &shortNode{n.Key, nn, nil, true}, nil
-				return false, n, nil
-			}
-			return true, shortNode{n.Key, nn, nil, true}, nil
 		}
 		// Otherwise branch out at the index where they differ.
-		branch := fullNode{dirty: true}
+		branch := &fullNode{dirty: true}
 		var err error
 		_, branch.Children[n.Key[matchlen]], err = t.insert(nil, append(prefix, n.Key[:matchlen+1]...), n.Key[matchlen+1:], n.Val)
 		if err != nil {
@@ -229,21 +224,19 @@ func (t *Trie) insert(n node, prefix, key []byte, value node) (bool, node, error
 			return true, branch, nil
 		}
 		// Otherwise, replace it with a short node leading up to the branch.
-		return true, shortNode{key[:matchlen], branch, nil, true}, nil
+		return true, &shortNode{key[:matchlen], branch, nil, true}, nil
 
-	case fullNode:
+	case *fullNode:
 		dirty, nn, err := t.insert(n.Children[key[0]], append(prefix, key[0]), key[1:], value)
-		if err != nil {
+		if !dirty || err != nil {
-			return false, nil, err
+			return false, n, err
-		}
-		if !dirty {
-			return false, n, nil
 		}
+		n = n.copy()
 		n.Children[key[0]], n.hash, n.dirty = nn, nil, true
 		return true, n, nil
 
 	case nil:
-		return true, shortNode{key, value, nil, true}, nil
+		return true, &shortNode{key, value, nil, true}, nil
 
 	case hashNode:
 		// We've hit a part of the trie that isn't loaded yet. Load
@@ -254,11 +247,8 @@ func (t *Trie) insert(n node, prefix, key []byte, value node) (bool, node, error
 			return false, nil, err
 		}
 		dirty, nn, err := t.insert(rn, prefix, key, value)
-		if err != nil {
+		if !dirty || err != nil {
-			return false, nil, err
+			return false, rn, err
-		}
-		if !dirty {
-			return false, rn, nil
 		}
 		return true, nn, nil
 
@@ -291,7 +281,7 @@ func (t *Trie) TryDelete(key []byte) error {
 // nodes on the way up after deleting recursively.
 func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) {
 	switch n := n.(type) {
-	case shortNode:
+	case *shortNode:
 		matchlen := prefixLen(key, n.Key)
 		if matchlen < len(n.Key) {
 			return false, n, nil // don't replace n on mismatch
@@ -304,33 +294,28 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) {
 		// subtrie must contain at least two other values with keys
 		// longer than n.Key.
 		dirty, child, err := t.delete(n.Val, append(prefix, key[:len(n.Key)]...), key[len(n.Key):])
-		if err != nil {
+		if !dirty || err != nil {
-			return false, nil, err
+			return false, n, err
-		}
-		if !dirty {
-			return false, n, nil
 		}
 		switch child := child.(type) {
-		case shortNode:
+		case *shortNode:
 			// Deleting from the subtrie reduced it to another
 			// short node. Merge the nodes to avoid creating a
 			// shortNode{..., shortNode{...}}. Use concat (which
 			// always creates a new slice) instead of append to
 			// avoid modifying n.Key since it might be shared with
 			// other nodes.
-			return true, shortNode{concat(n.Key, child.Key...), child.Val, nil, true}, nil
+			return true, &shortNode{concat(n.Key, child.Key...), child.Val, nil, true}, nil
 		default:
-			return true, shortNode{n.Key, child, nil, true}, nil
+			return true, &shortNode{n.Key, child, nil, true}, nil
 		}
 
-	case fullNode:
+	case *fullNode:
 		dirty, nn, err := t.delete(n.Children[key[0]], append(prefix, key[0]), key[1:])
-		if err != nil {
+		if !dirty || err != nil {
-			return false, nil, err
+			return false, n, err
-		}
-		if !dirty {
-			return false, n, nil
 		}
+		n = n.copy()
 		n.Children[key[0]], n.hash, n.dirty = nn, nil, true
 
 		// Check how many non-nil entries are left after deleting and
@@ -365,14 +350,14 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) {
 				if err != nil {
 					return false, nil, err
 				}
-				if cnode, ok := cnode.(shortNode); ok {
+				if cnode, ok := cnode.(*shortNode); ok {
 					k := append([]byte{byte(pos)}, cnode.Key...)
-					return true, shortNode{k, cnode.Val, nil, true}, nil
+					return true, &shortNode{k, cnode.Val, nil, true}, nil
 				}
 			}
 			// Otherwise, n is replaced by a one-nibble short node
 			// containing the child.
-			return true, shortNode{[]byte{byte(pos)}, n.Children[pos], nil, true}, nil
+			return true, &shortNode{[]byte{byte(pos)}, n.Children[pos], nil, true}, nil
 		}
 		// n still contains at least two values and cannot be reduced.
 		return true, n, nil
@@ -392,11 +377,8 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) {
 			return false, nil, err
 		}
 		dirty, nn, err := t.delete(rn, prefix, key)
-		if err != nil {
+		if !dirty || err != nil {
-			return false, nil, err
+			return false, rn, err
-		}
-		if !dirty {
-			return false, rn, nil
 		}
 		return true, nn, nil