From db4aaedcbdb409e17ea3de161e7b24a80ba0a58c Mon Sep 17 00:00:00 2001 From: obscuren Date: Thu, 8 Jan 2015 11:47:04 +0100 Subject: [PATCH 01/13] Moved ptrie => trie. Removed old trie --- core/types/derive_sha.go | 4 +- ptrie/iterator.go | 115 ---- ptrie/trie.go | 335 ------------ ptrie/trie_test.go | 259 ---------- state/state_object.go | 6 +- state/statedb.go | 9 +- tests/helper/trie.go | 6 +- {ptrie => trie}/cache.go | 2 +- {ptrie => trie}/fullnode.go | 2 +- {ptrie => trie}/hashnode.go | 2 +- trie/iterator.go | 154 +++--- {ptrie => trie}/iterator_test.go | 2 +- trie/main_test.go | 9 - {ptrie => trie}/node.go | 2 +- {ptrie => trie}/shortnode.go | 8 +- trie/trie.go | 863 ++++++++++--------------------- trie/trie_test.go | 521 +++++++------------ {ptrie => trie}/valuenode.go | 2 +- types/ethereum.go | 1 - 19 files changed, 558 insertions(+), 1744 deletions(-) delete mode 100644 ptrie/iterator.go delete mode 100644 ptrie/trie.go delete mode 100644 ptrie/trie_test.go rename {ptrie => trie}/cache.go (98%) rename {ptrie => trie}/fullnode.go (98%) rename {ptrie => trie}/hashnode.go (97%) rename {ptrie => trie}/iterator_test.go (97%) delete mode 100644 trie/main_test.go rename {ptrie => trie}/node.go (98%) rename {ptrie => trie}/shortnode.go (78%) rename {ptrie => trie}/valuenode.go (97%) delete mode 100644 types/ethereum.go diff --git a/core/types/derive_sha.go b/core/types/derive_sha.go index 0e286bd8b2..b2c4422103 100644 --- a/core/types/derive_sha.go +++ b/core/types/derive_sha.go @@ -3,7 +3,7 @@ package types import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethutil" - "github.com/ethereum/go-ethereum/ptrie" + "github.com/ethereum/go-ethereum/trie" ) type DerivableList interface { @@ -13,7 +13,7 @@ type DerivableList interface { func DeriveSha(list DerivableList) []byte { db, _ := ethdb.NewMemDatabase() - trie := ptrie.New(nil, db) + trie := trie.New(nil, db) for i := 0; i < list.Len(); i++ { trie.Update(ethutil.Encode(i), list.GetRlp(i)) } diff --git a/ptrie/iterator.go b/ptrie/iterator.go deleted file mode 100644 index 787ba09c02..0000000000 --- a/ptrie/iterator.go +++ /dev/null @@ -1,115 +0,0 @@ -package ptrie - -import ( - "bytes" - - "github.com/ethereum/go-ethereum/trie" -) - -type Iterator struct { - trie *Trie - - Key []byte - Value []byte -} - -func NewIterator(trie *Trie) *Iterator { - return &Iterator{trie: trie, Key: make([]byte, 32)} -} - -func (self *Iterator) Next() bool { - self.trie.mu.Lock() - defer self.trie.mu.Unlock() - - key := trie.RemTerm(trie.CompactHexDecode(string(self.Key))) - k := self.next(self.trie.root, key) - - self.Key = []byte(trie.DecodeCompact(k)) - - return len(k) > 0 - -} - -func (self *Iterator) next(node Node, key []byte) []byte { - if node == nil { - return nil - } - - switch node := node.(type) { - case *FullNode: - if len(key) > 0 { - k := self.next(node.branch(key[0]), key[1:]) - if k != nil { - return append([]byte{key[0]}, k...) - } - } - - var r byte - if len(key) > 0 { - r = key[0] + 1 - } - - for i := r; i < 16; i++ { - k := self.key(node.branch(byte(i))) - if k != nil { - return append([]byte{i}, k...) - } - } - - case *ShortNode: - k := trie.RemTerm(node.Key()) - if vnode, ok := node.Value().(*ValueNode); ok { - if bytes.Compare([]byte(k), key) > 0 { - self.Value = vnode.Val() - return k - } - } else { - cnode := node.Value() - - var ret []byte - skey := key[len(k):] - if trie.BeginsWith(key, k) { - ret = self.next(cnode, skey) - } else if bytes.Compare(k, key[:len(k)]) > 0 { - ret = self.key(node) - } - - if ret != nil { - return append(k, ret...) - } - } - } - - return nil -} - -func (self *Iterator) key(node Node) []byte { - switch node := node.(type) { - case *ShortNode: - // Leaf node - if vnode, ok := node.Value().(*ValueNode); ok { - k := trie.RemTerm(node.Key()) - self.Value = vnode.Val() - - return k - } else { - k := trie.RemTerm(node.Key()) - return append(k, self.key(node.Value())...) - } - case *FullNode: - if node.Value() != nil { - self.Value = node.Value().(*ValueNode).Val() - - return []byte{16} - } - - for i := 0; i < 16; i++ { - k := self.key(node.branch(byte(i))) - if k != nil { - return append([]byte{byte(i)}, k...) - } - } - } - - return nil -} diff --git a/ptrie/trie.go b/ptrie/trie.go deleted file mode 100644 index 5c83b57d05..0000000000 --- a/ptrie/trie.go +++ /dev/null @@ -1,335 +0,0 @@ -package ptrie - -import ( - "bytes" - "container/list" - "fmt" - "sync" - - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethutil" - "github.com/ethereum/go-ethereum/trie" -) - -func ParanoiaCheck(t1 *Trie, backend Backend) (bool, *Trie) { - t2 := New(nil, backend) - - it := t1.Iterator() - for it.Next() { - t2.Update(it.Key, it.Value) - } - - return bytes.Equal(t2.Hash(), t1.Hash()), t2 -} - -type Trie struct { - mu sync.Mutex - root Node - roothash []byte - cache *Cache - - revisions *list.List -} - -func New(root []byte, backend Backend) *Trie { - trie := &Trie{} - trie.revisions = list.New() - trie.roothash = root - trie.cache = NewCache(backend) - - if root != nil { - value := ethutil.NewValueFromBytes(trie.cache.Get(root)) - trie.root = trie.mknode(value) - } - - return trie -} - -func (self *Trie) Iterator() *Iterator { - return NewIterator(self) -} - -func (self *Trie) Copy() *Trie { - return New(self.roothash, self.cache.backend) -} - -// Legacy support -func (self *Trie) Root() []byte { return self.Hash() } -func (self *Trie) Hash() []byte { - var hash []byte - if self.root != nil { - t := self.root.Hash() - if byts, ok := t.([]byte); ok && len(byts) > 0 { - hash = byts - } else { - hash = crypto.Sha3(ethutil.Encode(self.root.RlpData())) - } - } else { - hash = crypto.Sha3(ethutil.Encode("")) - } - - if !bytes.Equal(hash, self.roothash) { - self.revisions.PushBack(self.roothash) - self.roothash = hash - } - - return hash -} -func (self *Trie) Commit() { - self.mu.Lock() - defer self.mu.Unlock() - - // Hash first - self.Hash() - - self.cache.Flush() -} - -// Reset should only be called if the trie has been hashed -func (self *Trie) Reset() { - self.mu.Lock() - defer self.mu.Unlock() - - self.cache.Reset() - - if self.revisions.Len() > 0 { - revision := self.revisions.Remove(self.revisions.Back()).([]byte) - self.roothash = revision - } - value := ethutil.NewValueFromBytes(self.cache.Get(self.roothash)) - self.root = self.mknode(value) -} - -func (self *Trie) UpdateString(key, value string) Node { return self.Update([]byte(key), []byte(value)) } -func (self *Trie) Update(key, value []byte) Node { - self.mu.Lock() - defer self.mu.Unlock() - - k := trie.CompactHexDecode(string(key)) - - if len(value) != 0 { - self.root = self.insert(self.root, k, &ValueNode{self, value}) - } else { - self.root = self.delete(self.root, k) - } - - return self.root -} - -func (self *Trie) GetString(key string) []byte { return self.Get([]byte(key)) } -func (self *Trie) Get(key []byte) []byte { - self.mu.Lock() - defer self.mu.Unlock() - - k := trie.CompactHexDecode(string(key)) - - n := self.get(self.root, k) - if n != nil { - return n.(*ValueNode).Val() - } - - return nil -} - -func (self *Trie) DeleteString(key string) Node { return self.Delete([]byte(key)) } -func (self *Trie) Delete(key []byte) Node { - self.mu.Lock() - defer self.mu.Unlock() - - k := trie.CompactHexDecode(string(key)) - self.root = self.delete(self.root, k) - - return self.root -} - -func (self *Trie) insert(node Node, key []byte, value Node) Node { - if len(key) == 0 { - return value - } - - if node == nil { - return NewShortNode(self, key, value) - } - - switch node := node.(type) { - case *ShortNode: - k := node.Key() - cnode := node.Value() - if bytes.Equal(k, key) { - return NewShortNode(self, key, value) - } - - var n Node - matchlength := trie.MatchingNibbleLength(key, k) - if matchlength == len(k) { - n = self.insert(cnode, key[matchlength:], value) - } else { - pnode := self.insert(nil, k[matchlength+1:], cnode) - nnode := self.insert(nil, key[matchlength+1:], value) - fulln := NewFullNode(self) - fulln.set(k[matchlength], pnode) - fulln.set(key[matchlength], nnode) - n = fulln - } - if matchlength == 0 { - return n - } - - return NewShortNode(self, key[:matchlength], n) - - case *FullNode: - cpy := node.Copy().(*FullNode) - cpy.set(key[0], self.insert(node.branch(key[0]), key[1:], value)) - - return cpy - - default: - panic(fmt.Sprintf("%T: invalid node: %v", node, node)) - } -} - -func (self *Trie) get(node Node, key []byte) Node { - if len(key) == 0 { - return node - } - - if node == nil { - return nil - } - - switch node := node.(type) { - case *ShortNode: - k := node.Key() - cnode := node.Value() - - if len(key) >= len(k) && bytes.Equal(k, key[:len(k)]) { - return self.get(cnode, key[len(k):]) - } - - return nil - case *FullNode: - return self.get(node.branch(key[0]), key[1:]) - default: - panic(fmt.Sprintf("%T: invalid node: %v", node, node)) - } -} - -func (self *Trie) delete(node Node, key []byte) Node { - if len(key) == 0 && node == nil { - return nil - } - - switch node := node.(type) { - case *ShortNode: - k := node.Key() - cnode := node.Value() - if bytes.Equal(key, k) { - return nil - } else if bytes.Equal(key[:len(k)], k) { - child := self.delete(cnode, key[len(k):]) - - var n Node - switch child := child.(type) { - case *ShortNode: - nkey := append(k, child.Key()...) - n = NewShortNode(self, nkey, child.Value()) - case *FullNode: - sn := NewShortNode(self, node.Key(), child) - sn.key = node.key - n = sn - } - - return n - } else { - return node - } - - case *FullNode: - n := node.Copy().(*FullNode) - n.set(key[0], self.delete(n.branch(key[0]), key[1:])) - - pos := -1 - for i := 0; i < 17; i++ { - if n.branch(byte(i)) != nil { - if pos == -1 { - pos = i - } else { - pos = -2 - } - } - } - - var nnode Node - if pos == 16 { - nnode = NewShortNode(self, []byte{16}, n.branch(byte(pos))) - } else if pos >= 0 { - cnode := n.branch(byte(pos)) - switch cnode := cnode.(type) { - case *ShortNode: - // Stitch keys - k := append([]byte{byte(pos)}, cnode.Key()...) - nnode = NewShortNode(self, k, cnode.Value()) - case *FullNode: - nnode = NewShortNode(self, []byte{byte(pos)}, n.branch(byte(pos))) - } - } else { - nnode = n - } - - return nnode - case nil: - return nil - default: - panic(fmt.Sprintf("%T: invalid node: %v (%v)", node, node, key)) - } -} - -// casting functions and cache storing -func (self *Trie) mknode(value *ethutil.Value) Node { - l := value.Len() - switch l { - case 0: - return nil - case 2: - // A value node may consists of 2 bytes. - if value.Get(0).Len() != 0 { - return NewShortNode(self, trie.CompactDecode(string(value.Get(0).Bytes())), self.mknode(value.Get(1))) - } - case 17: - fnode := NewFullNode(self) - for i := 0; i < l; i++ { - fnode.set(byte(i), self.mknode(value.Get(i))) - } - return fnode - case 32: - return &HashNode{value.Bytes()} - } - - return &ValueNode{self, value.Bytes()} -} - -func (self *Trie) trans(node Node) Node { - switch node := node.(type) { - case *HashNode: - value := ethutil.NewValueFromBytes(self.cache.Get(node.key)) - return self.mknode(value) - default: - return node - } -} - -func (self *Trie) store(node Node) interface{} { - data := ethutil.Encode(node) - if len(data) >= 32 { - key := crypto.Sha3(data) - self.cache.Put(key, data) - - return key - } - - return node.RlpData() -} - -func (self *Trie) PrintRoot() { - fmt.Println(self.root) -} diff --git a/ptrie/trie_test.go b/ptrie/trie_test.go deleted file mode 100644 index 63a8ed36e6..0000000000 --- a/ptrie/trie_test.go +++ /dev/null @@ -1,259 +0,0 @@ -package ptrie - -import ( - "bytes" - "fmt" - "testing" - - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethutil" -) - -type Db map[string][]byte - -func (self Db) Get(k []byte) ([]byte, error) { return self[string(k)], nil } -func (self Db) Put(k, v []byte) { self[string(k)] = v } - -// Used for testing -func NewEmpty() *Trie { - return New(nil, make(Db)) -} - -func TestEmptyTrie(t *testing.T) { - trie := NewEmpty() - res := trie.Hash() - exp := crypto.Sha3(ethutil.Encode("")) - if !bytes.Equal(res, exp) { - t.Errorf("expected %x got %x", exp, res) - } -} - -func TestInsert(t *testing.T) { - trie := NewEmpty() - - trie.UpdateString("doe", "reindeer") - trie.UpdateString("dog", "puppy") - trie.UpdateString("dogglesworth", "cat") - - exp := ethutil.Hex2Bytes("8aad789dff2f538bca5d8ea56e8abe10f4c7ba3a5dea95fea4cd6e7c3a1168d3") - root := trie.Hash() - if !bytes.Equal(root, exp) { - t.Errorf("exp %x got %x", exp, root) - } - - trie = NewEmpty() - trie.UpdateString("A", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") - - exp = ethutil.Hex2Bytes("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab") - root = trie.Hash() - if !bytes.Equal(root, exp) { - t.Errorf("exp %x got %x", exp, root) - } -} - -func TestGet(t *testing.T) { - trie := NewEmpty() - - trie.UpdateString("doe", "reindeer") - trie.UpdateString("dog", "puppy") - trie.UpdateString("dogglesworth", "cat") - - res := trie.GetString("dog") - if !bytes.Equal(res, []byte("puppy")) { - t.Errorf("expected puppy got %x", res) - } - - unknown := trie.GetString("unknown") - if unknown != nil { - t.Errorf("expected nil got %x", unknown) - } -} - -func TestDelete(t *testing.T) { - trie := NewEmpty() - - vals := []struct{ k, v string }{ - {"do", "verb"}, - {"ether", "wookiedoo"}, - {"horse", "stallion"}, - {"shaman", "horse"}, - {"doge", "coin"}, - {"ether", ""}, - {"dog", "puppy"}, - {"shaman", ""}, - } - for _, val := range vals { - if val.v != "" { - trie.UpdateString(val.k, val.v) - } else { - trie.DeleteString(val.k) - } - } - - hash := trie.Hash() - exp := ethutil.Hex2Bytes("5991bb8c6514148a29db676a14ac506cd2cd5775ace63c30a4fe457715e9ac84") - if !bytes.Equal(hash, exp) { - t.Errorf("expected %x got %x", exp, hash) - } -} - -func TestEmptyValues(t *testing.T) { - trie := NewEmpty() - - vals := []struct{ k, v string }{ - {"do", "verb"}, - {"ether", "wookiedoo"}, - {"horse", "stallion"}, - {"shaman", "horse"}, - {"doge", "coin"}, - {"ether", ""}, - {"dog", "puppy"}, - {"shaman", ""}, - } - for _, val := range vals { - trie.UpdateString(val.k, val.v) - } - - hash := trie.Hash() - exp := ethutil.Hex2Bytes("5991bb8c6514148a29db676a14ac506cd2cd5775ace63c30a4fe457715e9ac84") - if !bytes.Equal(hash, exp) { - t.Errorf("expected %x got %x", exp, hash) - } -} - -func TestReplication(t *testing.T) { - trie := NewEmpty() - vals := []struct{ k, v string }{ - {"do", "verb"}, - {"ether", "wookiedoo"}, - {"horse", "stallion"}, - {"shaman", "horse"}, - {"doge", "coin"}, - {"ether", ""}, - {"dog", "puppy"}, - {"shaman", ""}, - {"somethingveryoddindeedthis is", "myothernodedata"}, - } - for _, val := range vals { - trie.UpdateString(val.k, val.v) - } - trie.Commit() - - trie2 := New(trie.roothash, trie.cache.backend) - if string(trie2.GetString("horse")) != "stallion" { - t.Error("expected to have horse => stallion") - } - - hash := trie2.Hash() - exp := trie.Hash() - if !bytes.Equal(hash, exp) { - t.Errorf("root failure. expected %x got %x", exp, hash) - } - -} - -func TestReset(t *testing.T) { - trie := NewEmpty() - vals := []struct{ k, v string }{ - {"do", "verb"}, - {"ether", "wookiedoo"}, - {"horse", "stallion"}, - } - for _, val := range vals { - trie.UpdateString(val.k, val.v) - } - trie.Commit() - - before := ethutil.CopyBytes(trie.roothash) - trie.UpdateString("should", "revert") - trie.Hash() - // Should have no effect - trie.Hash() - trie.Hash() - // ### - - trie.Reset() - after := ethutil.CopyBytes(trie.roothash) - - if !bytes.Equal(before, after) { - t.Errorf("expected roots to be equal. %x - %x", before, after) - } -} - -func TestParanoia(t *testing.T) { - t.Skip() - trie := NewEmpty() - - vals := []struct{ k, v string }{ - {"do", "verb"}, - {"ether", "wookiedoo"}, - {"horse", "stallion"}, - {"shaman", "horse"}, - {"doge", "coin"}, - {"ether", ""}, - {"dog", "puppy"}, - {"shaman", ""}, - {"somethingveryoddindeedthis is", "myothernodedata"}, - } - for _, val := range vals { - trie.UpdateString(val.k, val.v) - } - trie.Commit() - - ok, t2 := ParanoiaCheck(trie, trie.cache.backend) - if !ok { - t.Errorf("trie paranoia check failed %x %x", trie.roothash, t2.roothash) - } -} - -// Not an actual test -func TestOutput(t *testing.T) { - t.Skip() - - base := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - trie := NewEmpty() - for i := 0; i < 50; i++ { - trie.UpdateString(fmt.Sprintf("%s%d", base, i), "valueeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee") - } - fmt.Println("############################## FULL ################################") - fmt.Println(trie.root) - - trie.Commit() - fmt.Println("############################## SMALL ################################") - trie2 := New(trie.roothash, trie.cache.backend) - trie2.GetString(base + "20") - fmt.Println(trie2.root) -} - -func BenchmarkGets(b *testing.B) { - trie := NewEmpty() - vals := []struct{ k, v string }{ - {"do", "verb"}, - {"ether", "wookiedoo"}, - {"horse", "stallion"}, - {"shaman", "horse"}, - {"doge", "coin"}, - {"ether", ""}, - {"dog", "puppy"}, - {"shaman", ""}, - {"somethingveryoddindeedthis is", "myothernodedata"}, - } - for _, val := range vals { - trie.UpdateString(val.k, val.v) - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - trie.Get([]byte("horse")) - } -} - -func BenchmarkUpdate(b *testing.B) { - trie := NewEmpty() - - b.ResetTimer() - for i := 0; i < b.N; i++ { - trie.UpdateString(fmt.Sprintf("aaaaaaaaa%d", i), "value") - } - trie.Hash() -} diff --git a/state/state_object.go b/state/state_object.go index c1c78bee02..913c57a316 100644 --- a/state/state_object.go +++ b/state/state_object.go @@ -6,7 +6,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethutil" - "github.com/ethereum/go-ethereum/ptrie" + "github.com/ethereum/go-ethereum/trie" ) type Code []byte @@ -152,7 +152,7 @@ func (self *StateObject) Sync() { } /* - valid, t2 := ptrie.ParanoiaCheck(self.State.trie, ethutil.Config.Db) + valid, t2 := trie.ParanoiaCheck(self.State.trie, ethutil.Config.Db) if !valid { statelogger.Infof("Warn: PARANOIA: Different state storage root during copy %x vs %x\n", self.State.Root(), t2.Root()) @@ -273,7 +273,7 @@ func (c *StateObject) Init() Code { return c.InitCode } -func (self *StateObject) Trie() *ptrie.Trie { +func (self *StateObject) Trie() *trie.Trie { return self.State.trie } diff --git a/state/statedb.go b/state/statedb.go index de73147905..3176ab7555 100644 --- a/state/statedb.go +++ b/state/statedb.go @@ -6,7 +6,7 @@ import ( "github.com/ethereum/go-ethereum/ethutil" "github.com/ethereum/go-ethereum/logger" - "github.com/ethereum/go-ethereum/ptrie" + "github.com/ethereum/go-ethereum/trie" ) var statelogger = logger.NewLogger("STATE") @@ -18,7 +18,7 @@ var statelogger = logger.NewLogger("STATE") // * Accounts type StateDB struct { db ethutil.Database - trie *ptrie.Trie + trie *trie.Trie stateObjects map[string]*StateObject @@ -30,9 +30,8 @@ type StateDB struct { } // Create a new state from a given trie -//func New(trie *ptrie.Trie) *StateDB { func New(root []byte, db ethutil.Database) *StateDB { - trie := ptrie.New(root, db) + trie := trie.New(root, db) return &StateDB{db: db, trie: trie, stateObjects: make(map[string]*StateObject), manifest: NewManifest(), refund: make(map[string]*big.Int)} } @@ -308,7 +307,7 @@ func (self *StateDB) Update(gasUsed *big.Int) { // FIXME trie delete is broken if deleted { - valid, t2 := ptrie.ParanoiaCheck(self.trie, self.db) + valid, t2 := trie.ParanoiaCheck(self.trie, self.db) if !valid { statelogger.Infof("Warn: PARANOIA: Different state root during copy %x vs %x\n", self.trie.Root(), t2.Root()) diff --git a/tests/helper/trie.go b/tests/helper/trie.go index 3cfb0bbe5e..9e666d333a 100644 --- a/tests/helper/trie.go +++ b/tests/helper/trie.go @@ -1,6 +1,6 @@ package helper -import "github.com/ethereum/go-ethereum/ptrie" +import "github.com/ethereum/go-ethereum/trie" type MemDatabase struct { db map[string][]byte @@ -24,8 +24,8 @@ func (db *MemDatabase) Print() {} func (db *MemDatabase) Close() {} func (db *MemDatabase) LastKnownTD() []byte { return nil } -func NewTrie() *ptrie.Trie { +func NewTrie() *trie.Trie { db, _ := NewMemDatabase() - return ptrie.New(nil, db) + return trie.New(nil, db) } diff --git a/ptrie/cache.go b/trie/cache.go similarity index 98% rename from ptrie/cache.go rename to trie/cache.go index 721dc4cf64..e03702b255 100644 --- a/ptrie/cache.go +++ b/trie/cache.go @@ -1,4 +1,4 @@ -package ptrie +package trie type Backend interface { Get([]byte) ([]byte, error) diff --git a/ptrie/fullnode.go b/trie/fullnode.go similarity index 98% rename from ptrie/fullnode.go rename to trie/fullnode.go index 4dd98049d5..ebbe7f3844 100644 --- a/ptrie/fullnode.go +++ b/trie/fullnode.go @@ -1,4 +1,4 @@ -package ptrie +package trie import "fmt" diff --git a/ptrie/hashnode.go b/trie/hashnode.go similarity index 97% rename from ptrie/hashnode.go rename to trie/hashnode.go index 4c17569d78..40ccd54c31 100644 --- a/ptrie/hashnode.go +++ b/trie/hashnode.go @@ -1,4 +1,4 @@ -package ptrie +package trie type HashNode struct { key []byte diff --git a/trie/iterator.go b/trie/iterator.go index 1114715a66..f0dae28bb0 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -1,124 +1,73 @@ package trie -/* -import ( - "bytes" - - "github.com/ethereum/go-ethereum/ethutil" -) - -type NodeType byte - -const ( - EmptyNode NodeType = iota - BranchNode - LeafNode - ExtNode -) - -func getType(node *ethutil.Value) NodeType { - if node.Len() == 0 { - return EmptyNode - } - - if node.Len() == 2 { - k := CompactDecode(node.Get(0).Str()) - if HasTerm(k) { - return LeafNode - } - - return ExtNode - } - - return BranchNode -} +import "bytes" type Iterator struct { - Path [][]byte trie *Trie Key []byte - Value *ethutil.Value + Value []byte } func NewIterator(trie *Trie) *Iterator { - return &Iterator{trie: trie} + return &Iterator{trie: trie, Key: make([]byte, 32)} } -func (self *Iterator) key(node *ethutil.Value, path [][]byte) []byte { - switch getType(node) { - case LeafNode: - k := RemTerm(CompactDecode(node.Get(0).Str())) +func (self *Iterator) Next() bool { + self.trie.mu.Lock() + defer self.trie.mu.Unlock() - self.Path = append(path, k) - self.Value = node.Get(1) + key := RemTerm(CompactHexDecode(string(self.Key))) + k := self.next(self.trie.root, key) - return k - case BranchNode: - if node.Get(16).Len() > 0 { - return []byte{16} - } + self.Key = []byte(DecodeCompact(k)) - for i := byte(0); i < 16; i++ { - o := self.key(self.trie.getNode(node.Get(int(i)).Raw()), append(path, []byte{i})) - if o != nil { - return append([]byte{i}, o...) - } - } - case ExtNode: - currKey := node.Get(0).Bytes() + return len(k) > 0 - return self.key(self.trie.getNode(node.Get(1).Raw()), append(path, currKey)) +} + +func (self *Iterator) next(node Node, key []byte) []byte { + if node == nil { + return nil } - return nil -} - -func (self *Iterator) next(node *ethutil.Value, key []byte, path [][]byte) []byte { - switch typ := getType(node); typ { - case EmptyNode: - return nil - case BranchNode: + switch node := node.(type) { + case *FullNode: if len(key) > 0 { - subNode := self.trie.getNode(node.Get(int(key[0])).Raw()) - - o := self.next(subNode, key[1:], append(path, key[:1])) - if o != nil { - return append([]byte{key[0]}, o...) + k := self.next(node.branch(key[0]), key[1:]) + if k != nil { + return append([]byte{key[0]}, k...) } } - var r byte = 0 + var r byte if len(key) > 0 { r = key[0] + 1 } for i := r; i < 16; i++ { - subNode := self.trie.getNode(node.Get(int(i)).Raw()) - o := self.key(subNode, append(path, []byte{i})) - if o != nil { - return append([]byte{i}, o...) + k := self.key(node.branch(byte(i))) + if k != nil { + return append([]byte{i}, k...) } } - case LeafNode, ExtNode: - k := RemTerm(CompactDecode(node.Get(0).Str())) - if typ == LeafNode { - if bytes.Compare([]byte(k), []byte(key)) > 0 { - self.Value = node.Get(1) - self.Path = append(path, k) + case *ShortNode: + k := RemTerm(node.Key()) + if vnode, ok := node.Value().(*ValueNode); ok { + if bytes.Compare([]byte(k), key) > 0 { + self.Value = vnode.Val() return k } } else { - subNode := self.trie.getNode(node.Get(1).Raw()) - subKey := key[len(k):] + cnode := node.Value() + var ret []byte + skey := key[len(k):] if BeginsWith(key, k) { - ret = self.next(subNode, subKey, append(path, k)) + ret = self.next(cnode, skey) } else if bytes.Compare(k, key[:len(k)]) > 0 { - ret = self.key(node, append(path, k)) - } else { - ret = nil + ret = self.key(node) } if ret != nil { @@ -130,16 +79,33 @@ func (self *Iterator) next(node *ethutil.Value, key []byte, path [][]byte) []byt return nil } -// Get the next in keys -func (self *Iterator) Next(key string) []byte { - self.trie.mut.Lock() - defer self.trie.mut.Unlock() +func (self *Iterator) key(node Node) []byte { + switch node := node.(type) { + case *ShortNode: + // Leaf node + if vnode, ok := node.Value().(*ValueNode); ok { + k := RemTerm(node.Key()) + self.Value = vnode.Val() - k := RemTerm(CompactHexDecode(key)) - n := self.next(self.trie.getNode(self.trie.Root), k, nil) + return k + } else { + k := RemTerm(node.Key()) + return append(k, self.key(node.Value())...) + } + case *FullNode: + if node.Value() != nil { + self.Value = node.Value().(*ValueNode).Val() - self.Key = []byte(DecodeCompact(n)) + return []byte{16} + } - return self.Key + for i := 0; i < 16; i++ { + k := self.key(node.branch(byte(i))) + if k != nil { + return append([]byte{byte(i)}, k...) + } + } + } + + return nil } -*/ diff --git a/ptrie/iterator_test.go b/trie/iterator_test.go similarity index 97% rename from ptrie/iterator_test.go rename to trie/iterator_test.go index acfc03d633..74d9e903cd 100644 --- a/ptrie/iterator_test.go +++ b/trie/iterator_test.go @@ -1,4 +1,4 @@ -package ptrie +package trie import "testing" diff --git a/trie/main_test.go b/trie/main_test.go deleted file mode 100644 index f6f64c06f7..0000000000 --- a/trie/main_test.go +++ /dev/null @@ -1,9 +0,0 @@ -package trie - -import ( - "testing" - - checker "gopkg.in/check.v1" -) - -func Test(t *testing.T) { checker.TestingT(t) } diff --git a/ptrie/node.go b/trie/node.go similarity index 98% rename from ptrie/node.go rename to trie/node.go index ab90a1a021..a1f68480f2 100644 --- a/ptrie/node.go +++ b/trie/node.go @@ -1,4 +1,4 @@ -package ptrie +package trie import "fmt" diff --git a/ptrie/shortnode.go b/trie/shortnode.go similarity index 78% rename from ptrie/shortnode.go rename to trie/shortnode.go index 73ff2914bf..f132b56d96 100644 --- a/ptrie/shortnode.go +++ b/trie/shortnode.go @@ -1,6 +1,4 @@ -package ptrie - -import "github.com/ethereum/go-ethereum/trie" +package trie type ShortNode struct { trie *Trie @@ -9,7 +7,7 @@ type ShortNode struct { } func NewShortNode(t *Trie, key []byte, value Node) *ShortNode { - return &ShortNode{t, []byte(trie.CompactEncode(key)), value} + return &ShortNode{t, []byte(CompactEncode(key)), value} } func (self *ShortNode) Value() Node { self.value = self.trie.trans(self.value) @@ -27,5 +25,5 @@ func (self *ShortNode) Hash() interface{} { } func (self *ShortNode) Key() []byte { - return trie.CompactDecode(string(self.key)) + return CompactDecode(string(self.key)) } diff --git a/trie/trie.go b/trie/trie.go index c9fd18e009..36f2af5d22 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -1,8 +1,8 @@ package trie -/* import ( "bytes" + "container/list" "fmt" "sync" @@ -10,618 +10,325 @@ import ( "github.com/ethereum/go-ethereum/ethutil" ) -func ParanoiaCheck(t1 *Trie) (bool, *Trie) { - t2 := New(ethutil.Config.Db, "") +func ParanoiaCheck(t1 *Trie, backend Backend) (bool, *Trie) { + t2 := New(nil, backend) - t1.NewIterator().Each(func(key string, v *ethutil.Value) { - t2.Update(key, v.Str()) - }) - - return bytes.Compare(t2.GetRoot(), t1.GetRoot()) == 0, t2 -} - -func (s *Cache) Len() int { - return len(s.nodes) -} - -// TODO -// A StateObject is an object that has a state root -// This is goig to be the object for the second level caching (the caching of object which have a state such as contracts) -type StateObject interface { - State() *Trie - Sync() - Undo() -} - -type Node struct { - Key []byte - Value *ethutil.Value - Dirty bool -} - -func NewNode(key []byte, val *ethutil.Value, dirty bool) *Node { - return &Node{Key: key, Value: val, Dirty: dirty} -} - -func (n *Node) Copy() *Node { - return NewNode(n.Key, n.Value, n.Dirty) -} - -type Cache struct { - nodes map[string]*Node - db ethutil.Database - IsDirty bool -} - -func NewCache(db ethutil.Database) *Cache { - return &Cache{db: db, nodes: make(map[string]*Node)} -} - -func (cache *Cache) PutValue(v interface{}, force bool) interface{} { - value := ethutil.NewValue(v) - - enc := value.Encode() - if len(enc) >= 32 || force { - sha := crypto.Sha3(enc) - - cache.nodes[string(sha)] = NewNode(sha, value, true) - cache.IsDirty = true - - return sha + it := t1.Iterator() + for it.Next() { + t2.Update(it.Key, it.Value) } - return v + return bytes.Equal(t2.Hash(), t1.Hash()), t2 } -func (cache *Cache) Put(v interface{}) interface{} { - return cache.PutValue(v, false) -} - -func (cache *Cache) Get(key []byte) *ethutil.Value { - // First check if the key is the cache - if cache.nodes[string(key)] != nil { - return cache.nodes[string(key)].Value - } - - // Get the key of the database instead and cache it - data, _ := cache.db.Get(key) - // Create the cached value - value := ethutil.NewValueFromBytes(data) - - defer func() { - if r := recover(); r != nil { - fmt.Println("RECOVER GET", cache, cache.nodes) - panic("bye") - } - }() - // Create caching node - cache.nodes[string(key)] = NewNode(key, value, true) - - return value -} - -func (cache *Cache) Delete(key []byte) { - delete(cache.nodes, string(key)) - - cache.db.Delete(key) -} - -func (cache *Cache) Commit() { - // Don't try to commit if it isn't dirty - if !cache.IsDirty { - return - } - - for key, node := range cache.nodes { - if node.Dirty { - cache.db.Put([]byte(key), node.Value.Encode()) - node.Dirty = false - } - } - cache.IsDirty = false - - // If the nodes grows beyond the 200 entries we simple empty it - // FIXME come up with something better - if len(cache.nodes) > 200 { - cache.nodes = make(map[string]*Node) - } -} - -func (cache *Cache) Undo() { - for key, node := range cache.nodes { - if node.Dirty { - delete(cache.nodes, key) - } - } - cache.IsDirty = false -} - -// A (modified) Radix Trie implementation. The Trie implements -// a caching mechanism and will used cached values if they are -// present. If a node is not present in the cache it will try to -// fetch it from the database and store the cached value. -// Please note that the data isn't persisted unless `Sync` is -// explicitly called. type Trie struct { - mut sync.RWMutex - prevRoot interface{} - Root interface{} - //db Database - cache *Cache + mu sync.Mutex + root Node + roothash []byte + cache *Cache + + revisions *list.List } -func copyRoot(root interface{}) interface{} { - var prevRootCopy interface{} - if b, ok := root.([]byte); ok { - prevRootCopy = ethutil.CopyBytes(b) - } else { - prevRootCopy = root - } +func New(root []byte, backend Backend) *Trie { + trie := &Trie{} + trie.revisions = list.New() + trie.roothash = root + trie.cache = NewCache(backend) - return prevRootCopy -} - -func New(db ethutil.Database, Root interface{}) *Trie { - // Make absolute sure the root is copied - r := copyRoot(Root) - p := copyRoot(Root) - - trie := &Trie{cache: NewCache(db), Root: r, prevRoot: p} - trie.setRoot(Root) - - return trie -} - -func (self *Trie) setRoot(root interface{}) { - switch t := root.(type) { - case string: - //if t == "" { - // root = crypto.Sha3(ethutil.Encode("")) - //} - self.Root = []byte(t) - case []byte: - self.Root = root - default: - self.Root = self.cache.PutValue(root, true) - } -} - -func (t *Trie) Update(key, value string) { - t.mut.Lock() - defer t.mut.Unlock() - - k := CompactHexDecode(key) - - var root interface{} - if value != "" { - root = t.UpdateState(t.Root, k, value) - } else { - root = t.deleteState(t.Root, k) - } - t.setRoot(root) -} - -func (t *Trie) Get(key string) string { - t.mut.Lock() - defer t.mut.Unlock() - - k := CompactHexDecode(key) - c := ethutil.NewValue(t.getState(t.Root, k)) - - return c.Str() -} - -func (t *Trie) Delete(key string) { - t.mut.Lock() - defer t.mut.Unlock() - - k := CompactHexDecode(key) - - root := t.deleteState(t.Root, k) - t.setRoot(root) -} - -func (self *Trie) GetRoot() []byte { - switch t := self.Root.(type) { - case string: - if t == "" { - return crypto.Sha3(ethutil.Encode("")) - } - return []byte(t) - case []byte: - if len(t) == 0 { - return crypto.Sha3(ethutil.Encode("")) - } - - return t - default: - panic(fmt.Sprintf("invalid root type %T (%v)", self.Root, self.Root)) - } -} - -// Simple compare function which creates a rlp value out of the evaluated objects -func (t *Trie) Cmp(trie *Trie) bool { - return ethutil.NewValue(t.Root).Cmp(ethutil.NewValue(trie.Root)) -} - -// Returns a copy of this trie -func (t *Trie) Copy() *Trie { - trie := New(t.cache.db, t.Root) - for key, node := range t.cache.nodes { - trie.cache.nodes[key] = node.Copy() + if root != nil { + value := ethutil.NewValueFromBytes(trie.cache.Get(root)) + trie.root = trie.mknode(value) } return trie } -// Save the cached value to the database. -func (t *Trie) Sync() { - t.cache.Commit() - t.prevRoot = copyRoot(t.Root) -} - -func (t *Trie) Undo() { - t.cache.Undo() - t.Root = t.prevRoot -} - -func (t *Trie) Cache() *Cache { - return t.cache -} - -func (t *Trie) getState(node interface{}, key []byte) interface{} { - n := ethutil.NewValue(node) - // Return the node if key is empty (= found) - if len(key) == 0 || n.IsNil() || n.Len() == 0 { - return node - } - - currentNode := t.getNode(node) - length := currentNode.Len() - - if length == 0 { - return "" - } else if length == 2 { - // Decode the key - k := CompactDecode(currentNode.Get(0).Str()) - v := currentNode.Get(1).Raw() - - if len(key) >= len(k) && bytes.Equal(k, key[:len(k)]) { //CompareIntSlice(k, key[:len(k)]) { - return t.getState(v, key[len(k):]) - } else { - return "" - } - } else if length == 17 { - return t.getState(currentNode.Get(int(key[0])).Raw(), key[1:]) - } - - // It shouldn't come this far - panic("unexpected return") -} - -func (t *Trie) getNode(node interface{}) *ethutil.Value { - n := ethutil.NewValue(node) - - if !n.Get(0).IsNil() { - return n - } - - str := n.Str() - if len(str) == 0 { - return n - } else if len(str) < 32 { - return ethutil.NewValueFromBytes([]byte(str)) - } - - data := t.cache.Get(n.Bytes()) - - return data -} - -func (t *Trie) UpdateState(node interface{}, key []byte, value string) interface{} { - return t.InsertState(node, key, value) -} - -func (t *Trie) Put(node interface{}) interface{} { - return t.cache.Put(node) - -} - -func EmptyStringSlice(l int) []interface{} { - slice := make([]interface{}, l) - for i := 0; i < l; i++ { - slice[i] = "" - } - return slice -} - -func (t *Trie) InsertState(node interface{}, key []byte, value interface{}) interface{} { - if len(key) == 0 { - return value - } - - // New node - n := ethutil.NewValue(node) - if node == nil || n.Len() == 0 { - newNode := []interface{}{CompactEncode(key), value} - - return t.Put(newNode) - } - - currentNode := t.getNode(node) - // Check for "special" 2 slice type node - if currentNode.Len() == 2 { - // Decode the key - - k := CompactDecode(currentNode.Get(0).Str()) - v := currentNode.Get(1).Raw() - - // Matching key pair (ie. there's already an object with this key) - if bytes.Equal(k, key) { //CompareIntSlice(k, key) { - newNode := []interface{}{CompactEncode(key), value} - return t.Put(newNode) - } - - var newHash interface{} - matchingLength := MatchingNibbleLength(key, k) - if matchingLength == len(k) { - // Insert the hash, creating a new node - newHash = t.InsertState(v, key[matchingLength:], value) - } else { - // Expand the 2 length slice to a 17 length slice - oldNode := t.InsertState("", k[matchingLength+1:], v) - newNode := t.InsertState("", key[matchingLength+1:], value) - // Create an expanded slice - scaledSlice := EmptyStringSlice(17) - // Set the copied and new node - scaledSlice[k[matchingLength]] = oldNode - scaledSlice[key[matchingLength]] = newNode - - newHash = t.Put(scaledSlice) - } - - if matchingLength == 0 { - // End of the chain, return - return newHash - } else { - newNode := []interface{}{CompactEncode(key[:matchingLength]), newHash} - return t.Put(newNode) - } - } else { - - // Copy the current node over to the new node and replace the first nibble in the key - newNode := EmptyStringSlice(17) - - for i := 0; i < 17; i++ { - cpy := currentNode.Get(i).Raw() - if cpy != nil { - newNode[i] = cpy - } - } - - newNode[key[0]] = t.InsertState(currentNode.Get(int(key[0])).Raw(), key[1:], value) - - return t.Put(newNode) - } - - panic("unexpected end") -} - -func (t *Trie) deleteState(node interface{}, key []byte) interface{} { - if len(key) == 0 { - return "" - } - - // New node - n := ethutil.NewValue(node) - //if node == nil || (n.Type() == reflect.String && (n.Str() == "" || n.Get(0).IsNil())) || n.Len() == 0 { - if node == nil || n.Len() == 0 { - //return nil - //fmt.Printf(" %x %d\n", n, len(n.Bytes())) - - return "" - } - - currentNode := t.getNode(node) - // Check for "special" 2 slice type node - if currentNode.Len() == 2 { - // Decode the key - k := CompactDecode(currentNode.Get(0).Str()) - v := currentNode.Get(1).Raw() - - // Matching key pair (ie. there's already an object with this key) - if bytes.Equal(k, key) { //CompareIntSlice(k, key) { - //fmt.Printf(" %x\n", v) - - return "" - } else if bytes.Equal(key[:len(k)], k) { //CompareIntSlice(key[:len(k)], k) { - hash := t.deleteState(v, key[len(k):]) - child := t.getNode(hash) - - var newNode []interface{} - if child.Len() == 2 { - newKey := append(k, CompactDecode(child.Get(0).Str())...) - newNode = []interface{}{CompactEncode(newKey), child.Get(1).Raw()} - } else { - newNode = []interface{}{currentNode.Get(0).Str(), hash} - } - - //fmt.Printf("%x\n", newNode) - - return t.Put(newNode) - } else { - return node - } - } else { - // Copy the current node over to the new node and replace the first nibble in the key - n := EmptyStringSlice(17) - var newNode []interface{} - - for i := 0; i < 17; i++ { - cpy := currentNode.Get(i).Raw() - if cpy != nil { - n[i] = cpy - } - } - - n[key[0]] = t.deleteState(n[key[0]], key[1:]) - amount := -1 - for i := 0; i < 17; i++ { - if n[i] != "" { - if amount == -1 { - amount = i - } else { - amount = -2 - } - } - } - if amount == 16 { - newNode = []interface{}{CompactEncode([]byte{16}), n[amount]} - } else if amount >= 0 { - child := t.getNode(n[amount]) - if child.Len() == 17 { - newNode = []interface{}{CompactEncode([]byte{byte(amount)}), n[amount]} - } else if child.Len() == 2 { - key := append([]byte{byte(amount)}, CompactDecode(child.Get(0).Str())...) - newNode = []interface{}{CompactEncode(key), child.Get(1).Str()} - } - - } else { - newNode = n - } - - //fmt.Printf("%x\n", newNode) - return t.Put(newNode) - } - - panic("unexpected return") -} - -type TrieIterator struct { - trie *Trie - key string - value string - - shas [][]byte - values []string - - lastNode []byte -} - -func (t *Trie) NewIterator() *TrieIterator { - return &TrieIterator{trie: t} -} - func (self *Trie) Iterator() *Iterator { return NewIterator(self) } -// Some time in the near future this will need refactoring :-) -// XXX Note to self, IsSlice == inline node. Str == sha3 to node -func (it *TrieIterator) workNode(currentNode *ethutil.Value) { - if currentNode.Len() == 2 { - k := CompactDecode(currentNode.Get(0).Str()) +func (self *Trie) Copy() *Trie { + return New(self.roothash, self.cache.backend) +} - if currentNode.Get(1).Str() == "" { - it.workNode(currentNode.Get(1)) +// Legacy support +func (self *Trie) Root() []byte { return self.Hash() } +func (self *Trie) Hash() []byte { + var hash []byte + if self.root != nil { + t := self.root.Hash() + if byts, ok := t.([]byte); ok && len(byts) > 0 { + hash = byts } else { - if k[len(k)-1] == 16 { - it.values = append(it.values, currentNode.Get(1).Str()) - } else { - it.shas = append(it.shas, currentNode.Get(1).Bytes()) - it.getNode(currentNode.Get(1).Bytes()) - } + hash = crypto.Sha3(ethutil.Encode(self.root.RlpData())) } } else { - for i := 0; i < currentNode.Len(); i++ { - if i == 16 && currentNode.Get(i).Len() != 0 { - it.values = append(it.values, currentNode.Get(i).Str()) - } else { - if currentNode.Get(i).Str() == "" { - it.workNode(currentNode.Get(i)) - } else { - val := currentNode.Get(i).Str() - if val != "" { - it.shas = append(it.shas, currentNode.Get(1).Bytes()) - it.getNode([]byte(val)) - } - } - } + hash = crypto.Sha3(ethutil.Encode("")) + } + + if !bytes.Equal(hash, self.roothash) { + self.revisions.PushBack(self.roothash) + self.roothash = hash + } + + return hash +} +func (self *Trie) Commit() { + self.mu.Lock() + defer self.mu.Unlock() + + // Hash first + self.Hash() + + self.cache.Flush() +} + +// Reset should only be called if the trie has been hashed +func (self *Trie) Reset() { + self.mu.Lock() + defer self.mu.Unlock() + + self.cache.Reset() + + if self.revisions.Len() > 0 { + revision := self.revisions.Remove(self.revisions.Back()).([]byte) + self.roothash = revision + } + value := ethutil.NewValueFromBytes(self.cache.Get(self.roothash)) + self.root = self.mknode(value) +} + +func (self *Trie) UpdateString(key, value string) Node { return self.Update([]byte(key), []byte(value)) } +func (self *Trie) Update(key, value []byte) Node { + self.mu.Lock() + defer self.mu.Unlock() + + k := CompactHexDecode(string(key)) + + if len(value) != 0 { + self.root = self.insert(self.root, k, &ValueNode{self, value}) + } else { + self.root = self.delete(self.root, k) + } + + return self.root +} + +func (self *Trie) GetString(key string) []byte { return self.Get([]byte(key)) } +func (self *Trie) Get(key []byte) []byte { + self.mu.Lock() + defer self.mu.Unlock() + + k := CompactHexDecode(string(key)) + + n := self.get(self.root, k) + if n != nil { + return n.(*ValueNode).Val() + } + + return nil +} + +func (self *Trie) DeleteString(key string) Node { return self.Delete([]byte(key)) } +func (self *Trie) Delete(key []byte) Node { + self.mu.Lock() + defer self.mu.Unlock() + + k := CompactHexDecode(string(key)) + self.root = self.delete(self.root, k) + + return self.root +} + +func (self *Trie) insert(node Node, key []byte, value Node) Node { + if len(key) == 0 { + return value + } + + if node == nil { + return NewShortNode(self, key, value) + } + + switch node := node.(type) { + case *ShortNode: + k := node.Key() + cnode := node.Value() + if bytes.Equal(k, key) { + return NewShortNode(self, key, value) } + + var n Node + matchlength := MatchingNibbleLength(key, k) + if matchlength == len(k) { + n = self.insert(cnode, key[matchlength:], value) + } else { + pnode := self.insert(nil, k[matchlength+1:], cnode) + nnode := self.insert(nil, key[matchlength+1:], value) + fulln := NewFullNode(self) + fulln.set(k[matchlength], pnode) + fulln.set(key[matchlength], nnode) + n = fulln + } + if matchlength == 0 { + return n + } + + return NewShortNode(self, key[:matchlength], n) + + case *FullNode: + cpy := node.Copy().(*FullNode) + cpy.set(key[0], self.insert(node.branch(key[0]), key[1:], value)) + + return cpy + + default: + panic(fmt.Sprintf("%T: invalid node: %v", node, node)) } } -func (it *TrieIterator) getNode(node []byte) { - currentNode := it.trie.cache.Get(node) - it.workNode(currentNode) -} +func (self *Trie) get(node Node, key []byte) Node { + if len(key) == 0 { + return node + } -func (it *TrieIterator) Collect() [][]byte { - if it.trie.Root == "" { + if node == nil { return nil } - it.getNode(ethutil.NewValue(it.trie.Root).Bytes()) + switch node := node.(type) { + case *ShortNode: + k := node.Key() + cnode := node.Value() - return it.shas -} - -func (it *TrieIterator) Purge() int { - shas := it.Collect() - for _, sha := range shas { - it.trie.cache.Delete(sha) - } - return len(it.values) -} - -func (it *TrieIterator) Key() string { - return "" -} - -func (it *TrieIterator) Value() string { - return "" -} - -type EachCallback func(key string, node *ethutil.Value) - -func (it *TrieIterator) Each(cb EachCallback) { - it.fetchNode(nil, ethutil.NewValue(it.trie.Root).Bytes(), cb) -} - -func (it *TrieIterator) fetchNode(key []byte, node []byte, cb EachCallback) { - it.iterateNode(key, it.trie.cache.Get(node), cb) -} - -func (it *TrieIterator) iterateNode(key []byte, currentNode *ethutil.Value, cb EachCallback) { - if currentNode.Len() == 2 { - k := CompactDecode(currentNode.Get(0).Str()) - - pk := append(key, k...) - if currentNode.Get(1).Len() != 0 && currentNode.Get(1).Str() == "" { - it.iterateNode(pk, currentNode.Get(1), cb) - } else { - if k[len(k)-1] == 16 { - cb(DecodeCompact(pk), currentNode.Get(1)) - } else { - it.fetchNode(pk, currentNode.Get(1).Bytes(), cb) - } + if len(key) >= len(k) && bytes.Equal(k, key[:len(k)]) { + return self.get(cnode, key[len(k):]) } - } else { - for i := 0; i < currentNode.Len(); i++ { - pk := append(key, byte(i)) - if i == 16 && currentNode.Get(i).Len() != 0 { - cb(DecodeCompact(pk), currentNode.Get(i)) - } else { - if currentNode.Get(i).Len() != 0 && currentNode.Get(i).Str() == "" { - it.iterateNode(pk, currentNode.Get(i), cb) + + return nil + case *FullNode: + return self.get(node.branch(key[0]), key[1:]) + default: + panic(fmt.Sprintf("%T: invalid node: %v", node, node)) + } +} + +func (self *Trie) delete(node Node, key []byte) Node { + if len(key) == 0 && node == nil { + return nil + } + + switch node := node.(type) { + case *ShortNode: + k := node.Key() + cnode := node.Value() + if bytes.Equal(key, k) { + return nil + } else if bytes.Equal(key[:len(k)], k) { + child := self.delete(cnode, key[len(k):]) + + var n Node + switch child := child.(type) { + case *ShortNode: + nkey := append(k, child.Key()...) + n = NewShortNode(self, nkey, child.Value()) + case *FullNode: + sn := NewShortNode(self, node.Key(), child) + sn.key = node.key + n = sn + } + + return n + } else { + return node + } + + case *FullNode: + n := node.Copy().(*FullNode) + n.set(key[0], self.delete(n.branch(key[0]), key[1:])) + + pos := -1 + for i := 0; i < 17; i++ { + if n.branch(byte(i)) != nil { + if pos == -1 { + pos = i } else { - val := currentNode.Get(i).Str() - if val != "" { - it.fetchNode(pk, []byte(val), cb) - } + pos = -2 } } } + + var nnode Node + if pos == 16 { + nnode = NewShortNode(self, []byte{16}, n.branch(byte(pos))) + } else if pos >= 0 { + cnode := n.branch(byte(pos)) + switch cnode := cnode.(type) { + case *ShortNode: + // Stitch keys + k := append([]byte{byte(pos)}, cnode.Key()...) + nnode = NewShortNode(self, k, cnode.Value()) + case *FullNode: + nnode = NewShortNode(self, []byte{byte(pos)}, n.branch(byte(pos))) + } + } else { + nnode = n + } + + return nnode + case nil: + return nil + default: + panic(fmt.Sprintf("%T: invalid node: %v (%v)", node, node, key)) } } -*/ + +// casting functions and cache storing +func (self *Trie) mknode(value *ethutil.Value) Node { + l := value.Len() + switch l { + case 0: + return nil + case 2: + // A value node may consists of 2 bytes. + if value.Get(0).Len() != 0 { + return NewShortNode(self, CompactDecode(string(value.Get(0).Bytes())), self.mknode(value.Get(1))) + } + case 17: + fnode := NewFullNode(self) + for i := 0; i < l; i++ { + fnode.set(byte(i), self.mknode(value.Get(i))) + } + return fnode + case 32: + return &HashNode{value.Bytes()} + } + + return &ValueNode{self, value.Bytes()} +} + +func (self *Trie) trans(node Node) Node { + switch node := node.(type) { + case *HashNode: + value := ethutil.NewValueFromBytes(self.cache.Get(node.key)) + return self.mknode(value) + default: + return node + } +} + +func (self *Trie) store(node Node) interface{} { + data := ethutil.Encode(node) + if len(data) >= 32 { + key := crypto.Sha3(data) + self.cache.Put(key, data) + + return key + } + + return node.RlpData() +} + +func (self *Trie) PrintRoot() { + fmt.Println(self.root) +} diff --git a/trie/trie_test.go b/trie/trie_test.go index 3abe56040f..ffb78d4f2b 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -1,345 +1,76 @@ package trie -/* import ( "bytes" - "encoding/hex" - "encoding/json" "fmt" - "io/ioutil" - "math/rand" - "net/http" "testing" - "time" - - checker "gopkg.in/check.v1" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethutil" ) -const LONG_WORD = "1234567890abcdefghijklmnopqrstuvwxxzABCEFGHIJKLMNOPQRSTUVWXYZ" +type Db map[string][]byte -type TrieSuite struct { - db *MemDatabase - trie *Trie +func (self Db) Get(k []byte) ([]byte, error) { return self[string(k)], nil } +func (self Db) Put(k, v []byte) { self[string(k)] = v } + +// Used for testing +func NewEmpty() *Trie { + return New(nil, make(Db)) } -type MemDatabase struct { - db map[string][]byte -} - -func NewMemDatabase() (*MemDatabase, error) { - db := &MemDatabase{db: make(map[string][]byte)} - return db, nil -} -func (db *MemDatabase) Put(key []byte, value []byte) { - db.db[string(key)] = value -} -func (db *MemDatabase) Get(key []byte) ([]byte, error) { - return db.db[string(key)], nil -} -func (db *MemDatabase) Delete(key []byte) error { - delete(db.db, string(key)) - return nil -} -func (db *MemDatabase) Print() {} -func (db *MemDatabase) Close() {} -func (db *MemDatabase) LastKnownTD() []byte { return nil } - -func NewTrie() (*MemDatabase, *Trie) { - db, _ := NewMemDatabase() - return db, New(db, "") -} - -func (s *TrieSuite) SetUpTest(c *checker.C) { - s.db, s.trie = NewTrie() -} - -func (s *TrieSuite) TestTrieSync(c *checker.C) { - s.trie.Update("dog", LONG_WORD) - c.Assert(s.db.db, checker.HasLen, 0, checker.Commentf("Expected no data in database")) - s.trie.Sync() - c.Assert(s.db.db, checker.HasLen, 3) -} - -func (s *TrieSuite) TestTrieDirtyTracking(c *checker.C) { - s.trie.Update("dog", LONG_WORD) - c.Assert(s.trie.cache.IsDirty, checker.Equals, true, checker.Commentf("Expected no data in database")) - - s.trie.Sync() - c.Assert(s.trie.cache.IsDirty, checker.Equals, false, checker.Commentf("Expected trie to be dirty")) - - s.trie.Update("test", LONG_WORD) - s.trie.cache.Undo() - c.Assert(s.trie.cache.IsDirty, checker.Equals, false) -} - -func (s *TrieSuite) TestTrieReset(c *checker.C) { - s.trie.Update("cat", LONG_WORD) - c.Assert(s.trie.cache.nodes, checker.HasLen, 1, checker.Commentf("Expected cached nodes")) - - s.trie.cache.Undo() - c.Assert(s.trie.cache.nodes, checker.HasLen, 0, checker.Commentf("Expected no nodes after undo")) -} - -func (s *TrieSuite) TestTrieGet(c *checker.C) { - s.trie.Update("cat", LONG_WORD) - x := s.trie.Get("cat") - c.Assert(x, checker.DeepEquals, LONG_WORD) -} - -func (s *TrieSuite) TestTrieUpdating(c *checker.C) { - s.trie.Update("cat", LONG_WORD) - s.trie.Update("cat", LONG_WORD+"1") - x := s.trie.Get("cat") - c.Assert(x, checker.DeepEquals, LONG_WORD+"1") -} - -func (s *TrieSuite) TestTrieCmp(c *checker.C) { - _, trie1 := NewTrie() - _, trie2 := NewTrie() - - trie1.Update("doge", LONG_WORD) - trie2.Update("doge", LONG_WORD) - c.Assert(trie1, checker.DeepEquals, trie2) - - trie1.Update("dog", LONG_WORD) - trie2.Update("cat", LONG_WORD) - c.Assert(trie1, checker.Not(checker.DeepEquals), trie2) -} - -func (s *TrieSuite) TestTrieDelete(c *checker.C) { - s.trie.Update("cat", LONG_WORD) - exp := s.trie.Root - s.trie.Update("dog", LONG_WORD) - s.trie.Delete("dog") - c.Assert(s.trie.Root, checker.DeepEquals, exp) - - s.trie.Update("dog", LONG_WORD) - exp = s.trie.Root - s.trie.Update("dude", LONG_WORD) - s.trie.Delete("dude") - c.Assert(s.trie.Root, checker.DeepEquals, exp) -} - -func (s *TrieSuite) TestTrieDeleteWithValue(c *checker.C) { - s.trie.Update("c", LONG_WORD) - exp := s.trie.Root - s.trie.Update("ca", LONG_WORD) - s.trie.Update("cat", LONG_WORD) - s.trie.Delete("ca") - s.trie.Delete("cat") - c.Assert(s.trie.Root, checker.DeepEquals, exp) -} - -func (s *TrieSuite) TestTriePurge(c *checker.C) { - s.trie.Update("c", LONG_WORD) - s.trie.Update("ca", LONG_WORD) - s.trie.Update("cat", LONG_WORD) - - lenBefore := len(s.trie.cache.nodes) - it := s.trie.NewIterator() - num := it.Purge() - c.Assert(num, checker.Equals, 3) - c.Assert(len(s.trie.cache.nodes), checker.Equals, lenBefore) -} - -func h(str string) string { - d, err := hex.DecodeString(str) - if err != nil { - panic(err) - } - - return string(d) -} - -func get(in string) (out string) { - if len(in) > 2 && in[:2] == "0x" { - out = h(in[2:]) - } else { - out = in - } - - return -} - -type TrieTest struct { - Name string - In map[string]string - Root string -} - -func CreateTest(name string, data []byte) (TrieTest, error) { - t := TrieTest{Name: name} - err := json.Unmarshal(data, &t) - if err != nil { - return TrieTest{}, fmt.Errorf("%v", err) - } - - return t, nil -} - -func CreateTests(uri string, cb func(TrieTest)) map[string]TrieTest { - resp, err := http.Get(uri) - if err != nil { - panic(err) - } - defer resp.Body.Close() - - data, err := ioutil.ReadAll(resp.Body) - - var objmap map[string]*json.RawMessage - err = json.Unmarshal(data, &objmap) - if err != nil { - panic(err) - } - - tests := make(map[string]TrieTest) - for name, testData := range objmap { - test, err := CreateTest(name, *testData) - if err != nil { - panic(err) - } - - if cb != nil { - cb(test) - } - tests[name] = test - } - - return tests -} - -func RandomData() [][]string { - data := [][]string{ - {"0x000000000000000000000000ec4f34c97e43fbb2816cfd95e388353c7181dab1", "0x4e616d6552656700000000000000000000000000000000000000000000000000"}, - {"0x0000000000000000000000000000000000000000000000000000000000000045", "0x22b224a1420a802ab51d326e29fa98e34c4f24ea"}, - {"0x0000000000000000000000000000000000000000000000000000000000000046", "0x67706c2076330000000000000000000000000000000000000000000000000000"}, - {"0x000000000000000000000000697c7b8c961b56f675d570498424ac8de1a918f6", "0x6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000"}, - {"0x0000000000000000000000007ef9e639e2733cb34e4dfc576d4b23f72db776b2", "0x4655474156000000000000000000000000000000000000000000000000000000"}, - {"0x6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000", "0x697c7b8c961b56f675d570498424ac8de1a918f6"}, - {"0x4655474156000000000000000000000000000000000000000000000000000000", "0x7ef9e639e2733cb34e4dfc576d4b23f72db776b2"}, - {"0x4e616d6552656700000000000000000000000000000000000000000000000000", "0xec4f34c97e43fbb2816cfd95e388353c7181dab1"}, - } - - var c [][]string - for len(data) != 0 { - e := rand.Intn(len(data)) - c = append(c, data[e]) - - copy(data[e:], data[e+1:]) - data[len(data)-1] = nil - data = data[:len(data)-1] - } - - return c -} - -const MaxTest = 1000 - -// This test insert data in random order and seeks to find indifferences between the different tries -func (s *TrieSuite) TestRegression(c *checker.C) { - rand.Seed(time.Now().Unix()) - - roots := make(map[string]int) - for i := 0; i < MaxTest; i++ { - _, trie := NewTrie() - data := RandomData() - - for _, test := range data { - trie.Update(test[0], test[1]) - } - trie.Delete("0x4e616d6552656700000000000000000000000000000000000000000000000000") - - roots[string(trie.Root.([]byte))] += 1 - } - - c.Assert(len(roots) <= 1, checker.Equals, true) - // if len(roots) > 1 { - // for root, num := range roots { - // t.Errorf("%x => %d\n", root, num) - // } - // } -} - -func (s *TrieSuite) TestDelete(c *checker.C) { - s.trie.Update("a", "jeffreytestlongstring") - s.trie.Update("aa", "otherstring") - s.trie.Update("aaa", "othermorestring") - s.trie.Update("aabbbbccc", "hithere") - s.trie.Update("abbcccdd", "hstanoehutnaheoustnh") - s.trie.Update("rnthaoeuabbcccdd", "hstanoehutnaheoustnh") - s.trie.Update("rneuabbcccdd", "hstanoehutnaheoustnh") - s.trie.Update("rneuabboeusntahoeucccdd", "hstanoehutnaheoustnh") - s.trie.Update("rnxabboeusntahoeucccdd", "hstanoehutnaheoustnh") - s.trie.Delete("aaboaestnuhbccc") - s.trie.Delete("a") - s.trie.Update("a", "nthaonethaosentuh") - s.trie.Update("c", "shtaosntehua") - s.trie.Delete("a") - s.trie.Update("aaaa", "testmegood") - - _, t2 := NewTrie() - s.trie.NewIterator().Each(func(key string, v *ethutil.Value) { - if key == "aaaa" { - t2.Update(key, v.Str()) - } else { - t2.Update(key, v.Str()) - } - }) - - a := ethutil.NewValue(s.trie.Root).Bytes() - b := ethutil.NewValue(t2.Root).Bytes() - - c.Assert(a, checker.DeepEquals, b) -} - -func (s *TrieSuite) TestTerminator(c *checker.C) { - key := CompactDecode("hello") - c.Assert(HasTerm(key), checker.Equals, true, checker.Commentf("Expected %v to have a terminator", key)) -} - -func (s *TrieSuite) TestIt(c *checker.C) { - s.trie.Update("cat", "cat") - s.trie.Update("doge", "doge") - s.trie.Update("wallace", "wallace") - it := s.trie.Iterator() - - inputs := []struct { - In, Out string - }{ - {"", "cat"}, - {"bobo", "cat"}, - {"c", "cat"}, - {"car", "cat"}, - {"catering", "doge"}, - {"w", "wallace"}, - {"wallace123", ""}, - } - - for _, test := range inputs { - res := string(it.Next(test.In)) - c.Assert(res, checker.Equals, test.Out) +func TestEmptyTrie(t *testing.T) { + trie := NewEmpty() + res := trie.Hash() + exp := crypto.Sha3(ethutil.Encode("")) + if !bytes.Equal(res, exp) { + t.Errorf("expected %x got %x", exp, res) } } -func (s *TrieSuite) TestBeginsWith(c *checker.C) { - a := CompactDecode("hello") - b := CompactDecode("hel") +func TestInsert(t *testing.T) { + trie := NewEmpty() - c.Assert(BeginsWith(a, b), checker.Equals, false) - c.Assert(BeginsWith(b, a), checker.Equals, true) + trie.UpdateString("doe", "reindeer") + trie.UpdateString("dog", "puppy") + trie.UpdateString("dogglesworth", "cat") + + exp := ethutil.Hex2Bytes("8aad789dff2f538bca5d8ea56e8abe10f4c7ba3a5dea95fea4cd6e7c3a1168d3") + root := trie.Hash() + if !bytes.Equal(root, exp) { + t.Errorf("exp %x got %x", exp, root) + } + + trie = NewEmpty() + trie.UpdateString("A", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + + exp = ethutil.Hex2Bytes("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab") + root = trie.Hash() + if !bytes.Equal(root, exp) { + t.Errorf("exp %x got %x", exp, root) + } } -func (s *TrieSuite) TestItems(c *checker.C) { - s.trie.Update("A", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") - exp := "d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab" +func TestGet(t *testing.T) { + trie := NewEmpty() - c.Assert(s.trie.GetRoot(), checker.DeepEquals, ethutil.Hex2Bytes(exp)) + trie.UpdateString("doe", "reindeer") + trie.UpdateString("dog", "puppy") + trie.UpdateString("dogglesworth", "cat") + + res := trie.GetString("dog") + if !bytes.Equal(res, []byte("puppy")) { + t.Errorf("expected puppy got %x", res) + } + + unknown := trie.GetString("unknown") + if unknown != nil { + t.Errorf("expected nil got %x", unknown) + } } -func TestOtherSomething(t *testing.T) { - _, trie := NewTrie() +func TestDelete(t *testing.T) { + trie := NewEmpty() vals := []struct{ k, v string }{ {"do", "verb"}, @@ -352,18 +83,46 @@ func TestOtherSomething(t *testing.T) { {"shaman", ""}, } for _, val := range vals { - trie.Update(val.k, val.v) + if val.v != "" { + trie.UpdateString(val.k, val.v) + } else { + trie.DeleteString(val.k) + } } + hash := trie.Hash() exp := ethutil.Hex2Bytes("5991bb8c6514148a29db676a14ac506cd2cd5775ace63c30a4fe457715e9ac84") - hash := trie.Root.([]byte) if !bytes.Equal(hash, exp) { t.Errorf("expected %x got %x", exp, hash) } } -func BenchmarkGets(b *testing.B) { - _, trie := NewTrie() +func TestEmptyValues(t *testing.T) { + trie := NewEmpty() + + vals := []struct{ k, v string }{ + {"do", "verb"}, + {"ether", "wookiedoo"}, + {"horse", "stallion"}, + {"shaman", "horse"}, + {"doge", "coin"}, + {"ether", ""}, + {"dog", "puppy"}, + {"shaman", ""}, + } + for _, val := range vals { + trie.UpdateString(val.k, val.v) + } + + hash := trie.Hash() + exp := ethutil.Hex2Bytes("5991bb8c6514148a29db676a14ac506cd2cd5775ace63c30a4fe457715e9ac84") + if !bytes.Equal(hash, exp) { + t.Errorf("expected %x got %x", exp, hash) + } +} + +func TestReplication(t *testing.T) { + trie := NewEmpty() vals := []struct{ k, v string }{ {"do", "verb"}, {"ether", "wookiedoo"}, @@ -376,21 +135,125 @@ func BenchmarkGets(b *testing.B) { {"somethingveryoddindeedthis is", "myothernodedata"}, } for _, val := range vals { - trie.Update(val.k, val.v) + trie.UpdateString(val.k, val.v) + } + trie.Commit() + + trie2 := New(trie.roothash, trie.cache.backend) + if string(trie2.GetString("horse")) != "stallion" { + t.Error("expected to have horse => stallion") + } + + hash := trie2.Hash() + exp := trie.Hash() + if !bytes.Equal(hash, exp) { + t.Errorf("root failure. expected %x got %x", exp, hash) + } + +} + +func TestReset(t *testing.T) { + trie := NewEmpty() + vals := []struct{ k, v string }{ + {"do", "verb"}, + {"ether", "wookiedoo"}, + {"horse", "stallion"}, + } + for _, val := range vals { + trie.UpdateString(val.k, val.v) + } + trie.Commit() + + before := ethutil.CopyBytes(trie.roothash) + trie.UpdateString("should", "revert") + trie.Hash() + // Should have no effect + trie.Hash() + trie.Hash() + // ### + + trie.Reset() + after := ethutil.CopyBytes(trie.roothash) + + if !bytes.Equal(before, after) { + t.Errorf("expected roots to be equal. %x - %x", before, after) + } +} + +func TestParanoia(t *testing.T) { + t.Skip() + trie := NewEmpty() + + vals := []struct{ k, v string }{ + {"do", "verb"}, + {"ether", "wookiedoo"}, + {"horse", "stallion"}, + {"shaman", "horse"}, + {"doge", "coin"}, + {"ether", ""}, + {"dog", "puppy"}, + {"shaman", ""}, + {"somethingveryoddindeedthis is", "myothernodedata"}, + } + for _, val := range vals { + trie.UpdateString(val.k, val.v) + } + trie.Commit() + + ok, t2 := ParanoiaCheck(trie, trie.cache.backend) + if !ok { + t.Errorf("trie paranoia check failed %x %x", trie.roothash, t2.roothash) + } +} + +// Not an actual test +func TestOutput(t *testing.T) { + t.Skip() + + base := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + trie := NewEmpty() + for i := 0; i < 50; i++ { + trie.UpdateString(fmt.Sprintf("%s%d", base, i), "valueeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee") + } + fmt.Println("############################## FULL ################################") + fmt.Println(trie.root) + + trie.Commit() + fmt.Println("############################## SMALL ################################") + trie2 := New(trie.roothash, trie.cache.backend) + trie2.GetString(base + "20") + fmt.Println(trie2.root) +} + +func BenchmarkGets(b *testing.B) { + trie := NewEmpty() + vals := []struct{ k, v string }{ + {"do", "verb"}, + {"ether", "wookiedoo"}, + {"horse", "stallion"}, + {"shaman", "horse"}, + {"doge", "coin"}, + {"ether", ""}, + {"dog", "puppy"}, + {"shaman", ""}, + {"somethingveryoddindeedthis is", "myothernodedata"}, + } + for _, val := range vals { + trie.UpdateString(val.k, val.v) } b.ResetTimer() for i := 0; i < b.N; i++ { - trie.Get("horse") + trie.Get([]byte("horse")) } } func BenchmarkUpdate(b *testing.B) { - _, trie := NewTrie() + trie := NewEmpty() b.ResetTimer() for i := 0; i < b.N; i++ { - trie.Update(fmt.Sprintf("aaaaaaaaaaaaaaa%d", i), "value") + trie.UpdateString(fmt.Sprintf("aaaaaaaaa%d", i), "value") } + trie.Hash() } -*/ diff --git a/ptrie/valuenode.go b/trie/valuenode.go similarity index 97% rename from ptrie/valuenode.go rename to trie/valuenode.go index c593eb6c60..689befb2ad 100644 --- a/ptrie/valuenode.go +++ b/trie/valuenode.go @@ -1,4 +1,4 @@ -package ptrie +package trie type ValueNode struct { trie *Trie diff --git a/types/ethereum.go b/types/ethereum.go deleted file mode 100644 index ab1254f4c2..0000000000 --- a/types/ethereum.go +++ /dev/null @@ -1 +0,0 @@ -package types From b25126a277da5253e4ce175dec5b68ccdf1b9b0f Mon Sep 17 00:00:00 2001 From: obscuren Date: Thu, 8 Jan 2015 16:37:06 +0100 Subject: [PATCH 02/13] Minor fixed and additions for block proc * Path check length * Genesis include TD * Output TD on last block --- core/chain_manager.go | 4 ++-- core/genesis.go | 1 + ethutil/path.go | 2 +- ethutil/rlp.go | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/core/chain_manager.go b/core/chain_manager.go index 2d4001f0f9..5ff3b88c90 100644 --- a/core/chain_manager.go +++ b/core/chain_manager.go @@ -139,7 +139,7 @@ func (bc *ChainManager) setLastBlock() { bc.Reset() } - chainlogger.Infof("Last block (#%d) %x\n", bc.lastBlockNumber, bc.currentBlock.Hash()) + chainlogger.Infof("Last block (#%d) %x TD=%v\n", bc.lastBlockNumber, bc.currentBlock.Hash(), bc.td) } // Block creation & chain handling @@ -215,7 +215,7 @@ func (bc *ChainManager) insert(block *types.Block) { func (bc *ChainManager) write(block *types.Block) { bc.writeBlockInfo(block) - encodedBlock := ethutil.Encode(block) + encodedBlock := ethutil.Encode(block.RlpDataForStorage()) bc.db.Put(block.Hash(), encodedBlock) } diff --git a/core/genesis.go b/core/genesis.go index 1590818a8f..d9edaace23 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -25,6 +25,7 @@ func GenesisBlock(db ethutil.Database) *types.Block { genesis.Header().GasLimit = big.NewInt(1000000) genesis.Header().GasUsed = ethutil.Big0 genesis.Header().Time = 0 + genesis.Td = ethutil.Big0 genesis.SetUncles([]*types.Header{}) genesis.SetTransactions(types.Transactions{}) diff --git a/ethutil/path.go b/ethutil/path.go index f64e3849e0..e545c87315 100644 --- a/ethutil/path.go +++ b/ethutil/path.go @@ -12,7 +12,7 @@ func ExpandHomePath(p string) (path string) { path = p // Check in case of paths like "/something/~/something/" - if path[:2] == "~/" { + if len(path) > 1 && path[:2] == "~/" { usr, _ := user.Current() dir := usr.HomeDir diff --git a/ethutil/rlp.go b/ethutil/rlp.go index 1bc1a58a71..0cb0d611ce 100644 --- a/ethutil/rlp.go +++ b/ethutil/rlp.go @@ -137,7 +137,7 @@ func Encode(object interface{}) []byte { case byte: buff.Write(Encode(big.NewInt(int64(t)))) case *big.Int: - // Not sure how this is possible while we check for + // Not sure how this is possible while we check for nil if t == nil { buff.WriteByte(0xc0) } else { From ee84b202473a5b629a3a2b43154edb989629ddfc Mon Sep 17 00:00:00 2001 From: obscuren Date: Thu, 8 Jan 2015 16:38:24 +0100 Subject: [PATCH 03/13] Reworking GUI interaction. Fixed javascript inject. Closes #132 --- cmd/mist/assets/qml/browser.qml | 12 +- cmd/mist/assets/qml/main.qml | 4 +- cmd/mist/gui.go | 242 +++++++++++++++++++------------- 3 files changed, 158 insertions(+), 100 deletions(-) diff --git a/cmd/mist/assets/qml/browser.qml b/cmd/mist/assets/qml/browser.qml index abaab4f15c..4cf6b24704 100644 --- a/cmd/mist/assets/qml/browser.qml +++ b/cmd/mist/assets/qml/browser.qml @@ -59,7 +59,7 @@ Rectangle { } Component.onCompleted: { - webview.url = "http://etherian.io" + webview.url = "/Users/jeffrey/test.html" } signal messages(var messages, int id); @@ -109,7 +109,8 @@ Rectangle { leftMargin: 5 rightMargin: 5 } - text: "http://etherian.io" + //text: "http://etherian.io" + text: webview.url; id: uriNav y: parent.height / 2 - this.height / 2 @@ -151,6 +152,10 @@ Rectangle { window.open(request.url.toString()); } + function injectJs(js) { + experimental.evaluateJavaScript(js) + } + function sendMessage(data) { webview.experimental.postMessage(JSON.stringify(data)) } @@ -159,8 +164,7 @@ Rectangle { experimental.preferences.javascriptEnabled: true experimental.preferences.navigatorQtObjectEnabled: true experimental.preferences.developerExtrasEnabled: true - //experimental.userScripts: ["../ext/qt_messaging_adapter.js", "../ext/q.js", "../ext/big.js", "../ext/string.js", "../ext/html_messaging.js"] - experimental.userScripts: ["../ext/q.js", "../ext/eth.js/main.js", "../ext/eth.js/qt.js", "../ext/setup.js"] + //experimental.userScripts: ["../ext/q.js", "../ext/eth.js/main.js", "../ext/eth.js/qt.js", "../ext/setup.js"] experimental.onMessageReceived: { console.log("[onMessageReceived]: ", message.data) // TODO move to messaging.js diff --git a/cmd/mist/assets/qml/main.qml b/cmd/mist/assets/qml/main.qml index 111bef8bb4..e287e384f2 100644 --- a/cmd/mist/assets/qml/main.qml +++ b/cmd/mist/assets/qml/main.qml @@ -59,8 +59,8 @@ ApplicationWindow { mainSplit.setView(wallet.view, wallet.menuItem); - // Call the ready handler - gui.done(); + // Command setup + gui.sendCommand(0) } function addViews(view, path, options) { diff --git a/cmd/mist/gui.go b/cmd/mist/gui.go index 2e3f329b2a..083fd5d0a3 100644 --- a/cmd/mist/gui.go +++ b/cmd/mist/gui.go @@ -26,7 +26,9 @@ import ( "bytes" "encoding/json" "fmt" + "io/ioutil" "math/big" + "os" "path" "runtime" "strconv" @@ -48,15 +50,22 @@ import ( var guilogger = logger.NewLogger("GUI") +type ServEv byte + +const ( + setup ServEv = iota + update +) + type Gui struct { // The main application window win *qml.Window // QML Engine engine *qml.Engine component *qml.Common - qmlDone bool // The ethereum interface - eth *eth.Ethereum + eth *eth.Ethereum + serviceEvents chan ServEv // The public Ethereum library uiLib *UiLib @@ -86,7 +95,17 @@ func NewWindow(ethereum *eth.Ethereum, config *ethutil.ConfigManager, clientIden } xeth := xeth.NewJSXEth(ethereum) - gui := &Gui{eth: ethereum, txDb: db, xeth: xeth, logLevel: logger.LogLevel(logLevel), Session: session, open: false, clientIdentity: clientIdentity, config: config, plugins: make(map[string]plugin)} + gui := &Gui{eth: ethereum, + txDb: db, + xeth: xeth, + logLevel: logger.LogLevel(logLevel), + Session: session, + open: false, + clientIdentity: clientIdentity, + config: config, + plugins: make(map[string]plugin), + serviceEvents: make(chan ServEv, 1), + } data, _ := ethutil.ReadAllFile(path.Join(ethutil.Config.ExecPath, "plugins.json")) json.Unmarshal([]byte(data), &gui.plugins) @@ -98,6 +117,8 @@ func (gui *Gui) Start(assetPath string) { guilogger.Infoln("Starting GUI") + go gui.service() + // Register ethereum functions qml.RegisterTypes("Ethereum", 1, 0, []qml.TypeSpec{{ Init: func(p *xeth.JSBlock, obj qml.Object) { p.Number = 0; p.Hash = "" }, @@ -154,18 +175,11 @@ func (gui *Gui) showWallet(context *qml.Context) (*qml.Window, error) { return nil, err } - gui.win = gui.createWindow(component) - - gui.update() + gui.createWindow(component) return gui.win, nil } -// The done handler will be called by QML when all views have been loaded -func (gui *Gui) Done() { - gui.qmlDone = true -} - func (gui *Gui) ImportKey(filePath string) { } @@ -179,10 +193,8 @@ func (gui *Gui) showKeyImport(context *qml.Context) (*qml.Window, error) { } func (gui *Gui) createWindow(comp qml.Object) *qml.Window { - win := comp.CreateWindow(nil) - - gui.win = win - gui.uiLib.win = win + gui.win = comp.CreateWindow(nil) + gui.uiLib.win = gui.win return gui.win } @@ -335,11 +347,48 @@ func (self *Gui) getObjectByName(objectName string) qml.Object { return self.win.Root().ObjectByName(objectName) } -// Simple go routine function that updates the list of peers in the GUI -func (gui *Gui) update() { - // We have to wait for qml to be done loading all the windows. - for !gui.qmlDone { - time.Sleep(300 * time.Millisecond) +func loadJavascriptAssets(gui *Gui) (jsfiles string) { + for _, fn := range []string{"ext/q.js", "ext/eth.js/main.js", "ext/eth.js/qt.js", "ext/setup.js"} { + f, err := os.Open(gui.uiLib.AssetPath(fn)) + if err != nil { + fmt.Println(err) + continue + } + + content, err := ioutil.ReadAll(f) + if err != nil { + fmt.Println(err) + continue + } + jsfiles += string(content) + } + + return +} + +func (gui *Gui) SendCommand(cmd ServEv) { + gui.serviceEvents <- cmd +} + +func (gui *Gui) service() { + for ev := range gui.serviceEvents { + switch ev { + case setup: + go gui.setup() + case update: + go gui.update() + } + } +} + +func (gui *Gui) setup() { + for gui.win == nil { + time.Sleep(time.Millisecond * 200) + } + + for _, plugin := range gui.plugins { + guilogger.Infoln("Loading plugin ", plugin.Name) + gui.win.Root().Call("addPlugin", plugin.Path, "") } go func() { @@ -349,14 +398,21 @@ func (gui *Gui) update() { gui.setPeerInfo() }() - gui.whisper.SetView(gui.win.Root().ObjectByName("whisperView")) + // Inject javascript files each time navigation is requested. + // Unfortunately webview.experimental.userScripts injects _after_ + // the page has loaded which kind of renders it useless... + jsfiles := loadJavascriptAssets(gui) + gui.getObjectByName("webView").On("navigationRequested", func() { + gui.getObjectByName("webView").Call("injectJs", jsfiles) + }) - for _, plugin := range gui.plugins { - guilogger.Infoln("Loading plugin ", plugin.Name) + gui.whisper.SetView(gui.getObjectByName("whisperView")) - gui.win.Root().Call("addPlugin", plugin.Path, "") - } + gui.SendCommand(update) +} +// Simple go routine function that updates the list of peers in the GUI +func (gui *Gui) update() { peerUpdateTicker := time.NewTicker(5 * time.Second) generalUpdateTicker := time.NewTicker(500 * time.Millisecond) statsUpdateTicker := time.NewTicker(5 * time.Second) @@ -375,77 +431,75 @@ func (gui *Gui) update() { core.TxPostEvent{}, ) - go func() { - defer events.Unsubscribe() - for { - select { - case ev, isopen := <-events.Chan(): - if !isopen { - return - } - switch ev := ev.(type) { - case core.NewBlockEvent: - gui.processBlock(ev.Block, false) - if bytes.Compare(ev.Block.Coinbase(), gui.address()) == 0 { - gui.setWalletValue(gui.eth.ChainManager().State().GetBalance(gui.address()), nil) - } - - case core.TxPreEvent: - tx := ev.Tx - - tstate := gui.eth.ChainManager().TransState() - cstate := gui.eth.ChainManager().State() - - taccount := tstate.GetAccount(gui.address()) - caccount := cstate.GetAccount(gui.address()) - unconfirmedFunds := new(big.Int).Sub(taccount.Balance(), caccount.Balance()) - - gui.setWalletValue(taccount.Balance(), unconfirmedFunds) - gui.insertTransaction("pre", tx) - - case core.TxPostEvent: - tx := ev.Tx - object := state.GetAccount(gui.address()) - - if bytes.Compare(tx.From(), gui.address()) == 0 { - object.SubAmount(tx.Value()) - - gui.txDb.Put(tx.Hash(), tx.RlpEncode()) - } else if bytes.Compare(tx.To(), gui.address()) == 0 { - object.AddAmount(tx.Value()) - - gui.txDb.Put(tx.Hash(), tx.RlpEncode()) - } - - gui.setWalletValue(object.Balance(), nil) - state.UpdateStateObject(object) - } - - case <-peerUpdateTicker.C: - gui.setPeerInfo() - case <-generalUpdateTicker.C: - statusText := "#" + gui.eth.ChainManager().CurrentBlock().Number().String() - lastBlockLabel.Set("text", statusText) - miningLabel.Set("text", "Mining @ "+strconv.FormatInt(gui.uiLib.miner.GetPow().GetHashrate(), 10)+"Khash") - - /* - blockLength := gui.eth.BlockPool().BlocksProcessed - chainLength := gui.eth.BlockPool().ChainLength - - var ( - pct float64 = 1.0 / float64(chainLength) * float64(blockLength) - dlWidget = gui.win.Root().ObjectByName("downloadIndicator") - dlLabel = gui.win.Root().ObjectByName("downloadLabel") - ) - dlWidget.Set("value", pct) - dlLabel.Set("text", fmt.Sprintf("%d / %d", blockLength, chainLength)) - */ - - case <-statsUpdateTicker.C: - gui.setStatsPane() + defer events.Unsubscribe() + for { + select { + case ev, isopen := <-events.Chan(): + if !isopen { + return } + switch ev := ev.(type) { + case core.NewBlockEvent: + gui.processBlock(ev.Block, false) + if bytes.Compare(ev.Block.Coinbase(), gui.address()) == 0 { + gui.setWalletValue(gui.eth.ChainManager().State().GetBalance(gui.address()), nil) + } + + case core.TxPreEvent: + tx := ev.Tx + + tstate := gui.eth.ChainManager().TransState() + cstate := gui.eth.ChainManager().State() + + taccount := tstate.GetAccount(gui.address()) + caccount := cstate.GetAccount(gui.address()) + unconfirmedFunds := new(big.Int).Sub(taccount.Balance(), caccount.Balance()) + + gui.setWalletValue(taccount.Balance(), unconfirmedFunds) + gui.insertTransaction("pre", tx) + + case core.TxPostEvent: + tx := ev.Tx + object := state.GetAccount(gui.address()) + + if bytes.Compare(tx.From(), gui.address()) == 0 { + object.SubAmount(tx.Value()) + + gui.txDb.Put(tx.Hash(), tx.RlpEncode()) + } else if bytes.Compare(tx.To(), gui.address()) == 0 { + object.AddAmount(tx.Value()) + + gui.txDb.Put(tx.Hash(), tx.RlpEncode()) + } + + gui.setWalletValue(object.Balance(), nil) + state.UpdateStateObject(object) + } + + case <-peerUpdateTicker.C: + gui.setPeerInfo() + case <-generalUpdateTicker.C: + statusText := "#" + gui.eth.ChainManager().CurrentBlock().Number().String() + lastBlockLabel.Set("text", statusText) + miningLabel.Set("text", "Mining @ "+strconv.FormatInt(gui.uiLib.miner.GetPow().GetHashrate(), 10)+"Khash") + + /* + blockLength := gui.eth.BlockPool().BlocksProcessed + chainLength := gui.eth.BlockPool().ChainLength + + var ( + pct float64 = 1.0 / float64(chainLength) * float64(blockLength) + dlWidget = gui.win.Root().ObjectByName("downloadIndicator") + dlLabel = gui.win.Root().ObjectByName("downloadLabel") + ) + dlWidget.Set("value", pct) + dlLabel.Set("text", fmt.Sprintf("%d / %d", blockLength, chainLength)) + */ + + case <-statsUpdateTicker.C: + gui.setStatsPane() } - }() + } } func (gui *Gui) setStatsPane() { From e27237a03a3f0ab8dce37705701ed73de252e1f4 Mon Sep 17 00:00:00 2001 From: obscuren Date: Thu, 8 Jan 2015 16:45:51 +0100 Subject: [PATCH 04/13] Changed to use hash for comparison DeepReflect would fail on TD since TD isn't included in the original block and thus the test would fail. --- core/chain_manager_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/chain_manager_test.go b/core/chain_manager_test.go index b384c49264..5bbc2db70b 100644 --- a/core/chain_manager_test.go +++ b/core/chain_manager_test.go @@ -1,6 +1,7 @@ package core import ( + "bytes" "fmt" "os" "path" @@ -76,11 +77,11 @@ func TestChainInsertions(t *testing.T) { <-done } - if reflect.DeepEqual(chain2[len(chain2)-1], chainMan.CurrentBlock()) { + if bytes.Equal(chain2[len(chain2)-1].Hash(), chainMan.CurrentBlock().Hash()) { t.Error("chain2 is canonical and shouldn't be") } - if !reflect.DeepEqual(chain1[len(chain1)-1], chainMan.CurrentBlock()) { + if !bytes.Equal(chain1[len(chain1)-1].Hash(), chainMan.CurrentBlock().Hash()) { t.Error("chain1 isn't canonical and should be") } } From 5f958a582d1326ada1cb34b4c6578590a7c40e6c Mon Sep 17 00:00:00 2001 From: obscuren Date: Thu, 8 Jan 2015 16:48:39 +0100 Subject: [PATCH 05/13] fixed other tests to use hashes as well --- core/chain_manager_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/chain_manager_test.go b/core/chain_manager_test.go index 5bbc2db70b..725352dafa 100644 --- a/core/chain_manager_test.go +++ b/core/chain_manager_test.go @@ -5,7 +5,6 @@ import ( "fmt" "os" "path" - "reflect" "runtime" "strconv" "testing" @@ -125,7 +124,7 @@ func TestChainMultipleInsertions(t *testing.T) { <-done } - if !reflect.DeepEqual(chains[longest][len(chains[longest])-1], chainMan.CurrentBlock()) { + if !bytes.Equal(chains[longest][len(chains[longest])-1].Hash(), chainMan.CurrentBlock().Hash()) { t.Error("Invalid canonical chain") } } From 4a0ade4788b0e8d53c6b0eabaf9652643b6a073a Mon Sep 17 00:00:00 2001 From: obscuren Date: Thu, 8 Jan 2015 21:41:32 +0100 Subject: [PATCH 06/13] Fixed some whisper issues --- cmd/mist/assets/ext/eth.js/main.js | 1 + cmd/mist/assets/qml/browser.qml | 42 +++++++++++++++++------------- ui/qt/qwhisper/whisper.go | 10 ++++--- whisper/message.go | 2 ++ whisper/peer.go | 2 +- whisper/whisper.go | 2 +- 6 files changed, 36 insertions(+), 23 deletions(-) diff --git a/cmd/mist/assets/ext/eth.js/main.js b/cmd/mist/assets/ext/eth.js/main.js index 5c7ca06034..71e304a550 100644 --- a/cmd/mist/assets/ext/eth.js/main.js +++ b/cmd/mist/assets/ext/eth.js/main.js @@ -352,6 +352,7 @@ web3.provider = new ProviderManager(); web3.setProvider = function(provider) { + console.log("setprovider", provider) provider.onmessage = messageHandler; web3.provider.set(provider); web3.provider.sendQueued(); diff --git a/cmd/mist/assets/qml/browser.qml b/cmd/mist/assets/qml/browser.qml index 4cf6b24704..c2f8741bc3 100644 --- a/cmd/mist/assets/qml/browser.qml +++ b/cmd/mist/assets/qml/browser.qml @@ -59,7 +59,7 @@ Rectangle { } Component.onCompleted: { - webview.url = "/Users/jeffrey/test.html" + webview.url = "http://etherian.io" } signal messages(var messages, int id); @@ -153,7 +153,9 @@ Rectangle { } function injectJs(js) { - experimental.evaluateJavaScript(js) + //webview.experimental.navigatorQtObjectEnabled = true; + //webview.experimental.evaluateJavaScript(js) + //webview.experimental.javascriptEnabled = true; } function sendMessage(data) { @@ -164,7 +166,7 @@ Rectangle { experimental.preferences.javascriptEnabled: true experimental.preferences.navigatorQtObjectEnabled: true experimental.preferences.developerExtrasEnabled: true - //experimental.userScripts: ["../ext/q.js", "../ext/eth.js/main.js", "../ext/eth.js/qt.js", "../ext/setup.js"] + experimental.userScripts: ["../ext/q.js", "../ext/eth.js/main.js", "../ext/eth.js/qt.js", "../ext/setup.js"] experimental.onMessageReceived: { console.log("[onMessageReceived]: ", message.data) // TODO move to messaging.js @@ -344,24 +346,28 @@ Rectangle { break; case "newIdentity": - postData(data._id, shh.newIdentity()) - break + var id = shh.newIdentity() + console.log("newIdentity", id) + postData(data._id, id) + + break case "post": - require(1); - var params = data.args[0]; - var fields = ["payload", "to", "from"]; - for(var i = 0; i < fields.length; i++) { - params[fields[i]] = params[fields[i]] || ""; - } - if(typeof params.payload !== "object") { params.payload = [params.payload]; } //params.payload = params.payload.join(""); } - params.topics = params.topics || []; - params.priority = params.priority || 1000; - params.ttl = params.ttl || 100; + require(1); - console.log(JSON.stringify(params)) - shh.post(params.payload, params.to, params.from, params.topics, params.priority, params.ttl); - break; + var params = data.args[0]; + var fields = ["payload", "to", "from"]; + for(var i = 0; i < fields.length; i++) { + params[fields[i]] = params[fields[i]] || ""; + } + if(typeof params.payload !== "object") { params.payload = [params.payload]; } //params.payload = params.payload.join(""); } + params.topics = params.topics || []; + params.priority = params.priority || 1000; + params.ttl = params.ttl || 100; + + shh.post(params.payload, params.to, params.from, params.topics, params.priority, params.ttl); + + break; } } catch(e) { console.log(data.call + ": " + e) diff --git a/ui/qt/qwhisper/whisper.go b/ui/qt/qwhisper/whisper.go index 0627acd298..8a064c81cd 100644 --- a/ui/qt/qwhisper/whisper.go +++ b/ui/qt/qwhisper/whisper.go @@ -6,10 +6,13 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethutil" + "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/whisper" "gopkg.in/qml.v1" ) +var qlogger = logger.NewLogger("QSHH") + func fromHex(s string) []byte { if len(s) > 1 { return ethutil.Hex2Bytes(s[2:]) @@ -36,9 +39,10 @@ func (self *Whisper) SetView(view qml.Object) { func (self *Whisper) Post(payload []string, to, from string, topics []string, priority, ttl uint32) { var data []byte for _, d := range payload { - data = append(data, fromHex(d)...) + data = append(data, ethutil.Hex2Bytes(d)...) } + fmt.Println(payload, data, "from", from, fromHex(from), crypto.ToECDSA(fromHex(from))) msg := whisper.NewMessage(data) envelope, err := msg.Seal(time.Duration(priority*100000), whisper.Opts{ Ttl: time.Duration(ttl), @@ -47,13 +51,13 @@ func (self *Whisper) Post(payload []string, to, from string, topics []string, pr Topics: whisper.TopicsFromString(topics...), }) if err != nil { - fmt.Println(err) + qlogger.Infoln(err) // handle error return } if err := self.Whisper.Send(envelope); err != nil { - fmt.Println(err) + qlogger.Infoln(err) // handle error return } diff --git a/whisper/message.go b/whisper/message.go index db0110b4a7..5bda849ec8 100644 --- a/whisper/message.go +++ b/whisper/message.go @@ -2,6 +2,7 @@ package whisper import ( "crypto/ecdsa" + "fmt" "time" "github.com/ethereum/go-ethereum/crypto" @@ -53,6 +54,7 @@ type Opts struct { } func (self *Message) Seal(pow time.Duration, opts Opts) (*Envelope, error) { + fmt.Println(opts) if opts.From != nil { err := self.sign(opts.From) if err != nil { diff --git a/whisper/peer.go b/whisper/peer.go index d42b374b53..f82cc6e3e7 100644 --- a/whisper/peer.go +++ b/whisper/peer.go @@ -55,7 +55,7 @@ out: case <-relay.C: err := self.broadcast(self.host.envelopes()) if err != nil { - self.peer.Infoln(err) + self.peer.Infoln("broadcast err:", err) break out } diff --git a/whisper/whisper.go b/whisper/whisper.go index ffcdd7d409..3ff4bac5ac 100644 --- a/whisper/whisper.go +++ b/whisper/whisper.go @@ -229,11 +229,11 @@ func (self *Whisper) envelopes() (envelopes []*Envelope) { func (self *Whisper) postEvent(envelope *Envelope) { for _, key := range self.keys { if message, err := envelope.Open(key); err == nil || (err != nil && err == ecies.ErrInvalidPublicKey) { - // Create a custom filter? self.filters.Notify(filter.Generic{ Str1: string(crypto.FromECDSA(key)), Str2: string(crypto.FromECDSAPub(message.Recover())), Data: bytesToMap(envelope.Topics), }, message) + break } else { wlogger.Infoln(err) } From 26f066f0c7570bcca43299721c2b7a1a70186ee3 Mon Sep 17 00:00:00 2001 From: obscuren Date: Thu, 8 Jan 2015 22:18:23 +0100 Subject: [PATCH 07/13] just enable by default --- eth/backend.go | 10 +++------- whisper/whisper.go | 3 +++ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/eth/backend.go b/eth/backend.go index 02d3dc942f..ad04863092 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -134,24 +134,20 @@ func New(config *Config) (*Ethereum, error) { eth.blockPool = NewBlockPool(hasBlock, insertChain, ezp.Verify) ethProto := EthProtocol(eth.txPool, eth.chainManager, eth.blockPool) - protocols := []p2p.Protocol{ethProto} - - if config.Shh { - eth.whisper = whisper.New() - protocols = append(protocols, eth.whisper.Protocol()) - } + protocols := []p2p.Protocol{ethProto, eth.whisper.Protocol()} nat, err := p2p.ParseNAT(config.NATType, config.PMPGateway) if err != nil { return nil, err } + fmt.Println(nat) eth.net = &p2p.Server{ Identity: clientId, MaxPeers: config.MaxPeers, Protocols: protocols, Blacklist: eth.blacklist, - NAT: nat, + NAT: p2p.UPNP(), NoDial: !config.Dial, } diff --git a/whisper/whisper.go b/whisper/whisper.go index 3ff4bac5ac..71a4e841ed 100644 --- a/whisper/whisper.go +++ b/whisper/whisper.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/ecdsa" "errors" + "fmt" "sync" "time" @@ -143,6 +144,7 @@ func (self *Whisper) msgHandler(peer *p2p.Peer, ws p2p.MsgReadWriter) error { if err != nil { return err } + fmt.Println("reading message") envelope, err := NewEnvelopeFromReader(msg.Payload) if err != nil { @@ -160,6 +162,7 @@ func (self *Whisper) msgHandler(peer *p2p.Peer, ws p2p.MsgReadWriter) error { // takes care of adding envelopes to the messages pool. At this moment no sanity checks are being performed. func (self *Whisper) add(envelope *Envelope) error { + fmt.Println("adding") if !envelope.valid() { return errors.New("invalid pow provided for envelope") } From 3bdf28c1fef6644f17bf66a9240af5168465ad5a Mon Sep 17 00:00:00 2001 From: zelig Date: Fri, 9 Jan 2015 05:03:26 +0000 Subject: [PATCH 08/13] GetBlockHashesFromHash(hash, max) gives back max hashes starting from PARENT of hash --- core/chain_manager.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/chain_manager.go b/core/chain_manager.go index 2d4001f0f9..0b57406228 100644 --- a/core/chain_manager.go +++ b/core/chain_manager.go @@ -238,13 +238,11 @@ func (self *ChainManager) GetBlockHashesFromHash(hash []byte, max uint64) (chain // XXX Could be optimised by using a different database which only holds hashes (i.e., linked list) for i := uint64(0); i < max; i++ { + block = self.GetBlock(block.Header().ParentHash) chain = append(chain, block.Hash()) - if block.Header().Number.Cmp(ethutil.Big0) <= 0 { break } - - block = self.GetBlock(block.Header().ParentHash) } return From 69dfca2feb5c94f7a28a0b24c28181b6da4b9da3 Mon Sep 17 00:00:00 2001 From: zelig Date: Fri, 9 Jan 2015 05:04:32 +0000 Subject: [PATCH 09/13] minor changes in integration tests --- eth/test/tests/02.sh | 6 +++--- eth/test/tests/03.sh | 4 ++-- eth/test/tests/common.sh | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/eth/test/tests/02.sh b/eth/test/tests/02.sh index 5231dbd788..619217ed87 100644 --- a/eth/test/tests/02.sh +++ b/eth/test/tests/02.sh @@ -12,8 +12,8 @@ EOF peer 11 01 peer 12 02 -P13ID=$PID +P12ID=$PID test_node $NAME "" -loglevel 5 $JSFILE -sleep 0.5 -kill $P13ID +sleep 0.3 +kill $P12ID diff --git a/eth/test/tests/03.sh b/eth/test/tests/03.sh index 8c9d6565ef..d7dba737f0 100644 --- a/eth/test/tests/03.sh +++ b/eth/test/tests/03.sh @@ -1,10 +1,10 @@ #!/bin/bash -TIMEOUT=35 +TIMEOUT=12 cat >> $JSFILE < Date: Fri, 9 Jan 2015 05:06:04 +0000 Subject: [PATCH 10/13] no need to call AddBlockHashes when receiving new block --- eth/protocol.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/eth/protocol.go b/eth/protocol.go index 736bcd94bb..c9a5dea8df 100644 --- a/eth/protocol.go +++ b/eth/protocol.go @@ -211,16 +211,6 @@ func (self *ethProtocol) handle() error { // uses AddPeer followed by AddHashes, AddBlock only if peer is the best peer // (or selected as new best peer) if self.blockPool.AddPeer(request.TD, hash, self.id, self.requestBlockHashes, self.requestBlocks, self.protoErrorDisconnect) { - called := true - iter := func() ([]byte, bool) { - if called { - called = false - return hash, true - } else { - return nil, false - } - } - self.blockPool.AddBlockHashes(iter, self.id) self.blockPool.AddBlock(request.Block, self.id) } From f72cb28b0f688d99d7936900474e7d10d06ffb3a Mon Sep 17 00:00:00 2001 From: zelig Date: Fri, 9 Jan 2015 05:57:09 +0000 Subject: [PATCH 11/13] adapt unit tests to spec - AddBlockHashes ignores the first hash (just used to match getBlockHashes query) sends the rest as blocksMsg - new test TestPeerWithKnownParentBlock - new test TestChainConnectingWithParentHash - adapt all other tests to the new scheme --- eth/block_pool_test.go | 192 +++++++++++++++++++++++++++++------------ 1 file changed, 139 insertions(+), 53 deletions(-) diff --git a/eth/block_pool_test.go b/eth/block_pool_test.go index b50a314ead..94c3b43d29 100644 --- a/eth/block_pool_test.go +++ b/eth/block_pool_test.go @@ -18,7 +18,7 @@ import ( const waitTimeout = 60 // seconds -var logsys = ethlogger.NewStdLogSystem(os.Stdout, log.LstdFlags, ethlogger.LogLevel(ethlogger.DebugLevel)) +var logsys = ethlogger.NewStdLogSystem(os.Stdout, log.LstdFlags, ethlogger.LogLevel(ethlogger.DebugDetailLevel)) var ini = false @@ -336,12 +336,12 @@ func (self *peerTester) AddPeer() bool { // peer sends blockhashes if and when gets a request func (self *peerTester) AddBlockHashes(indexes ...int) { - i := 0 fmt.Printf("ready to add block hashes %v\n", indexes) self.waitBlockHashesRequests(indexes[0]) fmt.Printf("adding block hashes %v\n", indexes) hashes := self.hashPool.indexesToHashes(indexes) + i := 1 next := func() (hash []byte, ok bool) { if i < len(hashes) { hash = hashes[i] @@ -415,7 +415,7 @@ func TestAddPeer(t *testing.T) { if blockPool.peer.id != "peer0" { t.Errorf("peer0 (TD=1) not set as best") } - peer0.checkBlockHashesRequests(0) + // peer0.checkBlockHashesRequests(0) best = peer2.AddPeer() if !best { @@ -424,7 +424,7 @@ func TestAddPeer(t *testing.T) { if blockPool.peer.id != "peer2" { t.Errorf("peer2 (TD=3) not set as best") } - peer2.checkBlockHashesRequests(2) + peer2.waitBlocksRequests(2) best = peer1.AddPeer() if best { @@ -449,7 +449,7 @@ func TestAddPeer(t *testing.T) { if blockPool.peer.td.Cmp(big.NewInt(int64(4))) != 0 { t.Errorf("peer2 TD not updated") } - peer2.checkBlockHashesRequests(2, 3) + peer2.waitBlocksRequests(3) peer1.td = 3 peer1.currentBlock = 2 @@ -474,7 +474,7 @@ func TestAddPeer(t *testing.T) { if blockPool.peer.id != "peer1" { t.Errorf("existing peer1 (TD=3) should be set as best peer") } - peer1.checkBlockHashesRequests(2) + peer1.waitBlocksRequests(2) blockPool.RemovePeer("peer1") peer, best = blockPool.getPeer("peer1") @@ -485,6 +485,7 @@ func TestAddPeer(t *testing.T) { if blockPool.peer.id != "peer0" { t.Errorf("existing peer0 (TD=1) should be set as best peer") } + peer0.waitBlocksRequests(0) blockPool.RemovePeer("peer0") peer, best = blockPool.getPeer("peer0") @@ -502,7 +503,7 @@ func TestAddPeer(t *testing.T) { if blockPool.peer.id != "peer0" { t.Errorf("peer0 (TD=1) should be set as best") } - peer0.checkBlockHashesRequests(0, 0, 3) + peer0.waitBlocksRequests(3) blockPool.Stop() @@ -513,17 +514,36 @@ func TestPeerWithKnownBlock(t *testing.T) { _, blockPool, blockPoolTester := newTestBlockPool(t) blockPoolTester.refBlockChain[0] = nil blockPoolTester.blockChain[0] = nil - // hashPool, blockPool, blockPoolTester := newTestBlockPool() blockPool.Start() peer0 := blockPoolTester.newPeer("0", 1, 0) peer0.AddPeer() + blockPool.Wait(waitTimeout * time.Second) blockPool.Stop() // no request on known block peer0.checkBlockHashesRequests() } +func TestPeerWithKnownParentBlock(t *testing.T) { + logInit() + _, blockPool, blockPoolTester := newTestBlockPool(t) + blockPoolTester.initRefBlockChain(1) + blockPoolTester.blockChain[0] = nil + blockPool.Start() + + peer0 := blockPoolTester.newPeer("0", 1, 1) + peer0.AddPeer() + peer0.AddBlocks(0, 1) + + blockPool.Wait(waitTimeout * time.Second) + blockPool.Stop() + peer0.checkBlocksRequests([]int{1}) + peer0.checkBlockHashesRequests() + blockPoolTester.refBlockChain[1] = []int{} + blockPoolTester.checkBlockChain(blockPoolTester.refBlockChain) +} + func TestSimpleChain(t *testing.T) { logInit() _, blockPool, blockPoolTester := newTestBlockPool(t) @@ -534,8 +554,9 @@ func TestSimpleChain(t *testing.T) { peer1 := blockPoolTester.newPeer("peer1", 1, 2) peer1.AddPeer() + peer1.AddBlocks(1, 2) go peer1.AddBlockHashes(2, 1, 0) - peer1.AddBlocks(0, 1, 2) + peer1.AddBlocks(0, 1) blockPool.Wait(waitTimeout * time.Second) blockPool.Stop() @@ -543,6 +564,26 @@ func TestSimpleChain(t *testing.T) { blockPoolTester.checkBlockChain(blockPoolTester.refBlockChain) } +func TestChainConnectingWithParentHash(t *testing.T) { + logInit() + _, blockPool, blockPoolTester := newTestBlockPool(t) + blockPoolTester.blockChain[0] = nil + blockPoolTester.initRefBlockChain(3) + + blockPool.Start() + + peer1 := blockPoolTester.newPeer("peer1", 1, 3) + peer1.AddPeer() + go peer1.AddBlocks(2, 3) + go peer1.AddBlockHashes(3, 2, 1) + peer1.AddBlocks(0, 1, 2) + + blockPool.Wait(waitTimeout * time.Second) + blockPool.Stop() + blockPoolTester.refBlockChain[3] = []int{} + blockPoolTester.checkBlockChain(blockPoolTester.refBlockChain) +} + func TestInvalidBlock(t *testing.T) { logInit() _, blockPool, blockPoolTester := newTestBlockPool(t) @@ -554,8 +595,9 @@ func TestInvalidBlock(t *testing.T) { peer1 := blockPoolTester.newPeer("peer1", 1, 3) peer1.AddPeer() + go peer1.AddBlocks(2, 3) go peer1.AddBlockHashes(3, 2, 1, 0) - peer1.AddBlocks(0, 1, 2, 3) + peer1.AddBlocks(0, 1, 2) blockPool.Wait(waitTimeout * time.Second) blockPool.Stop() @@ -566,7 +608,7 @@ func TestInvalidBlock(t *testing.T) { t.Errorf("wrong error, got %v, expected %v", peer1.peerErrors[0], ErrInvalidBlock) } } else { - t.Errorf("expected invalid block error, got nothing") + t.Errorf("expected invalid block error, got nothing %v", peer1.peerErrors) } } @@ -579,7 +621,7 @@ func TestVerifyPoW(t *testing.T) { blockPoolTester.blockPool.verifyPoW = func(b pow.Block) bool { bb, _ := b.(*types.Block) indexes := blockPoolTester.hashPool.hashesToIndexes([][]byte{bb.Hash()}) - if indexes[0] == 1 && !first { + if indexes[0] == 2 && !first { first = true return false } else { @@ -590,15 +632,17 @@ func TestVerifyPoW(t *testing.T) { blockPool.Start() - peer1 := blockPoolTester.newPeer("peer1", 1, 2) + peer1 := blockPoolTester.newPeer("peer1", 1, 3) peer1.AddPeer() - go peer1.AddBlockHashes(2, 1, 0) + go peer1.AddBlocks(2, 3) + go peer1.AddBlockHashes(3, 2, 1, 0) peer1.AddBlocks(0, 1, 2) - peer1.AddBlocks(0, 1) - blockPool.Wait(waitTimeout * time.Second) + // blockPool.Wait(waitTimeout * time.Second) + time.Sleep(1 * time.Second) blockPool.Stop() - blockPoolTester.refBlockChain[2] = []int{} + blockPoolTester.refBlockChain[1] = []int{} + delete(blockPoolTester.refBlockChain, 2) blockPoolTester.checkBlockChain(blockPoolTester.refBlockChain) if len(peer1.peerErrors) == 1 { if peer1.peerErrors[0] != ErrInvalidPoW { @@ -620,8 +664,9 @@ func TestMultiSectionChain(t *testing.T) { peer1 := blockPoolTester.newPeer("peer1", 1, 5) peer1.AddPeer() + go peer1.AddBlocks(4, 5) go peer1.AddBlockHashes(5, 4, 3) - go peer1.AddBlocks(2, 3, 4, 5) + go peer1.AddBlocks(2, 3, 4) go peer1.AddBlockHashes(3, 2, 1, 0) peer1.AddBlocks(0, 1, 2) @@ -641,14 +686,17 @@ func TestNewBlocksOnPartialChain(t *testing.T) { peer1 := blockPoolTester.newPeer("peer1", 1, 5) peer1.AddPeer() + go peer1.AddBlocks(4, 5) // partially complete section go peer1.AddBlockHashes(5, 4, 3) - peer1.AddBlocks(2, 3) // partially complete section + peer1.AddBlocks(3, 4) // partially complete section // peer1 found new blocks peer1.td = 2 peer1.currentBlock = 7 peer1.AddPeer() + go peer1.AddBlocks(6, 7) go peer1.AddBlockHashes(7, 6, 5) - go peer1.AddBlocks(3, 4, 5, 6, 7) + go peer1.AddBlocks(2, 3) + go peer1.AddBlocks(5, 6) go peer1.AddBlockHashes(3, 2, 1, 0) // tests that hash request from known chain root is remembered peer1.AddBlocks(0, 1, 2) @@ -658,35 +706,37 @@ func TestNewBlocksOnPartialChain(t *testing.T) { blockPoolTester.checkBlockChain(blockPoolTester.refBlockChain) } -func TestPeerSwitch(t *testing.T) { +func TestPeerSwitchUp(t *testing.T) { logInit() _, blockPool, blockPoolTester := newTestBlockPool(t) blockPoolTester.blockChain[0] = nil - blockPoolTester.initRefBlockChain(6) + blockPoolTester.initRefBlockChain(7) blockPool.Start() - peer1 := blockPoolTester.newPeer("peer1", 1, 5) - peer2 := blockPoolTester.newPeer("peer2", 2, 6) + peer1 := blockPoolTester.newPeer("peer1", 1, 6) + peer2 := blockPoolTester.newPeer("peer2", 2, 7) peer2.blocksRequestsMap = peer1.blocksRequestsMap peer1.AddPeer() - go peer1.AddBlockHashes(5, 4, 3) + go peer1.AddBlocks(5, 6) + go peer1.AddBlockHashes(6, 5, 4, 3) // peer1.AddBlocks(2, 3) // section partially complete, block 3 will be preserved after peer demoted peer2.AddPeer() // peer2 is promoted as best peer, peer1 is demoted - go peer2.AddBlockHashes(6, 5) // - go peer2.AddBlocks(4, 5, 6) // tests that block request for earlier section is remembered + go peer2.AddBlocks(6, 7) + go peer2.AddBlockHashes(7, 6) // + go peer2.AddBlocks(4, 5) // tests that block request for earlier section is remembered go peer1.AddBlocks(3, 4) // tests that connecting section by demoted peer is remembered and blocks are accepted from demoted peer go peer2.AddBlockHashes(3, 2, 1, 0) // tests that known chain section is activated, hash requests from 3 is remembered peer2.AddBlocks(0, 1, 2) // final blocks linking to blockchain sent blockPool.Wait(waitTimeout * time.Second) blockPool.Stop() - blockPoolTester.refBlockChain[6] = []int{} + blockPoolTester.refBlockChain[7] = []int{} blockPoolTester.checkBlockChain(blockPoolTester.refBlockChain) } -func TestPeerDownSwitch(t *testing.T) { +func TestPeerSwitchDown(t *testing.T) { logInit() _, blockPool, blockPoolTester := newTestBlockPool(t) blockPoolTester.blockChain[0] = nil @@ -698,12 +748,39 @@ func TestPeerDownSwitch(t *testing.T) { peer2.blocksRequestsMap = peer1.blocksRequestsMap peer2.AddPeer() - go peer2.AddBlockHashes(6, 5, 4) peer2.AddBlocks(5, 6) // partially complete, section will be preserved + go peer2.AddBlockHashes(6, 5, 4) // + peer2.AddBlocks(4, 5) // blockPool.RemovePeer("peer2") // peer2 disconnects peer1.AddPeer() // inferior peer1 is promoted as best peer go peer1.AddBlockHashes(4, 3, 2, 1, 0) // - go peer1.AddBlocks(3, 4, 5) // tests that section set by demoted peer is remembered and blocks are accepted + go peer1.AddBlocks(3, 4) // tests that section set by demoted peer is remembered and blocks are accepted , this connects the chain sections together + peer1.AddBlocks(0, 1, 2, 3) + + blockPool.Wait(waitTimeout * time.Second) + blockPool.Stop() + blockPoolTester.refBlockChain[6] = []int{} + blockPoolTester.checkBlockChain(blockPoolTester.refBlockChain) +} + +func TestPeerCompleteSectionSwitchDown(t *testing.T) { + logInit() + _, blockPool, blockPoolTester := newTestBlockPool(t) + blockPoolTester.blockChain[0] = nil + blockPoolTester.initRefBlockChain(6) + blockPool.Start() + + peer1 := blockPoolTester.newPeer("peer1", 1, 4) + peer2 := blockPoolTester.newPeer("peer2", 2, 6) + peer2.blocksRequestsMap = peer1.blocksRequestsMap + + peer2.AddPeer() + peer2.AddBlocks(5, 6) // partially complete, section will be preserved + go peer2.AddBlockHashes(6, 5, 4) // + peer2.AddBlocks(3, 4, 5) // complete section + blockPool.RemovePeer("peer2") // peer2 disconnects + peer1.AddPeer() // inferior peer1 is promoted as best peer + peer1.AddBlockHashes(4, 3, 2, 1, 0) // tests that hash request are directly connecting if the head block exists peer1.AddBlocks(0, 1, 2, 3) blockPool.Wait(waitTimeout * time.Second) @@ -725,11 +802,13 @@ func TestPeerSwitchBack(t *testing.T) { peer2.blocksRequestsMap = peer1.blocksRequestsMap peer2.AddPeer() + go peer2.AddBlocks(7, 8) go peer2.AddBlockHashes(8, 7, 6) go peer2.AddBlockHashes(6, 5, 4) peer2.AddBlocks(4, 5) // section partially complete peer1.AddPeer() // peer1 is promoted as best peer - go peer1.AddBlockHashes(11, 10) // only gives useless results + go peer1.AddBlocks(10, 11) // + peer1.AddBlockHashes(11, 10) // only gives useless results blockPool.RemovePeer("peer1") // peer1 disconnects go peer2.AddBlockHashes(4, 3, 2, 1, 0) // tests that asking for hashes from 4 is remembered go peer2.AddBlocks(3, 4, 5, 6, 7, 8) // tests that section 4, 5, 6 and 7, 8 are remembered for missing blocks @@ -756,11 +835,13 @@ func TestForkSimple(t *testing.T) { peer2.blocksRequestsMap = peer1.blocksRequestsMap peer1.AddPeer() + go peer1.AddBlocks(8, 9) go peer1.AddBlockHashes(9, 8, 7, 3, 2) - peer1.AddBlocks(1, 2, 3, 7, 8, 9) + peer1.AddBlocks(1, 2, 3, 7, 8) peer2.AddPeer() // peer2 is promoted as best peer + go peer2.AddBlocks(5, 6) // go peer2.AddBlockHashes(6, 5, 4, 3, 2) // fork on 3 -> 4 (earlier child: 7) - go peer2.AddBlocks(1, 2, 3, 4, 5, 6) + go peer2.AddBlocks(1, 2, 3, 4, 5) go peer2.AddBlockHashes(2, 1, 0) peer2.AddBlocks(0, 1, 2) @@ -790,23 +871,24 @@ func TestForkSwitchBackByNewBlocks(t *testing.T) { peer2.blocksRequestsMap = peer1.blocksRequestsMap peer1.AddPeer() - go peer1.AddBlockHashes(9, 8, 7, 3, 2) - peer1.AddBlocks(8, 9) // partial section + peer1.AddBlocks(8, 9) // + go peer1.AddBlockHashes(9, 8, 7, 3, 2) // + peer1.AddBlocks(7, 8) // partial section peer2.AddPeer() // + peer2.AddBlocks(5, 6) // go peer2.AddBlockHashes(6, 5, 4, 3, 2) // peer2 forks on block 3 - peer2.AddBlocks(1, 2, 3, 4, 5, 6) // + peer2.AddBlocks(1, 2, 3, 4, 5) // // peer1 finds new blocks peer1.td = 3 peer1.currentBlock = 11 peer1.AddPeer() + go peer1.AddBlocks(10, 11) go peer1.AddBlockHashes(11, 10, 9) - peer1.AddBlocks(7, 8, 9, 10, 11) - go peer1.AddBlockHashes(7, 3) // tests that hash request from fork root is remembered - go peer1.AddBlocks(3, 7) // tests that block requests on earlier fork are remembered - // go peer1.AddBlockHashes(1, 0) // tests that hash request from root of connecting chain section (added by demoted peer) is remembered + peer1.AddBlocks(9, 10) + go peer1.AddBlocks(3, 7) // tests that block requests on earlier fork are remembered go peer1.AddBlockHashes(2, 1, 0) // tests that hash request from root of connecting chain section (added by demoted peer) is remembered - peer1.AddBlocks(0, 1, 2, 3) + peer1.AddBlocks(0, 1) blockPool.Wait(waitTimeout * time.Second) blockPool.Stop() @@ -834,16 +916,18 @@ func TestForkSwitchBackByPeerSwitchBack(t *testing.T) { peer2.blocksRequestsMap = peer1.blocksRequestsMap peer1.AddPeer() + go peer1.AddBlocks(8, 9) go peer1.AddBlockHashes(9, 8, 7, 3, 2) - peer1.AddBlocks(8, 9) - peer2.AddPeer() // + peer1.AddBlocks(7, 8) + peer2.AddPeer() + go peer2.AddBlocks(5, 6) // go peer2.AddBlockHashes(6, 5, 4, 3, 2) // peer2 forks on block 3 - peer2.AddBlocks(2, 3, 4, 5, 6) // + peer2.AddBlocks(2, 3, 4, 5) // blockPool.RemovePeer("peer2") // peer2 disconnects, peer1 is promoted again as best peer - peer1.AddBlockHashes(7, 3) // tests that hash request from fork root is remembered - go peer1.AddBlocks(3, 7, 8) // tests that block requests on earlier fork are remembered + go peer1.AddBlocks(3, 7) // tests that block requests on earlier fork are remembered and orphan section relinks to existing parent block + go peer1.AddBlocks(1, 2) // go peer1.AddBlockHashes(2, 1, 0) // - peer1.AddBlocks(0, 1, 2, 3) + peer1.AddBlocks(0, 1) blockPool.Wait(waitTimeout * time.Second) blockPool.Stop() @@ -871,17 +955,19 @@ func TestForkCompleteSectionSwitchBackByPeerSwitchBack(t *testing.T) { peer2.blocksRequestsMap = peer1.blocksRequestsMap peer1.AddPeer() + go peer1.AddBlocks(8, 9) go peer1.AddBlockHashes(9, 8, 7) - peer1.AddBlocks(3, 7, 8, 9) // make sure this section is complete + peer1.AddBlocks(3, 7, 8) // make sure this section is complete time.Sleep(1 * time.Second) go peer1.AddBlockHashes(7, 3, 2) // block 3/7 is section boundary - peer1.AddBlocks(2, 3) // partially complete sections + peer1.AddBlocks(2, 3) // partially complete sections block 2 missing peer2.AddPeer() // + go peer2.AddBlocks(5, 6) // go peer2.AddBlockHashes(6, 5, 4, 3, 2) // peer2 forks on block 3 - peer2.AddBlocks(2, 3, 4, 5, 6) // block 2 still missing. + peer2.AddBlocks(2, 3, 4, 5) // block 2 still missing. blockPool.RemovePeer("peer2") // peer2 disconnects, peer1 is promoted again as best peer - peer1.AddBlockHashes(7, 3) // tests that hash request from fork root is remembered even though section process completed - go peer1.AddBlockHashes(2, 1, 0) // + // peer1.AddBlockHashes(7, 3) // tests that hash request from fork root is remembered even though section process completed + go peer1.AddBlockHashes(2, 1, 0) // peer1.AddBlocks(0, 1, 2) blockPool.Wait(waitTimeout * time.Second) From 8ecc9509b3ac490fe8ac9d91e24e8271963ee443 Mon Sep 17 00:00:00 2001 From: zelig Date: Fri, 9 Jan 2015 06:03:32 +0000 Subject: [PATCH 12/13] add ErrInsufficientChainInfo error --- eth/error.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eth/error.go b/eth/error.go index 1d9f806380..9c4a684818 100644 --- a/eth/error.go +++ b/eth/error.go @@ -16,6 +16,7 @@ const ( ErrInvalidBlock ErrInvalidPoW ErrUnrequestedBlock + ErrInsufficientChainInfo ) var errorToString = map[int]string{ @@ -30,6 +31,7 @@ var errorToString = map[int]string{ ErrInvalidBlock: "Invalid block", ErrInvalidPoW: "Invalid PoW", ErrUnrequestedBlock: "Unrequested block", + ErrInsufficientChainInfo: "Insufficient chain info", } type protocolError struct { From 5a9952c7b47f20451feea1158286450e08b85551 Mon Sep 17 00:00:00 2001 From: zelig Date: Fri, 9 Jan 2015 06:03:45 +0000 Subject: [PATCH 13/13] major blockpool change - the spec says response to getBlockHashes(from, max) should return all hashes starting from PARENT of from. This required major changes and results in much hackier code. - Introduced a first round block request after peer introduces with current head, so that hashes can be linked to the head - peerInfo records currentBlockHash, currentBlock, parentHash and headSection - AddBlockHashes checks header section and creates the top node from the peerInfo of the best peer - AddBlock checks peerInfo and updates the block there rather than in a node - request further hashes once a section is created but then no more until the root block is found (so that we know when to stop asking) - in processSection, when root node is checked and receives a block, we need to check if the section has a parent known to blockchain or blockPool - when peers are switched, new peer launches a new requestHeadSection loop or activates its actual head section, i.e., the section for it currentBlockHash - all tests pass --- eth/block_pool.go | 467 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 350 insertions(+), 117 deletions(-) diff --git a/eth/block_pool.go b/eth/block_pool.go index 2334330e0b..b624d064a5 100644 --- a/eth/block_pool.go +++ b/eth/block_pool.go @@ -1,6 +1,7 @@ package eth import ( + "bytes" "fmt" "math" "math/big" @@ -24,8 +25,8 @@ const ( blocksRequestRepetition = 1 blockHashesRequestInterval = 500 // ms blocksRequestMaxIdleRounds = 100 - cacheTimeout = 3 // minutes - blockTimeout = 5 // minutes + blockHashesTimeout = 60 // seconds + blocksTimeout = 120 // seconds ) type poolNode struct { @@ -70,9 +71,14 @@ type BlockPool struct { type peerInfo struct { lock sync.RWMutex - td *big.Int - currentBlock []byte - id string + td *big.Int + currentBlockHash []byte + currentBlock *types.Block + currentBlockC chan *types.Block + parentHash []byte + headSection *section + headSectionC chan *section + id string requestBlockHashes func([]byte) error requestBlocks func([][]byte) error @@ -203,30 +209,39 @@ func (self *BlockPool) Wait(t time.Duration) { // AddPeer is called by the eth protocol instance running on the peer after // the status message has been received with total difficulty and current block hash // AddPeer can only be used once, RemovePeer needs to be called when the peer disconnects -func (self *BlockPool) AddPeer(td *big.Int, currentBlock []byte, peerId string, requestBlockHashes func([]byte) error, requestBlocks func([][]byte) error, peerError func(int, string, ...interface{})) bool { +func (self *BlockPool) AddPeer(td *big.Int, currentBlockHash []byte, peerId string, requestBlockHashes func([]byte) error, requestBlocks func([][]byte) error, peerError func(int, string, ...interface{})) (best bool) { self.peersLock.Lock() defer self.peersLock.Unlock() peer, ok := self.peers[peerId] if ok { - poolLogger.Debugf("Update peer %v with td %v and current block %x", peerId, td, currentBlock[:4]) - peer.td = td - peer.currentBlock = currentBlock + if bytes.Compare(peer.currentBlockHash, currentBlockHash) != 0 { + poolLogger.Debugf("Update peer %v with td %v and current block %s", peerId, td, name(currentBlockHash)) + peer.lock.Lock() + peer.td = td + peer.currentBlockHash = currentBlockHash + peer.currentBlock = nil + peer.parentHash = nil + peer.headSection = nil + peer.lock.Unlock() + } } else { peer = &peerInfo{ td: td, - currentBlock: currentBlock, + currentBlockHash: currentBlockHash, id: peerId, //peer.Identity().Pubkey() requestBlockHashes: requestBlockHashes, requestBlocks: requestBlocks, peerError: peerError, sections: make(map[string]*section), + currentBlockC: make(chan *types.Block), + headSectionC: make(chan *section), } self.peers[peerId] = peer - poolLogger.Debugf("add new peer %v with td %v and current block %x", peerId, td, currentBlock[:4]) + poolLogger.Debugf("add new peer %v with td %v and current block %x", peerId, td, currentBlockHash[:4]) } // check peer current head - if self.hasBlock(currentBlock) { + if self.hasBlock(currentBlockHash) { // peer not ahead return false } @@ -234,22 +249,135 @@ func (self *BlockPool) AddPeer(td *big.Int, currentBlock []byte, peerId string, if self.peer == peer { // new block update // peer is already active best peer, request hashes - poolLogger.Debugf("[%s] already the best peer. request hashes from %s", peerId, name(currentBlock)) - peer.requestBlockHashes(currentBlock) - return true + poolLogger.Debugf("[%s] already the best peer. Request new head section info from %s", peerId, name(currentBlockHash)) + peer.headSectionC <- nil + best = true + } else { + currentTD := ethutil.Big0 + if self.peer != nil { + currentTD = self.peer.td + } + if td.Cmp(currentTD) > 0 { + poolLogger.Debugf("peer %v promoted best peer", peerId) + self.switchPeer(self.peer, peer) + self.peer = peer + best = true + } } + return +} - currentTD := ethutil.Big0 - if self.peer != nil { - currentTD = self.peer.td - } - if td.Cmp(currentTD) > 0 { - poolLogger.Debugf("peer %v promoted best peer", peerId) - self.switchPeer(self.peer, peer) - self.peer = peer - return true - } - return false +func (self *BlockPool) requestHeadSection(peer *peerInfo) { + self.wg.Add(1) + self.procWg.Add(1) + poolLogger.Debugf("[%s] head section at [%s] requesting info", peer.id, name(peer.currentBlockHash)) + + go func() { + var idle bool + peer.lock.RLock() + quitC := peer.quitC + currentBlockHash := peer.currentBlockHash + peer.lock.RUnlock() + blockHashesRequestTimer := time.NewTimer(0) + blocksRequestTimer := time.NewTimer(0) + suicide := time.NewTimer(blockHashesTimeout * time.Second) + blockHashesRequestTimer.Stop() + defer blockHashesRequestTimer.Stop() + defer blocksRequestTimer.Stop() + + entry := self.get(currentBlockHash) + if entry != nil { + entry.node.lock.RLock() + currentBlock := entry.node.block + entry.node.lock.RUnlock() + if currentBlock != nil { + peer.lock.Lock() + peer.currentBlock = currentBlock + peer.parentHash = currentBlock.ParentHash() + poolLogger.Debugf("[%s] head block [%s] found", peer.id, name(currentBlockHash)) + peer.lock.Unlock() + blockHashesRequestTimer.Reset(0) + blocksRequestTimer.Stop() + } + } + + LOOP: + for { + + select { + case <-self.quit: + break LOOP + + case <-quitC: + poolLogger.Debugf("[%s] head section at [%s] incomplete - quit request loop", peer.id, name(currentBlockHash)) + break LOOP + + case headSection := <-peer.headSectionC: + peer.lock.Lock() + peer.headSection = headSection + if headSection == nil { + oldBlockHash := currentBlockHash + currentBlockHash = peer.currentBlockHash + poolLogger.Debugf("[%s] head section changed [%s] -> [%s]", peer.id, name(oldBlockHash), name(currentBlockHash)) + if idle { + idle = false + suicide.Reset(blockHashesTimeout * time.Second) + self.procWg.Add(1) + } + blocksRequestTimer.Reset(blocksRequestInterval * time.Millisecond) + } else { + poolLogger.DebugDetailf("[%s] head section at [%s] created", peer.id, name(currentBlockHash)) + if !idle { + idle = true + suicide.Stop() + self.procWg.Done() + } + } + peer.lock.Unlock() + blockHashesRequestTimer.Stop() + + case <-blockHashesRequestTimer.C: + poolLogger.DebugDetailf("[%s] head section at [%s] not found, requesting block hashes", peer.id, name(currentBlockHash)) + peer.requestBlockHashes(currentBlockHash) + blockHashesRequestTimer.Reset(blockHashesRequestInterval * time.Millisecond) + + case currentBlock := <-peer.currentBlockC: + peer.lock.Lock() + peer.currentBlock = currentBlock + peer.parentHash = currentBlock.ParentHash() + poolLogger.DebugDetailf("[%s] head block [%s] found", peer.id, name(currentBlockHash)) + peer.lock.Unlock() + if self.hasBlock(currentBlock.ParentHash()) { + if err := self.insertChain(types.Blocks([]*types.Block{currentBlock})); err != nil { + peer.peerError(ErrInvalidBlock, "%v", err) + } + if !idle { + idle = true + suicide.Stop() + self.procWg.Done() + } + } else { + blockHashesRequestTimer.Reset(0) + } + blocksRequestTimer.Stop() + + case <-blocksRequestTimer.C: + peer.lock.RLock() + poolLogger.DebugDetailf("[%s] head block [%s] not found, requesting", peer.id, name(currentBlockHash)) + peer.requestBlocks([][]byte{peer.currentBlockHash}) + peer.lock.RUnlock() + blocksRequestTimer.Reset(blocksRequestInterval * time.Millisecond) + + case <-suicide.C: + peer.peerError(ErrInsufficientChainInfo, "peer failed to provide block hashes or head block for block hash %x", currentBlockHash) + break LOOP + } + } + self.wg.Done() + if !idle { + self.procWg.Done() + } + }() } // RemovePeer is called by the eth protocol when the peer disconnects @@ -274,13 +402,13 @@ func (self *BlockPool) RemovePeer(peerId string) { newPeer = info } } - self.peer = newPeer - self.switchPeer(peer, newPeer) if newPeer != nil { poolLogger.Debugf("peer %v with td %v promoted to best peer", newPeer.id, newPeer.td) } else { poolLogger.Warnln("no peers") } + self.peer = newPeer + self.switchPeer(peer, newPeer) } } @@ -299,25 +427,56 @@ func (self *BlockPool) AddBlockHashes(next func() ([]byte, bool), peerId string) return } // peer is still the best - poolLogger.Debugf("adding hashes for best peer %s", peerId) var size, n int var hash []byte - var ok bool - var section, child, parent *section + var ok, headSection bool + var sec, child, parent *section var entry *poolEntry var nodes []*poolNode + bestPeer := peer + + hash, ok = next() + peer.lock.Lock() + if bytes.Compare(peer.parentHash, hash) == 0 { + if self.hasBlock(peer.currentBlockHash) { + return + } + poolLogger.Debugf("adding hashes at chain head for best peer %s starting from [%s]", peerId, name(peer.currentBlockHash)) + headSection = true + + if entry := self.get(peer.currentBlockHash); entry == nil { + node := &poolNode{ + hash: peer.currentBlockHash, + block: peer.currentBlock, + peer: peerId, + blockBy: peerId, + } + if size == 0 { + sec = newSection() + } + nodes = append(nodes, node) + size++ + n++ + } else { + child = entry.section + } + } else { + poolLogger.Debugf("adding hashes for best peer %s starting from [%s]", peerId, name(hash)) + } + quitC := peer.quitC + peer.lock.Unlock() LOOP: // iterate using next (rlp stream lazy decoder) feeding hashesC - for hash, ok = next(); ok; hash, ok = next() { + for ; ok; hash, ok = next() { n++ select { case <-self.quit: return - case <-peer.quitC: + case <-quitC: // if the peer is demoted, no more hashes taken - peer = nil + bestPeer = nil break LOOP default: } @@ -325,8 +484,8 @@ LOOP: // check if known block connecting the downloaded chain to our blockchain poolLogger.DebugDetailf("[%s] known block", name(hash)) // mark child as absolute pool root with parent known to blockchain - if section != nil { - self.connectToBlockChain(section) + if sec != nil { + self.connectToBlockChain(sec) } else { if child != nil { self.connectToBlockChain(child) @@ -340,6 +499,7 @@ LOOP: // reached a known chain in the pool if entry.node == entry.section.bottom && n == 1 { // the first block hash received is an orphan in the pool, so rejoice and continue + poolLogger.DebugDetailf("[%s] connecting child section", sectionName(entry.section)) child = entry.section continue LOOP } @@ -353,7 +513,7 @@ LOOP: peer: peerId, } if size == 0 { - section = newSection() + sec = newSection() } nodes = append(nodes, node) size++ @@ -379,10 +539,10 @@ LOOP: } if size > 0 { - self.processSection(section, nodes) - poolLogger.DebugDetailf("[%s]->[%s](%v)->[%s] new chain section", sectionName(parent), sectionName(section), size, sectionName(child)) - self.link(parent, section) - self.link(section, child) + self.processSection(sec, nodes) + poolLogger.DebugDetailf("[%s]->[%s](%v)->[%s] new chain section", sectionName(parent), sectionName(sec), size, sectionName(child)) + self.link(parent, sec) + self.link(sec, child) } else { poolLogger.DebugDetailf("[%s]->[%s] connecting known sections", sectionName(parent), sectionName(child)) self.link(parent, child) @@ -390,15 +550,31 @@ LOOP: self.chainLock.Unlock() - if parent != nil && peer != nil { + if parent != nil && bestPeer != nil { self.activateChain(parent, peer) poolLogger.Debugf("[%s] activate parent section [%s]", name(parent.top.hash), sectionName(parent)) } - if section != nil { - peer.addSection(section.top.hash, section) - section.controlC <- peer - poolLogger.Debugf("[%s] activate new section", sectionName(section)) + if sec != nil { + peer.addSection(sec.top.hash, sec) + // request next section here once, only repeat if bottom block arrives, + // otherwise no way to check if it arrived + peer.requestBlockHashes(sec.bottom.hash) + sec.controlC <- bestPeer + poolLogger.Debugf("[%s] activate new section", sectionName(sec)) + } + + if headSection { + var headSec *section + switch { + case sec != nil: + headSec = sec + case child != nil: + headSec = child + default: + headSec = parent + } + peer.headSectionC <- headSec } } @@ -426,14 +602,21 @@ func sectionName(section *section) (name string) { // only the first PoW-valid block for a hash is considered legit func (self *BlockPool) AddBlock(block *types.Block, peerId string) { hash := block.Hash() - if self.hasBlock(hash) { - poolLogger.DebugDetailf("block [%s] already known", name(hash)) - return - } + self.peersLock.Lock() + peer := self.peer + self.peersLock.Unlock() + entry := self.get(hash) + if bytes.Compare(hash, peer.currentBlockHash) == 0 { + poolLogger.Debugf("add head block [%s] for peer %s", name(hash), peerId) + peer.currentBlockC <- block + } else { + if entry == nil { + poolLogger.Warnf("unrequested block [%s] by peer %s", name(hash), peerId) + self.peerError(peerId, ErrUnrequestedBlock, "%x", hash) + } + } if entry == nil { - poolLogger.Warnf("unrequested block [%x] by peer %s", hash, peerId) - self.peerError(peerId, ErrUnrequestedBlock, "%x", hash) return } @@ -443,17 +626,21 @@ func (self *BlockPool) AddBlock(block *types.Block, peerId string) { // check if block already present if node.block != nil { - poolLogger.DebugDetailf("block [%x] already sent by %s", name(hash), node.blockBy) + poolLogger.DebugDetailf("block [%s] already sent by %s", name(hash), node.blockBy) return } - // validate block for PoW - if !self.verifyPoW(block) { - poolLogger.Warnf("invalid pow on block [%x] by peer %s", hash, peerId) - self.peerError(peerId, ErrInvalidPoW, "%x", hash) - return - } + if self.hasBlock(hash) { + poolLogger.DebugDetailf("block [%s] already known", name(hash)) + } else { + // validate block for PoW + if !self.verifyPoW(block) { + poolLogger.Warnf("invalid pow on block [%s] by peer %s", name(hash), peerId) + self.peerError(peerId, ErrInvalidPoW, "%x", hash) + return + } + } poolLogger.Debugf("added block [%s] sent by peer %s", name(hash), peerId) node.block = block node.blockBy = peerId @@ -544,23 +731,23 @@ LOOP: // - when turned off (if peer disconnects and new peer connects with alternative chain), no blockrequests are made but absolute expiry timer is ticking // - when turned back on it recursively calls itself on the root of the next chain section // - when exits, signals to -func (self *BlockPool) processSection(section *section, nodes []*poolNode) { +func (self *BlockPool) processSection(sec *section, nodes []*poolNode) { for i, node := range nodes { - entry := &poolEntry{node: node, section: section, index: i} + entry := &poolEntry{node: node, section: sec, index: i} self.set(node.hash, entry) } - section.bottom = nodes[len(nodes)-1] - section.top = nodes[0] - section.nodes = nodes - poolLogger.DebugDetailf("[%s] setup section process", sectionName(section)) + sec.bottom = nodes[len(nodes)-1] + sec.top = nodes[0] + sec.nodes = nodes + poolLogger.DebugDetailf("[%s] setup section process", sectionName(sec)) self.wg.Add(1) go func() { // absolute time after which sub-chain is killed if not complete (some blocks are missing) - suicideTimer := time.After(blockTimeout * time.Minute) + suicideTimer := time.After(blocksTimeout * time.Second) var peer, newPeer *peerInfo @@ -580,21 +767,23 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) { var insertChain bool var quitC chan bool - var blockChainC = section.blockChainC + var blockChainC = sec.blockChainC + + var parentHash []byte LOOP: for { if insertChain { insertChain = false - rest, err := self.addSectionToBlockChain(section) + rest, err := self.addSectionToBlockChain(sec) if err != nil { - close(section.suicideC) + close(sec.suicideC) continue LOOP } if rest == 0 { blocksRequestsComplete = true - child := self.getChild(section) + child := self.getChild(sec) if child != nil { self.connectToBlockChain(child) } @@ -603,7 +792,7 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) { if blockHashesRequestsComplete && blocksRequestsComplete { // not waiting for hashes any more - poolLogger.Debugf("[%s] section complete %v blocks retrieved (%v attempts), hash requests complete on root (%v attempts)", sectionName(section), depth, blocksRequests, blockHashesRequests) + poolLogger.Debugf("[%s] section complete %v blocks retrieved (%v attempts), hash requests complete on root (%v attempts)", sectionName(sec), depth, blocksRequests, blockHashesRequests) break LOOP } // otherwise suicide if no hashes coming @@ -611,11 +800,12 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) { // went through all blocks in section if missing == 0 { // no missing blocks - poolLogger.DebugDetailf("[%s] got all blocks. process complete (%v total blocksRequests): missing %v/%v/%v", sectionName(section), blocksRequests, missing, lastMissing, depth) + poolLogger.DebugDetailf("[%s] got all blocks. process complete (%v total blocksRequests): missing %v/%v/%v", sectionName(sec), blocksRequests, missing, lastMissing, depth) blocksRequestsComplete = true blocksRequestTimer = nil blocksRequestTime = false } else { + poolLogger.DebugDetailf("[%s] section checked: missing %v/%v/%v", sectionName(sec), missing, lastMissing, depth) // some missing blocks blocksRequests++ if len(hashes) > 0 { @@ -630,8 +820,8 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) { idle++ // too many idle rounds if idle >= blocksRequestMaxIdleRounds { - poolLogger.DebugDetailf("[%s] block requests had %v idle rounds (%v total attempts): missing %v/%v/%v\ngiving up...", sectionName(section), idle, blocksRequests, missing, lastMissing, depth) - close(section.suicideC) + poolLogger.DebugDetailf("[%s] block requests had %v idle rounds (%v total attempts): missing %v/%v/%v\ngiving up...", sectionName(sec), idle, blocksRequests, missing, lastMissing, depth) + close(sec.suicideC) } } else { idle = 0 @@ -653,22 +843,39 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) { // if ready && blocksRequestTime && !blocksRequestsComplete { - poolLogger.DebugDetailf("[%s] check if new blocks arrived (attempt %v): missing %v/%v/%v", sectionName(section), blocksRequests, missing, lastMissing, depth) + poolLogger.DebugDetailf("[%s] check if new blocks arrived (attempt %v): missing %v/%v/%v", sectionName(sec), blocksRequests, missing, lastMissing, depth) blocksRequestTimer = time.After(blocksRequestInterval * time.Millisecond) blocksRequestTime = false processC = offC } if blockHashesRequestTime { - if self.getParent(section) != nil { + var parentSection = self.getParent(sec) + if parentSection == nil { + if parent := self.get(parentHash); parent != nil { + parentSection = parent.section + self.chainLock.Lock() + self.link(parentSection, sec) + self.chainLock.Unlock() + } else { + if self.hasBlock(parentHash) { + insertChain = true + blockHashesRequestTime = false + blockHashesRequestTimer = nil + blockHashesRequestsComplete = true + continue LOOP + } + } + } + if parentSection != nil { // if not root of chain, switch off - poolLogger.DebugDetailf("[%s] parent found, hash requests deactivated (after %v total attempts)\n", sectionName(section), blockHashesRequests) + poolLogger.DebugDetailf("[%s] parent found, hash requests deactivated (after %v total attempts)\n", sectionName(sec), blockHashesRequests) blockHashesRequestTimer = nil blockHashesRequestsComplete = true } else { blockHashesRequests++ - poolLogger.Debugf("[%s] hash request on root (%v total attempts)\n", sectionName(section), blockHashesRequests) - peer.requestBlockHashes(section.bottom.hash) + poolLogger.Debugf("[%s] hash request on root (%v total attempts)\n", sectionName(sec), blockHashesRequests) + peer.requestBlockHashes(sec.bottom.hash) blockHashesRequestTimer = time.After(blockHashesRequestInterval * time.Millisecond) } blockHashesRequestTime = false @@ -682,27 +889,27 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) { // peer quit or demoted, put section in idle mode quitC = nil go func() { - section.controlC <- nil + sec.controlC <- nil }() case <-self.purgeC: suicideTimer = time.After(0) case <-suicideTimer: - close(section.suicideC) - poolLogger.Debugf("[%s] timeout. (%v total attempts): missing %v/%v/%v", sectionName(section), blocksRequests, missing, lastMissing, depth) + close(sec.suicideC) + poolLogger.Debugf("[%s] timeout. (%v total attempts): missing %v/%v/%v", sectionName(sec), blocksRequests, missing, lastMissing, depth) - case <-section.suicideC: - poolLogger.Debugf("[%s] suicide", sectionName(section)) + case <-sec.suicideC: + poolLogger.Debugf("[%s] suicide", sectionName(sec)) // first delink from child and parent under chainlock self.chainLock.Lock() - self.link(nil, section) - self.link(section, nil) + self.link(nil, sec) + self.link(sec, nil) self.chainLock.Unlock() // delete node entries from pool index under pool lock self.lock.Lock() - for _, node := range section.nodes { + for _, node := range sec.nodes { delete(self.pool, string(node.hash)) } self.lock.Unlock() @@ -710,20 +917,20 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) { break LOOP case <-blocksRequestTimer: - poolLogger.DebugDetailf("[%s] block request time", sectionName(section)) + poolLogger.DebugDetailf("[%s] block request time", sectionName(sec)) blocksRequestTime = true case <-blockHashesRequestTimer: - poolLogger.DebugDetailf("[%s] hash request time", sectionName(section)) + poolLogger.DebugDetailf("[%s] hash request time", sectionName(sec)) blockHashesRequestTime = true - case newPeer = <-section.controlC: + case newPeer = <-sec.controlC: // active -> idle if peer != nil && newPeer == nil { self.procWg.Done() if init { - poolLogger.Debugf("[%s] idle mode (%v total attempts): missing %v/%v/%v", sectionName(section), blocksRequests, missing, lastMissing, depth) + poolLogger.Debugf("[%s] idle mode (%v total attempts): missing %v/%v/%v", sectionName(sec), blocksRequests, missing, lastMissing, depth) } blocksRequestTime = false blocksRequestTimer = nil @@ -739,11 +946,11 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) { if peer == nil && newPeer != nil { self.procWg.Add(1) - poolLogger.Debugf("[%s] active mode", sectionName(section)) + poolLogger.Debugf("[%s] active mode", sectionName(sec)) if !blocksRequestsComplete { blocksRequestTime = true } - if !blockHashesRequestsComplete { + if !blockHashesRequestsComplete && parentHash != nil { blockHashesRequestTime = true } if !init { @@ -753,13 +960,13 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) { missing = 0 self.wg.Add(1) self.procWg.Add(1) - depth = len(section.nodes) + depth = len(sec.nodes) lastMissing = depth // if not run at least once fully, launch iterator go func() { var node *poolNode IT: - for _, node = range section.nodes { + for _, node = range sec.nodes { select { case processC <- node: case <-self.quit: @@ -771,7 +978,7 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) { self.procWg.Done() }() } else { - poolLogger.Debugf("[%s] restore earlier state", sectionName(section)) + poolLogger.Debugf("[%s] restore earlier state", sectionName(sec)) processC = offC } } @@ -781,7 +988,7 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) { } peer = newPeer - case waiter := <-section.forkC: + case waiter := <-sec.forkC: // this case just blocks the process until section is split at the fork <-waiter init = false @@ -794,7 +1001,7 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) { init = true done = true processC = make(chan *poolNode, missing) - poolLogger.DebugDetailf("[%s] section initalised: missing %v/%v/%v", sectionName(section), missing, lastMissing, depth) + poolLogger.DebugDetailf("[%s] section initalised: missing %v/%v/%v", sectionName(sec), missing, lastMissing, depth) continue LOOP } if ready { @@ -811,17 +1018,24 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) { missing++ hashes = append(hashes, node.hash) if len(hashes) == blockBatchSize { - poolLogger.Debugf("[%s] request %v missing blocks", sectionName(section), len(hashes)) + poolLogger.Debugf("[%s] request %v missing blocks", sectionName(sec), len(hashes)) self.requestBlocks(blocksRequests, hashes) hashes = nil } missingC <- node } else { - if blockChainC == nil && i == lastMissing { - insertChain = true + if i == lastMissing { + if blockChainC == nil { + insertChain = true + } else { + if parentHash == nil { + parentHash = block.ParentHash() + poolLogger.Debugf("[%s] found root block [%s]", sectionName(sec), name(parentHash)) + blockHashesRequestTime = true + } + } } } - poolLogger.Debugf("[%s] %v/%v/%v/%v", sectionName(section), i, missing, lastMissing, depth) if i == lastMissing && init { done = true } @@ -829,23 +1043,22 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) { case <-blockChainC: // closed blockChain channel indicates that the blockpool is reached // connected to the blockchain, insert the longest chain of blocks - poolLogger.Debugf("[%s] reached blockchain", sectionName(section)) + poolLogger.Debugf("[%s] reached blockchain", sectionName(sec)) blockChainC = nil // switch off hash requests in case they were on blockHashesRequestTime = false blockHashesRequestTimer = nil blockHashesRequestsComplete = true // section root has block - if len(section.nodes) > 0 && section.nodes[len(section.nodes)-1].block != nil { + if len(sec.nodes) > 0 && sec.nodes[len(sec.nodes)-1].block != nil { insertChain = true } continue LOOP } // select } // for - poolLogger.Debugf("[%s] section complete: %v block hashes requests - %v block requests - missing %v/%v/%v", sectionName(section), blockHashesRequests, blocksRequests, missing, lastMissing, depth) - close(section.offC) + close(sec.offC) self.wg.Done() if peer != nil { @@ -917,22 +1130,28 @@ func (self *peerInfo) addSection(hash []byte, section *section) (found *section) defer self.lock.Unlock() key := string(hash) found = self.sections[key] - poolLogger.DebugDetailf("[%s] section process %s registered", sectionName(section), self.id) + poolLogger.DebugDetailf("[%s] section process stored for %s", sectionName(section), self.id) self.sections[key] = section return } func (self *BlockPool) switchPeer(oldPeer, newPeer *peerInfo) { if newPeer != nil { - entry := self.get(newPeer.currentBlock) - if entry == nil { - poolLogger.Debugf("[%s] head block [%s] not found, requesting hashes", newPeer.id, name(newPeer.currentBlock)) - newPeer.requestBlockHashes(newPeer.currentBlock) - } else { - poolLogger.Debugf("[%s] head block [%s] found, activate chain at section [%s]", newPeer.id, name(newPeer.currentBlock), sectionName(entry.section)) - self.activateChain(entry.section, newPeer) - } + newPeer.quitC = make(chan bool) poolLogger.DebugDetailf("[%s] activate section processes", newPeer.id) + var addSections []*section + for hash, section := range newPeer.sections { + // split sections get reorganised here + if string(section.top.hash) != hash { + addSections = append(addSections, section) + if entry := self.get([]byte(hash)); entry != nil { + addSections = append(addSections, entry.section) + } + } + } + for _, section := range addSections { + newPeer.sections[string(section.top.hash)] = section + } for hash, section := range newPeer.sections { // this will block if section process is waiting for peer lock select { @@ -940,12 +1159,26 @@ func (self *BlockPool) switchPeer(oldPeer, newPeer *peerInfo) { poolLogger.DebugDetailf("[%s][%x] section process complete - remove", newPeer.id, hash[:4]) delete(newPeer.sections, hash) case section.controlC <- newPeer: - poolLogger.DebugDetailf("[%s][%x] registered peer with section", newPeer.id, hash[:4]) + poolLogger.DebugDetailf("[%s][%x] activates section [%s]", newPeer.id, hash[:4], sectionName(section)) } } - newPeer.quitC = make(chan bool) + newPeer.lock.Lock() + headSection := newPeer.headSection + currentBlockHash := newPeer.currentBlockHash + newPeer.lock.Unlock() + if headSection == nil { + poolLogger.DebugDetailf("[%s] head section for [%s] not created, requesting info", newPeer.id, name(currentBlockHash)) + self.requestHeadSection(newPeer) + } else { + if entry := self.get(currentBlockHash); entry != nil { + headSection = entry.section + } + poolLogger.DebugDetailf("[%s] activate chain at head section [%s] for current head [%s]", newPeer.id, sectionName(headSection), name(currentBlockHash)) + self.activateChain(headSection, newPeer) + } } if oldPeer != nil { + poolLogger.DebugDetailf("[%s] quit section processes", oldPeer.id) close(oldPeer.quitC) } }