Merge remote-tracking branch 'origin/develop' into kademlia

cmd/mist/assets/ext/eth.js/main.js

@@ -352,6 +352,7 @@
     web3.provider = new ProviderManager();
 
     web3.setProvider = function(provider) {
+	console.log("setprovider", provider)
         provider.onmessage = messageHandler;
         web3.provider.set(provider);
         web3.provider.sendQueued();

cmd/mist/assets/qml/browser.qml

@@ -109,7 +109,8 @@ Rectangle {
                     leftMargin: 5
                     rightMargin: 5
                 }
-                text: "http://etherian.io"
+                //text: "http://etherian.io"
+		text: webview.url;
                 id: uriNav
                 y: parent.height / 2 - this.height / 2
 
@@ -151,6 +152,12 @@ Rectangle {
                 window.open(request.url.toString());					
             }
 
+	    function injectJs(js) {
+		//webview.experimental.navigatorQtObjectEnabled = true;
+		//webview.experimental.evaluateJavaScript(js)
+		//webview.experimental.javascriptEnabled = true;
+	    }
+
             function sendMessage(data) {
                 webview.experimental.postMessage(JSON.stringify(data))
             }
@@ -159,7 +166,6 @@ Rectangle {
             experimental.preferences.javascriptEnabled: true
             experimental.preferences.navigatorQtObjectEnabled: true
             experimental.preferences.developerExtrasEnabled: true
-            //experimental.userScripts: ["../ext/qt_messaging_adapter.js", "../ext/q.js", "../ext/big.js", "../ext/string.js", "../ext/html_messaging.js"]
             experimental.userScripts: ["../ext/q.js", "../ext/eth.js/main.js", "../ext/eth.js/qt.js", "../ext/setup.js"]
             experimental.onMessageReceived: {
                 console.log("[onMessageReceived]: ", message.data)
@@ -340,11 +346,15 @@ Rectangle {
 			break;
 
 			case "newIdentity":
-			postData(data._id, shh.newIdentity())
+				var id = shh.newIdentity()
+				console.log("newIdentity", id)
+				postData(data._id, id)
+
 				break
 
 			case "post":
 				require(1);
+
 				var params = data.args[0];
 				var fields = ["payload", "to", "from"];
 				for(var i = 0; i < fields.length; i++) {
@@ -355,8 +365,8 @@ Rectangle {
 				params.priority = params.priority || 1000;
 				params.ttl = params.ttl || 100;
 
-			console.log(JSON.stringify(params))
 				shh.post(params.payload, params.to, params.from, params.topics, params.priority, params.ttl);
+
 				break;
                     }
                 } catch(e) {

cmd/mist/assets/qml/main.qml

@@ -59,8 +59,8 @@ ApplicationWindow {
 
 		mainSplit.setView(wallet.view, wallet.menuItem);
 
-		// Call the ready handler
+		// Command setup
-		gui.done();
+		gui.sendCommand(0)
 	}
 
 	function addViews(view, path, options) {

cmd/mist/gui.go

@@ -26,7 +26,9 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"math/big"
+	"os"
 	"path"
 	"runtime"
 	"strconv"
@@ -48,15 +50,22 @@ import (
 
 var guilogger = logger.NewLogger("GUI")
 
+type ServEv byte
+
+const (
+	setup ServEv = iota
+	update
+)
+
 type Gui struct {
 	// The main application window
 	win *qml.Window
 	// QML Engine
 	engine    *qml.Engine
 	component *qml.Common
-	qmlDone   bool
 	// The ethereum interface
 	eth           *eth.Ethereum
+	serviceEvents chan ServEv
 
 	// The public Ethereum library
 	uiLib   *UiLib
@@ -86,7 +95,17 @@ func NewWindow(ethereum *eth.Ethereum, config *ethutil.ConfigManager, clientIden
 	}
 
 	xeth := xeth.NewJSXEth(ethereum)
-	gui := &Gui{eth: ethereum, txDb: db, xeth: xeth, logLevel: logger.LogLevel(logLevel), Session: session, open: false, clientIdentity: clientIdentity, config: config, plugins: make(map[string]plugin)}
+	gui := &Gui{eth: ethereum,
+		txDb:           db,
+		xeth:           xeth,
+		logLevel:       logger.LogLevel(logLevel),
+		Session:        session,
+		open:           false,
+		clientIdentity: clientIdentity,
+		config:         config,
+		plugins:        make(map[string]plugin),
+		serviceEvents:  make(chan ServEv, 1),
+	}
 	data, _ := ethutil.ReadAllFile(path.Join(ethutil.Config.ExecPath, "plugins.json"))
 	json.Unmarshal([]byte(data), &gui.plugins)
 
@@ -98,6 +117,8 @@ func (gui *Gui) Start(assetPath string) {
 
 	guilogger.Infoln("Starting GUI")
 
+	go gui.service()
+
 	// Register ethereum functions
 	qml.RegisterTypes("Ethereum", 1, 0, []qml.TypeSpec{{
 		Init: func(p *xeth.JSBlock, obj qml.Object) { p.Number = 0; p.Hash = "" },
@@ -154,18 +175,11 @@ func (gui *Gui) showWallet(context *qml.Context) (*qml.Window, error) {
 		return nil, err
 	}
 
-	gui.win = gui.createWindow(component)
+	gui.createWindow(component)
-
-	gui.update()
 
 	return gui.win, nil
 }
 
-// The done handler will be called by QML when all views have been loaded
-func (gui *Gui) Done() {
-	gui.qmlDone = true
-}
-
 func (gui *Gui) ImportKey(filePath string) {
 }
 
@@ -179,10 +193,8 @@ func (gui *Gui) showKeyImport(context *qml.Context) (*qml.Window, error) {
 }
 
 func (gui *Gui) createWindow(comp qml.Object) *qml.Window {
-	win := comp.CreateWindow(nil)
+	gui.win = comp.CreateWindow(nil)
-
+	gui.uiLib.win = gui.win
-	gui.win = win
-	gui.uiLib.win = win
 
 	return gui.win
 }
@@ -335,11 +347,48 @@ func (self *Gui) getObjectByName(objectName string) qml.Object {
 	return self.win.Root().ObjectByName(objectName)
 }
 
-// Simple go routine function that updates the list of peers in the GUI
+func loadJavascriptAssets(gui *Gui) (jsfiles string) {
-func (gui *Gui) update() {
+	for _, fn := range []string{"ext/q.js", "ext/eth.js/main.js", "ext/eth.js/qt.js", "ext/setup.js"} {
-	// We have to wait for qml to be done loading all the windows.
+		f, err := os.Open(gui.uiLib.AssetPath(fn))
-	for !gui.qmlDone {
+		if err != nil {
-		time.Sleep(300 * time.Millisecond)
+			fmt.Println(err)
+			continue
+		}
+
+		content, err := ioutil.ReadAll(f)
+		if err != nil {
+			fmt.Println(err)
+			continue
+		}
+		jsfiles += string(content)
+	}
+
+	return
+}
+
+func (gui *Gui) SendCommand(cmd ServEv) {
+	gui.serviceEvents <- cmd
+}
+
+func (gui *Gui) service() {
+	for ev := range gui.serviceEvents {
+		switch ev {
+		case setup:
+			go gui.setup()
+		case update:
+			go gui.update()
+		}
+	}
+}
+
+func (gui *Gui) setup() {
+	for gui.win == nil {
+		time.Sleep(time.Millisecond * 200)
+	}
+
+	for _, plugin := range gui.plugins {
+		guilogger.Infoln("Loading plugin ", plugin.Name)
+		gui.win.Root().Call("addPlugin", plugin.Path, "")
 	}
 
 	go func() {
@@ -349,14 +398,21 @@ func (gui *Gui) update() {
 		gui.setPeerInfo()
 	}()
 
-	gui.whisper.SetView(gui.win.Root().ObjectByName("whisperView"))
+	// Inject javascript files each time navigation is requested.
+	// Unfortunately webview.experimental.userScripts injects _after_
+	// the page has loaded which kind of renders it useless...
+	jsfiles := loadJavascriptAssets(gui)
+	gui.getObjectByName("webView").On("navigationRequested", func() {
+		gui.getObjectByName("webView").Call("injectJs", jsfiles)
+	})
 
-	for _, plugin := range gui.plugins {
+	gui.whisper.SetView(gui.getObjectByName("whisperView"))
-		guilogger.Infoln("Loading plugin ", plugin.Name)
 
-		gui.win.Root().Call("addPlugin", plugin.Path, "")
+	gui.SendCommand(update)
-	}
+}
 
+// Simple go routine function that updates the list of peers in the GUI
+func (gui *Gui) update() {
 	peerUpdateTicker := time.NewTicker(5 * time.Second)
 	generalUpdateTicker := time.NewTicker(500 * time.Millisecond)
 	statsUpdateTicker := time.NewTicker(5 * time.Second)
@@ -375,7 +431,6 @@ func (gui *Gui) update() {
 		core.TxPostEvent{},
 	)
 
-	go func() {
 	defer events.Unsubscribe()
 	for {
 		select {
@@ -445,7 +500,6 @@ func (gui *Gui) update() {
 			gui.setStatsPane()
 		}
 	}
-	}()
 }
 
 func (gui *Gui) setStatsPane() {

core/chain_manager.go

@@ -139,7 +139,7 @@ func (bc *ChainManager) setLastBlock() {
 		bc.Reset()
 	}
 
-	chainlogger.Infof("Last block (#%d) %x\n", bc.lastBlockNumber, bc.currentBlock.Hash())
+	chainlogger.Infof("Last block (#%d) %x TD=%v\n", bc.lastBlockNumber, bc.currentBlock.Hash(), bc.td)
 }
 
 // Block creation & chain handling
@@ -215,7 +215,7 @@ func (bc *ChainManager) insert(block *types.Block) {
 func (bc *ChainManager) write(block *types.Block) {
 	bc.writeBlockInfo(block)
 
-	encodedBlock := ethutil.Encode(block)
+	encodedBlock := ethutil.Encode(block.RlpDataForStorage())
 	bc.db.Put(block.Hash(), encodedBlock)
 }
 
@@ -238,13 +238,11 @@ func (self *ChainManager) GetBlockHashesFromHash(hash []byte, max uint64) (chain
 
 	// XXX Could be optimised by using a different database which only holds hashes (i.e., linked list)
 	for i := uint64(0); i < max; i++ {
+		block = self.GetBlock(block.Header().ParentHash)
 		chain = append(chain, block.Hash())
-
 		if block.Header().Number.Cmp(ethutil.Big0) <= 0 {
 			break
 		}
-
-		block = self.GetBlock(block.Header().ParentHash)
 	}
 
 	return

core/chain_manager_test.go

@@ -1,10 +1,10 @@
 package core
 
 import (
+	"bytes"
 	"fmt"
 	"os"
 	"path"
-	"reflect"
 	"runtime"
 	"strconv"
 	"testing"
@@ -76,11 +76,11 @@ func TestChainInsertions(t *testing.T) {
 		<-done
 	}
 
-	if reflect.DeepEqual(chain2[len(chain2)-1], chainMan.CurrentBlock()) {
+	if bytes.Equal(chain2[len(chain2)-1].Hash(), chainMan.CurrentBlock().Hash()) {
 		t.Error("chain2 is canonical and shouldn't be")
 	}
 
-	if !reflect.DeepEqual(chain1[len(chain1)-1], chainMan.CurrentBlock()) {
+	if !bytes.Equal(chain1[len(chain1)-1].Hash(), chainMan.CurrentBlock().Hash()) {
 		t.Error("chain1 isn't canonical and should be")
 	}
 }
@@ -124,7 +124,7 @@ func TestChainMultipleInsertions(t *testing.T) {
 		<-done
 	}
 
-	if !reflect.DeepEqual(chains[longest][len(chains[longest])-1], chainMan.CurrentBlock()) {
+	if !bytes.Equal(chains[longest][len(chains[longest])-1].Hash(), chainMan.CurrentBlock().Hash()) {
 		t.Error("Invalid canonical chain")
 	}
 }

core/genesis.go

@@ -25,6 +25,7 @@ func GenesisBlock(db ethutil.Database) *types.Block {
 	genesis.Header().GasLimit = big.NewInt(1000000)
 	genesis.Header().GasUsed = ethutil.Big0
 	genesis.Header().Time = 0
+	genesis.Td = ethutil.Big0
 
 	genesis.SetUncles([]*types.Header{})
 	genesis.SetTransactions(types.Transactions{})

core/types/derive_sha.go

@@ -3,7 +3,7 @@ package types
 import (
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/ethutil"
-	"github.com/ethereum/go-ethereum/ptrie"
+	"github.com/ethereum/go-ethereum/trie"
 )
 
 type DerivableList interface {
@@ -13,7 +13,7 @@ type DerivableList interface {
 
 func DeriveSha(list DerivableList) []byte {
 	db, _ := ethdb.NewMemDatabase()
-	trie := ptrie.New(nil, db)
+	trie := trie.New(nil, db)
 	for i := 0; i < list.Len(); i++ {
 		trie.Update(ethutil.Encode(i), list.GetRlp(i))
 	}

eth/backend.go

@@ -135,24 +135,20 @@ func New(config *Config) (*Ethereum, error) {
 	eth.blockPool = NewBlockPool(hasBlock, insertChain, ezp.Verify)
 
 	ethProto := EthProtocol(eth.txPool, eth.chainManager, eth.blockPool)
-	protocols := []p2p.Protocol{ethProto}
+	protocols := []p2p.Protocol{ethProto, eth.whisper.Protocol()}
-
-	if config.Shh {
-		eth.whisper = whisper.New()
-		protocols = append(protocols, eth.whisper.Protocol())
-	}
 
 	nat, err := p2p.ParseNAT(config.NATType, config.PMPGateway)
 	if err != nil {
 		return nil, err
 	}
+	fmt.Println(nat)
 
 	eth.net = &p2p.Server{
 		Identity:  clientId,
 		MaxPeers:  config.MaxPeers,
 		Protocols: protocols,
 		Blacklist: eth.blacklist,
-		NAT:       nat,
+		NAT:       p2p.UPNP(),
 		NoDial:    !config.Dial,
 	}
 

eth/block_pool.go

@@ -1,6 +1,7 @@
 package eth
 
 import (
+	"bytes"
 	"fmt"
 	"math"
 	"math/big"
@@ -24,8 +25,8 @@ const (
 	blocksRequestRepetition    = 1
 	blockHashesRequestInterval = 500 // ms
 	blocksRequestMaxIdleRounds = 100
-	cacheTimeout               = 3 // minutes
+	blockHashesTimeout         = 60  // seconds
-	blockTimeout               = 5 // minutes
+	blocksTimeout              = 120 // seconds
 )
 
 type poolNode struct {
@@ -71,7 +72,12 @@ type peerInfo struct {
 	lock sync.RWMutex
 
 	td               *big.Int
-	currentBlock []byte
+	currentBlockHash []byte
+	currentBlock     *types.Block
+	currentBlockC    chan *types.Block
+	parentHash       []byte
+	headSection      *section
+	headSectionC     chan *section
 	id               string
 
 	requestBlockHashes func([]byte) error
@@ -203,30 +209,39 @@ func (self *BlockPool) Wait(t time.Duration) {
 // AddPeer is called by the eth protocol instance running on the peer after
 // the status message has been received with total difficulty and current block hash
 // AddPeer can only be used once, RemovePeer needs to be called when the peer disconnects
-func (self *BlockPool) AddPeer(td *big.Int, currentBlock []byte, peerId string, requestBlockHashes func([]byte) error, requestBlocks func([][]byte) error, peerError func(int, string, ...interface{})) bool {
+func (self *BlockPool) AddPeer(td *big.Int, currentBlockHash []byte, peerId string, requestBlockHashes func([]byte) error, requestBlocks func([][]byte) error, peerError func(int, string, ...interface{})) (best bool) {
 
 	self.peersLock.Lock()
 	defer self.peersLock.Unlock()
 	peer, ok := self.peers[peerId]
 	if ok {
-		poolLogger.Debugf("Update peer %v with td %v and current block %x", peerId, td, currentBlock[:4])
+		if bytes.Compare(peer.currentBlockHash, currentBlockHash) != 0 {
+			poolLogger.Debugf("Update peer %v with td %v and current block %s", peerId, td, name(currentBlockHash))
+			peer.lock.Lock()
 			peer.td = td
-		peer.currentBlock = currentBlock
+			peer.currentBlockHash = currentBlockHash
+			peer.currentBlock = nil
+			peer.parentHash = nil
+			peer.headSection = nil
+			peer.lock.Unlock()
+		}
 	} else {
 		peer = &peerInfo{
 			td:                 td,
-			currentBlock:       currentBlock,
+			currentBlockHash:   currentBlockHash,
 			id:                 peerId, //peer.Identity().Pubkey()
 			requestBlockHashes: requestBlockHashes,
 			requestBlocks:      requestBlocks,
 			peerError:          peerError,
 			sections:           make(map[string]*section),
+			currentBlockC:      make(chan *types.Block),
+			headSectionC:       make(chan *section),
 		}
 		self.peers[peerId] = peer
-		poolLogger.Debugf("add new peer %v with td %v and current block %x", peerId, td, currentBlock[:4])
+		poolLogger.Debugf("add new peer %v with td %v and current block %x", peerId, td, currentBlockHash[:4])
 	}
 	// check peer current head
-	if self.hasBlock(currentBlock) {
+	if self.hasBlock(currentBlockHash) {
 		// peer not ahead
 		return false
 	}
@@ -234,11 +249,10 @@ func (self *BlockPool) AddPeer(td *big.Int, currentBlock []byte, peerId string,
 	if self.peer == peer {
 		// new block update
 		// peer is already active best peer, request hashes
-		poolLogger.Debugf("[%s] already the best peer. request hashes from %s", peerId, name(currentBlock))
+		poolLogger.Debugf("[%s] already the best peer. Request new head section info from %s", peerId, name(currentBlockHash))
-		peer.requestBlockHashes(currentBlock)
+		peer.headSectionC <- nil
-		return true
+		best = true
-	}
+	} else {
-
 		currentTD := ethutil.Big0
 		if self.peer != nil {
 			currentTD = self.peer.td
@@ -247,9 +261,123 @@ func (self *BlockPool) AddPeer(td *big.Int, currentBlock []byte, peerId string,
 			poolLogger.Debugf("peer %v promoted best peer", peerId)
 			self.switchPeer(self.peer, peer)
 			self.peer = peer
-		return true
+			best = true
 		}
-	return false
+	}
+	return
+}
+
+func (self *BlockPool) requestHeadSection(peer *peerInfo) {
+	self.wg.Add(1)
+	self.procWg.Add(1)
+	poolLogger.Debugf("[%s] head section at [%s] requesting info", peer.id, name(peer.currentBlockHash))
+
+	go func() {
+		var idle bool
+		peer.lock.RLock()
+		quitC := peer.quitC
+		currentBlockHash := peer.currentBlockHash
+		peer.lock.RUnlock()
+		blockHashesRequestTimer := time.NewTimer(0)
+		blocksRequestTimer := time.NewTimer(0)
+		suicide := time.NewTimer(blockHashesTimeout * time.Second)
+		blockHashesRequestTimer.Stop()
+		defer blockHashesRequestTimer.Stop()
+		defer blocksRequestTimer.Stop()
+
+		entry := self.get(currentBlockHash)
+		if entry != nil {
+			entry.node.lock.RLock()
+			currentBlock := entry.node.block
+			entry.node.lock.RUnlock()
+			if currentBlock != nil {
+				peer.lock.Lock()
+				peer.currentBlock = currentBlock
+				peer.parentHash = currentBlock.ParentHash()
+				poolLogger.Debugf("[%s] head block [%s] found", peer.id, name(currentBlockHash))
+				peer.lock.Unlock()
+				blockHashesRequestTimer.Reset(0)
+				blocksRequestTimer.Stop()
+			}
+		}
+
+	LOOP:
+		for {
+
+			select {
+			case <-self.quit:
+				break LOOP
+
+			case <-quitC:
+				poolLogger.Debugf("[%s] head section at [%s] incomplete - quit request loop", peer.id, name(currentBlockHash))
+				break LOOP
+
+			case headSection := <-peer.headSectionC:
+				peer.lock.Lock()
+				peer.headSection = headSection
+				if headSection == nil {
+					oldBlockHash := currentBlockHash
+					currentBlockHash = peer.currentBlockHash
+					poolLogger.Debugf("[%s] head section changed [%s] -> [%s]", peer.id, name(oldBlockHash), name(currentBlockHash))
+					if idle {
+						idle = false
+						suicide.Reset(blockHashesTimeout * time.Second)
+						self.procWg.Add(1)
+					}
+					blocksRequestTimer.Reset(blocksRequestInterval * time.Millisecond)
+				} else {
+					poolLogger.DebugDetailf("[%s] head section at [%s] created", peer.id, name(currentBlockHash))
+					if !idle {
+						idle = true
+						suicide.Stop()
+						self.procWg.Done()
+					}
+				}
+				peer.lock.Unlock()
+				blockHashesRequestTimer.Stop()
+
+			case <-blockHashesRequestTimer.C:
+				poolLogger.DebugDetailf("[%s] head section at [%s] not found, requesting block hashes", peer.id, name(currentBlockHash))
+				peer.requestBlockHashes(currentBlockHash)
+				blockHashesRequestTimer.Reset(blockHashesRequestInterval * time.Millisecond)
+
+			case currentBlock := <-peer.currentBlockC:
+				peer.lock.Lock()
+				peer.currentBlock = currentBlock
+				peer.parentHash = currentBlock.ParentHash()
+				poolLogger.DebugDetailf("[%s] head block [%s] found", peer.id, name(currentBlockHash))
+				peer.lock.Unlock()
+				if self.hasBlock(currentBlock.ParentHash()) {
+					if err := self.insertChain(types.Blocks([]*types.Block{currentBlock})); err != nil {
+						peer.peerError(ErrInvalidBlock, "%v", err)
+					}
+					if !idle {
+						idle = true
+						suicide.Stop()
+						self.procWg.Done()
+					}
+				} else {
+					blockHashesRequestTimer.Reset(0)
+				}
+				blocksRequestTimer.Stop()
+
+			case <-blocksRequestTimer.C:
+				peer.lock.RLock()
+				poolLogger.DebugDetailf("[%s] head block [%s] not found, requesting", peer.id, name(currentBlockHash))
+				peer.requestBlocks([][]byte{peer.currentBlockHash})
+				peer.lock.RUnlock()
+				blocksRequestTimer.Reset(blocksRequestInterval * time.Millisecond)
+
+			case <-suicide.C:
+				peer.peerError(ErrInsufficientChainInfo, "peer failed to provide block hashes or head block for block hash %x", currentBlockHash)
+				break LOOP
+			}
+		}
+		self.wg.Done()
+		if !idle {
+			self.procWg.Done()
+		}
+	}()
 }
 
 // RemovePeer is called by the eth protocol when the peer disconnects
@@ -274,13 +402,13 @@ func (self *BlockPool) RemovePeer(peerId string) {
 				newPeer = info
 			}
 		}
-		self.peer = newPeer
-		self.switchPeer(peer, newPeer)
 		if newPeer != nil {
 			poolLogger.Debugf("peer %v with td %v promoted to best peer", newPeer.id, newPeer.td)
 		} else {
 			poolLogger.Warnln("no peers")
 		}
+		self.peer = newPeer
+		self.switchPeer(peer, newPeer)
 	}
 }
 
@@ -299,25 +427,56 @@ func (self *BlockPool) AddBlockHashes(next func() ([]byte, bool), peerId string)
 		return
 	}
 	// peer is still the best
-	poolLogger.Debugf("adding hashes for best peer %s", peerId)
 
 	var size, n int
 	var hash []byte
-	var ok bool
+	var ok, headSection bool
-	var section, child, parent *section
+	var sec, child, parent *section
 	var entry *poolEntry
 	var nodes []*poolNode
+	bestPeer := peer
+
+	hash, ok = next()
+	peer.lock.Lock()
+	if bytes.Compare(peer.parentHash, hash) == 0 {
+		if self.hasBlock(peer.currentBlockHash) {
+			return
+		}
+		poolLogger.Debugf("adding hashes at chain head for best peer %s starting from [%s]", peerId, name(peer.currentBlockHash))
+		headSection = true
+
+		if entry := self.get(peer.currentBlockHash); entry == nil {
+			node := &poolNode{
+				hash:    peer.currentBlockHash,
+				block:   peer.currentBlock,
+				peer:    peerId,
+				blockBy: peerId,
+			}
+			if size == 0 {
+				sec = newSection()
+			}
+			nodes = append(nodes, node)
+			size++
+			n++
+		} else {
+			child = entry.section
+		}
+	} else {
+		poolLogger.Debugf("adding hashes for best peer %s starting from [%s]", peerId, name(hash))
+	}
+	quitC := peer.quitC
+	peer.lock.Unlock()
 
 LOOP:
 	// iterate using next (rlp stream lazy decoder) feeding hashesC
-	for hash, ok = next(); ok; hash, ok = next() {
+	for ; ok; hash, ok = next() {
 		n++
 		select {
 		case <-self.quit:
 			return
-		case <-peer.quitC:
+		case <-quitC:
 			// if the peer is demoted, no more hashes taken
-			peer = nil
+			bestPeer = nil
 			break LOOP
 		default:
 		}
@@ -325,8 +484,8 @@ LOOP:
 			// check if known block connecting the downloaded chain to our blockchain
 			poolLogger.DebugDetailf("[%s] known block", name(hash))
 			// mark child as absolute pool root with parent known to blockchain
-			if section != nil {
+			if sec != nil {
-				self.connectToBlockChain(section)
+				self.connectToBlockChain(sec)
 			} else {
 				if child != nil {
 					self.connectToBlockChain(child)
@@ -340,6 +499,7 @@ LOOP:
 			// reached a known chain in the pool
 			if entry.node == entry.section.bottom && n == 1 {
 				// the first block hash received is an orphan in the pool, so rejoice and continue
+				poolLogger.DebugDetailf("[%s] connecting child section", sectionName(entry.section))
 				child = entry.section
 				continue LOOP
 			}
@@ -353,7 +513,7 @@ LOOP:
 			peer: peerId,
 		}
 		if size == 0 {
-			section = newSection()
+			sec = newSection()
 		}
 		nodes = append(nodes, node)
 		size++
@@ -379,10 +539,10 @@ LOOP:
 	}
 
 	if size > 0 {
-		self.processSection(section, nodes)
+		self.processSection(sec, nodes)
-		poolLogger.DebugDetailf("[%s]->[%s](%v)->[%s] new chain section", sectionName(parent), sectionName(section), size, sectionName(child))
+		poolLogger.DebugDetailf("[%s]->[%s](%v)->[%s] new chain section", sectionName(parent), sectionName(sec), size, sectionName(child))
-		self.link(parent, section)
+		self.link(parent, sec)
-		self.link(section, child)
+		self.link(sec, child)
 	} else {
 		poolLogger.DebugDetailf("[%s]->[%s] connecting known sections", sectionName(parent), sectionName(child))
 		self.link(parent, child)
@@ -390,15 +550,31 @@ LOOP:
 
 	self.chainLock.Unlock()
 
-	if parent != nil && peer != nil {
+	if parent != nil && bestPeer != nil {
 		self.activateChain(parent, peer)
 		poolLogger.Debugf("[%s] activate parent section [%s]", name(parent.top.hash), sectionName(parent))
 	}
 
-	if section != nil {
+	if sec != nil {
-		peer.addSection(section.top.hash, section)
+		peer.addSection(sec.top.hash, sec)
-		section.controlC <- peer
+		// request next section here once, only repeat if bottom block arrives,
-		poolLogger.Debugf("[%s] activate new section", sectionName(section))
+		// otherwise no way to check if it arrived
+		peer.requestBlockHashes(sec.bottom.hash)
+		sec.controlC <- bestPeer
+		poolLogger.Debugf("[%s] activate new section", sectionName(sec))
+	}
+
+	if headSection {
+		var headSec *section
+		switch {
+		case sec != nil:
+			headSec = sec
+		case child != nil:
+			headSec = child
+		default:
+			headSec = parent
+		}
+		peer.headSectionC <- headSec
 	}
 }
 
@@ -426,14 +602,21 @@ func sectionName(section *section) (name string) {
 // only the first PoW-valid block for a hash is considered legit
 func (self *BlockPool) AddBlock(block *types.Block, peerId string) {
 	hash := block.Hash()
-	if self.hasBlock(hash) {
+	self.peersLock.Lock()
-		poolLogger.DebugDetailf("block [%s] already known", name(hash))
+	peer := self.peer
-		return
+	self.peersLock.Unlock()
-	}
+
 	entry := self.get(hash)
+	if bytes.Compare(hash, peer.currentBlockHash) == 0 {
+		poolLogger.Debugf("add head block [%s] for peer %s", name(hash), peerId)
+		peer.currentBlockC <- block
+	} else {
 		if entry == nil {
-		poolLogger.Warnf("unrequested block [%x] by peer %s", hash, peerId)
+			poolLogger.Warnf("unrequested block [%s] by peer %s", name(hash), peerId)
 			self.peerError(peerId, ErrUnrequestedBlock, "%x", hash)
+		}
+	}
+	if entry == nil {
 		return
 	}
 
@@ -443,17 +626,21 @@ func (self *BlockPool) AddBlock(block *types.Block, peerId string) {
 
 	// check if block already present
 	if node.block != nil {
-		poolLogger.DebugDetailf("block [%x] already sent by %s", name(hash), node.blockBy)
+		poolLogger.DebugDetailf("block [%s] already sent by %s", name(hash), node.blockBy)
 		return
 	}
 
+	if self.hasBlock(hash) {
+		poolLogger.DebugDetailf("block [%s] already known", name(hash))
+	} else {
+
 		// validate block for PoW
 		if !self.verifyPoW(block) {
-		poolLogger.Warnf("invalid pow on block [%x] by peer %s", hash, peerId)
+			poolLogger.Warnf("invalid pow on block [%s] by peer %s", name(hash), peerId)
 			self.peerError(peerId, ErrInvalidPoW, "%x", hash)
 			return
 		}
-
+	}
 	poolLogger.Debugf("added block [%s] sent by peer %s", name(hash), peerId)
 	node.block = block
 	node.blockBy = peerId
@@ -544,23 +731,23 @@ LOOP:
 // - when turned off (if peer disconnects and new peer connects with alternative chain), no blockrequests are made but absolute expiry timer is ticking
 // - when turned back on it recursively calls itself on the root of the next chain section
 // - when exits, signals to
-func (self *BlockPool) processSection(section *section, nodes []*poolNode) {
+func (self *BlockPool) processSection(sec *section, nodes []*poolNode) {
 
 	for i, node := range nodes {
-		entry := &poolEntry{node: node, section: section, index: i}
+		entry := &poolEntry{node: node, section: sec, index: i}
 		self.set(node.hash, entry)
 	}
 
-	section.bottom = nodes[len(nodes)-1]
+	sec.bottom = nodes[len(nodes)-1]
-	section.top = nodes[0]
+	sec.top = nodes[0]
-	section.nodes = nodes
+	sec.nodes = nodes
-	poolLogger.DebugDetailf("[%s] setup section process", sectionName(section))
+	poolLogger.DebugDetailf("[%s] setup section process", sectionName(sec))
 
 	self.wg.Add(1)
 	go func() {
 
 		// absolute time after which sub-chain is killed if not complete (some blocks are missing)
-		suicideTimer := time.After(blockTimeout * time.Minute)
+		suicideTimer := time.After(blocksTimeout * time.Second)
 
 		var peer, newPeer *peerInfo
 
@@ -580,21 +767,23 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) {
 		var insertChain bool
 		var quitC chan bool
 
-		var blockChainC = section.blockChainC
+		var blockChainC = sec.blockChainC
+
+		var parentHash []byte
 
 	LOOP:
 		for {
 
 			if insertChain {
 				insertChain = false
-				rest, err := self.addSectionToBlockChain(section)
+				rest, err := self.addSectionToBlockChain(sec)
 				if err != nil {
-					close(section.suicideC)
+					close(sec.suicideC)
 					continue LOOP
 				}
 				if rest == 0 {
 					blocksRequestsComplete = true
-					child := self.getChild(section)
+					child := self.getChild(sec)
 					if child != nil {
 						self.connectToBlockChain(child)
 					}
@@ -603,7 +792,7 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) {
 
 			if blockHashesRequestsComplete && blocksRequestsComplete {
 				// not waiting for hashes any more
-				poolLogger.Debugf("[%s] section complete %v blocks retrieved (%v attempts), hash requests complete on root (%v attempts)", sectionName(section), depth, blocksRequests, blockHashesRequests)
+				poolLogger.Debugf("[%s] section complete %v blocks retrieved (%v attempts), hash requests complete on root (%v attempts)", sectionName(sec), depth, blocksRequests, blockHashesRequests)
 				break LOOP
 			} // otherwise suicide if no hashes coming
 
@@ -611,11 +800,12 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) {
 				// went through all blocks in section
 				if missing == 0 {
 					// no missing blocks
-					poolLogger.DebugDetailf("[%s] got all blocks. process complete (%v total blocksRequests): missing %v/%v/%v", sectionName(section), blocksRequests, missing, lastMissing, depth)
+					poolLogger.DebugDetailf("[%s] got all blocks. process complete (%v total blocksRequests): missing %v/%v/%v", sectionName(sec), blocksRequests, missing, lastMissing, depth)
 					blocksRequestsComplete = true
 					blocksRequestTimer = nil
 					blocksRequestTime = false
 				} else {
+					poolLogger.DebugDetailf("[%s] section checked: missing %v/%v/%v", sectionName(sec), missing, lastMissing, depth)
 					// some missing blocks
 					blocksRequests++
 					if len(hashes) > 0 {
@@ -630,8 +820,8 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) {
 							idle++
 							// too many idle rounds
 							if idle >= blocksRequestMaxIdleRounds {
-								poolLogger.DebugDetailf("[%s] block requests had %v idle rounds (%v total attempts): missing %v/%v/%v\ngiving up...", sectionName(section), idle, blocksRequests, missing, lastMissing, depth)
+								poolLogger.DebugDetailf("[%s] block requests had %v idle rounds (%v total attempts): missing %v/%v/%v\ngiving up...", sectionName(sec), idle, blocksRequests, missing, lastMissing, depth)
-								close(section.suicideC)
+								close(sec.suicideC)
 							}
 						} else {
 							idle = 0
@@ -653,22 +843,39 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) {
 			//
 
 			if ready && blocksRequestTime && !blocksRequestsComplete {
-				poolLogger.DebugDetailf("[%s] check if new blocks arrived (attempt %v): missing %v/%v/%v", sectionName(section), blocksRequests, missing, lastMissing, depth)
+				poolLogger.DebugDetailf("[%s] check if new blocks arrived (attempt %v): missing %v/%v/%v", sectionName(sec), blocksRequests, missing, lastMissing, depth)
 				blocksRequestTimer = time.After(blocksRequestInterval * time.Millisecond)
 				blocksRequestTime = false
 				processC = offC
 			}
 
 			if blockHashesRequestTime {
-				if self.getParent(section) != nil {
+				var parentSection = self.getParent(sec)
+				if parentSection == nil {
+					if parent := self.get(parentHash); parent != nil {
+						parentSection = parent.section
+						self.chainLock.Lock()
+						self.link(parentSection, sec)
+						self.chainLock.Unlock()
+					} else {
+						if self.hasBlock(parentHash) {
+							insertChain = true
+							blockHashesRequestTime = false
+							blockHashesRequestTimer = nil
+							blockHashesRequestsComplete = true
+							continue LOOP
+						}
+					}
+				}
+				if parentSection != nil {
 					// if not root of chain, switch off
-					poolLogger.DebugDetailf("[%s] parent found, hash requests deactivated (after %v total attempts)\n", sectionName(section), blockHashesRequests)
+					poolLogger.DebugDetailf("[%s] parent found, hash requests deactivated (after %v total attempts)\n", sectionName(sec), blockHashesRequests)
 					blockHashesRequestTimer = nil
 					blockHashesRequestsComplete = true
 				} else {
 					blockHashesRequests++
-					poolLogger.Debugf("[%s] hash request on root (%v total attempts)\n", sectionName(section), blockHashesRequests)
+					poolLogger.Debugf("[%s] hash request on root (%v total attempts)\n", sectionName(sec), blockHashesRequests)
-					peer.requestBlockHashes(section.bottom.hash)
+					peer.requestBlockHashes(sec.bottom.hash)
 					blockHashesRequestTimer = time.After(blockHashesRequestInterval * time.Millisecond)
 				}
 				blockHashesRequestTime = false
@@ -682,27 +889,27 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) {
 				// peer quit or demoted, put section in idle mode
 				quitC = nil
 				go func() {
-					section.controlC <- nil
+					sec.controlC <- nil
 				}()
 
 			case <-self.purgeC:
 				suicideTimer = time.After(0)
 
 			case <-suicideTimer:
-				close(section.suicideC)
+				close(sec.suicideC)
-				poolLogger.Debugf("[%s] timeout. (%v total attempts): missing %v/%v/%v", sectionName(section), blocksRequests, missing, lastMissing, depth)
+				poolLogger.Debugf("[%s] timeout. (%v total attempts): missing %v/%v/%v", sectionName(sec), blocksRequests, missing, lastMissing, depth)
 
-			case <-section.suicideC:
+			case <-sec.suicideC:
-				poolLogger.Debugf("[%s] suicide", sectionName(section))
+				poolLogger.Debugf("[%s] suicide", sectionName(sec))
 
 				// first delink from child and parent under chainlock
 				self.chainLock.Lock()
-				self.link(nil, section)
+				self.link(nil, sec)
-				self.link(section, nil)
+				self.link(sec, nil)
 				self.chainLock.Unlock()
 				// delete node entries from pool index under pool lock
 				self.lock.Lock()
-				for _, node := range section.nodes {
+				for _, node := range sec.nodes {
 					delete(self.pool, string(node.hash))
 				}
 				self.lock.Unlock()
@@ -710,20 +917,20 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) {
 				break LOOP
 
 			case <-blocksRequestTimer:
-				poolLogger.DebugDetailf("[%s] block request time", sectionName(section))
+				poolLogger.DebugDetailf("[%s] block request time", sectionName(sec))
 				blocksRequestTime = true
 
 			case <-blockHashesRequestTimer:
-				poolLogger.DebugDetailf("[%s] hash request time", sectionName(section))
+				poolLogger.DebugDetailf("[%s] hash request time", sectionName(sec))
 				blockHashesRequestTime = true
 
-			case newPeer = <-section.controlC:
+			case newPeer = <-sec.controlC:
 
 				// active -> idle
 				if peer != nil && newPeer == nil {
 					self.procWg.Done()
 					if init {
-						poolLogger.Debugf("[%s] idle mode (%v total attempts): missing %v/%v/%v", sectionName(section), blocksRequests, missing, lastMissing, depth)
+						poolLogger.Debugf("[%s] idle mode (%v total attempts): missing %v/%v/%v", sectionName(sec), blocksRequests, missing, lastMissing, depth)
 					}
 					blocksRequestTime = false
 					blocksRequestTimer = nil
@@ -739,11 +946,11 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) {
 				if peer == nil && newPeer != nil {
 					self.procWg.Add(1)
 
-					poolLogger.Debugf("[%s] active mode", sectionName(section))
+					poolLogger.Debugf("[%s] active mode", sectionName(sec))
 					if !blocksRequestsComplete {
 						blocksRequestTime = true
 					}
-					if !blockHashesRequestsComplete {
+					if !blockHashesRequestsComplete && parentHash != nil {
 						blockHashesRequestTime = true
 					}
 					if !init {
@@ -753,13 +960,13 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) {
 						missing = 0
 						self.wg.Add(1)
 						self.procWg.Add(1)
-						depth = len(section.nodes)
+						depth = len(sec.nodes)
 						lastMissing = depth
 						// if not run at least once fully, launch iterator
 						go func() {
 							var node *poolNode
 						IT:
-							for _, node = range section.nodes {
+							for _, node = range sec.nodes {
 								select {
 								case processC <- node:
 								case <-self.quit:
@@ -771,7 +978,7 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) {
 							self.procWg.Done()
 						}()
 					} else {
-						poolLogger.Debugf("[%s] restore earlier state", sectionName(section))
+						poolLogger.Debugf("[%s] restore earlier state", sectionName(sec))
 						processC = offC
 					}
 				}
@@ -781,7 +988,7 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) {
 				}
 				peer = newPeer
 
-			case waiter := <-section.forkC:
+			case waiter := <-sec.forkC:
 				// this case just blocks the process until section is split at the fork
 				<-waiter
 				init = false
@@ -794,7 +1001,7 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) {
 					init = true
 					done = true
 					processC = make(chan *poolNode, missing)
-					poolLogger.DebugDetailf("[%s] section initalised: missing %v/%v/%v", sectionName(section), missing, lastMissing, depth)
+					poolLogger.DebugDetailf("[%s] section initalised: missing %v/%v/%v", sectionName(sec), missing, lastMissing, depth)
 					continue LOOP
 				}
 				if ready {
@@ -811,17 +1018,24 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) {
 					missing++
 					hashes = append(hashes, node.hash)
 					if len(hashes) == blockBatchSize {
-						poolLogger.Debugf("[%s] request %v missing blocks", sectionName(section), len(hashes))
+						poolLogger.Debugf("[%s] request %v missing blocks", sectionName(sec), len(hashes))
 						self.requestBlocks(blocksRequests, hashes)
 						hashes = nil
 					}
 					missingC <- node
 				} else {
-					if blockChainC == nil && i == lastMissing {
+					if i == lastMissing {
+						if blockChainC == nil {
 							insertChain = true
+						} else {
+							if parentHash == nil {
+								parentHash = block.ParentHash()
+								poolLogger.Debugf("[%s] found root block [%s]", sectionName(sec), name(parentHash))
+								blockHashesRequestTime = true
+							}
+						}
 					}
 				}
-				poolLogger.Debugf("[%s] %v/%v/%v/%v", sectionName(section), i, missing, lastMissing, depth)
 				if i == lastMissing && init {
 					done = true
 				}
@@ -829,23 +1043,22 @@ func (self *BlockPool) processSection(section *section, nodes []*poolNode) {
 			case <-blockChainC:
 				// closed blockChain channel indicates that the blockpool is reached
 				// connected to the blockchain, insert the longest chain of blocks
-				poolLogger.Debugf("[%s] reached blockchain", sectionName(section))
+				poolLogger.Debugf("[%s] reached blockchain", sectionName(sec))
 				blockChainC = nil
 				// switch off hash requests in case they were on
 				blockHashesRequestTime = false
 				blockHashesRequestTimer = nil
 				blockHashesRequestsComplete = true
 				// section root has block
-				if len(section.nodes) > 0 && section.nodes[len(section.nodes)-1].block != nil {
+				if len(sec.nodes) > 0 && sec.nodes[len(sec.nodes)-1].block != nil {
 					insertChain = true
 				}
 				continue LOOP
 
 			} // select
 		} // for
-		poolLogger.Debugf("[%s] section complete: %v block hashes requests - %v block requests - missing %v/%v/%v", sectionName(section), blockHashesRequests, blocksRequests, missing, lastMissing, depth)
 
-		close(section.offC)
+		close(sec.offC)
 
 		self.wg.Done()
 		if peer != nil {
@@ -917,22 +1130,28 @@ func (self *peerInfo) addSection(hash []byte, section *section) (found *section)
 	defer self.lock.Unlock()
 	key := string(hash)
 	found = self.sections[key]
-	poolLogger.DebugDetailf("[%s] section process %s registered", sectionName(section), self.id)
+	poolLogger.DebugDetailf("[%s] section process stored for %s", sectionName(section), self.id)
 	self.sections[key] = section
 	return
 }
 
 func (self *BlockPool) switchPeer(oldPeer, newPeer *peerInfo) {
 	if newPeer != nil {
-		entry := self.get(newPeer.currentBlock)
+		newPeer.quitC = make(chan bool)
-		if entry == nil {
-			poolLogger.Debugf("[%s] head block [%s] not found, requesting hashes", newPeer.id, name(newPeer.currentBlock))
-			newPeer.requestBlockHashes(newPeer.currentBlock)
-		} else {
-			poolLogger.Debugf("[%s] head block [%s] found, activate chain at section [%s]", newPeer.id, name(newPeer.currentBlock), sectionName(entry.section))
-			self.activateChain(entry.section, newPeer)
-		}
 		poolLogger.DebugDetailf("[%s] activate section processes", newPeer.id)
+		var addSections []*section
+		for hash, section := range newPeer.sections {
+			// split sections get reorganised here
+			if string(section.top.hash) != hash {
+				addSections = append(addSections, section)
+				if entry := self.get([]byte(hash)); entry != nil {
+					addSections = append(addSections, entry.section)
+				}
+			}
+		}
+		for _, section := range addSections {
+			newPeer.sections[string(section.top.hash)] = section
+		}
 		for hash, section := range newPeer.sections {
 			// this will block if section process is waiting for peer lock
 			select {
@@ -940,12 +1159,26 @@ func (self *BlockPool) switchPeer(oldPeer, newPeer *peerInfo) {
 				poolLogger.DebugDetailf("[%s][%x] section process complete - remove", newPeer.id, hash[:4])
 				delete(newPeer.sections, hash)
 			case section.controlC <- newPeer:
-				poolLogger.DebugDetailf("[%s][%x] registered peer with section", newPeer.id, hash[:4])
+				poolLogger.DebugDetailf("[%s][%x] activates section [%s]", newPeer.id, hash[:4], sectionName(section))
 			}
 		}
-		newPeer.quitC = make(chan bool)
+		newPeer.lock.Lock()
+		headSection := newPeer.headSection
+		currentBlockHash := newPeer.currentBlockHash
+		newPeer.lock.Unlock()
+		if headSection == nil {
+			poolLogger.DebugDetailf("[%s] head section for [%s] not created, requesting info", newPeer.id, name(currentBlockHash))
+			self.requestHeadSection(newPeer)
+		} else {
+			if entry := self.get(currentBlockHash); entry != nil {
+				headSection = entry.section
+			}
+			poolLogger.DebugDetailf("[%s] activate chain at head section [%s] for current head [%s]", newPeer.id, sectionName(headSection), name(currentBlockHash))
+			self.activateChain(headSection, newPeer)
+		}
 	}
 	if oldPeer != nil {
+		poolLogger.DebugDetailf("[%s] quit section processes", oldPeer.id)
 		close(oldPeer.quitC)
 	}
 }

eth/block_pool_test.go

@@ -18,7 +18,7 @@ import (
 
 const waitTimeout = 60 // seconds
 
-var logsys = ethlogger.NewStdLogSystem(os.Stdout, log.LstdFlags, ethlogger.LogLevel(ethlogger.DebugLevel))
+var logsys = ethlogger.NewStdLogSystem(os.Stdout, log.LstdFlags, ethlogger.LogLevel(ethlogger.DebugDetailLevel))
 
 var ini = false
 
@@ -336,12 +336,12 @@ func (self *peerTester) AddPeer() bool {
 
 // peer sends blockhashes if and when gets a request
 func (self *peerTester) AddBlockHashes(indexes ...int) {
-	i := 0
 	fmt.Printf("ready to add block hashes %v\n", indexes)
 
 	self.waitBlockHashesRequests(indexes[0])
 	fmt.Printf("adding block hashes %v\n", indexes)
 	hashes := self.hashPool.indexesToHashes(indexes)
+	i := 1
 	next := func() (hash []byte, ok bool) {
 		if i < len(hashes) {
 			hash = hashes[i]
@@ -415,7 +415,7 @@ func TestAddPeer(t *testing.T) {
 	if blockPool.peer.id != "peer0" {
 		t.Errorf("peer0 (TD=1) not set as best")
 	}
-	peer0.checkBlockHashesRequests(0)
+	// peer0.checkBlockHashesRequests(0)
 
 	best = peer2.AddPeer()
 	if !best {
@@ -424,7 +424,7 @@ func TestAddPeer(t *testing.T) {
 	if blockPool.peer.id != "peer2" {
 		t.Errorf("peer2 (TD=3) not set as best")
 	}
-	peer2.checkBlockHashesRequests(2)
+	peer2.waitBlocksRequests(2)
 
 	best = peer1.AddPeer()
 	if best {
@@ -449,7 +449,7 @@ func TestAddPeer(t *testing.T) {
 	if blockPool.peer.td.Cmp(big.NewInt(int64(4))) != 0 {
 		t.Errorf("peer2 TD not updated")
 	}
-	peer2.checkBlockHashesRequests(2, 3)
+	peer2.waitBlocksRequests(3)
 
 	peer1.td = 3
 	peer1.currentBlock = 2
@@ -474,7 +474,7 @@ func TestAddPeer(t *testing.T) {
 	if blockPool.peer.id != "peer1" {
 		t.Errorf("existing peer1 (TD=3) should be set as best peer")
 	}
-	peer1.checkBlockHashesRequests(2)
+	peer1.waitBlocksRequests(2)
 
 	blockPool.RemovePeer("peer1")
 	peer, best = blockPool.getPeer("peer1")
@@ -485,6 +485,7 @@ func TestAddPeer(t *testing.T) {
 	if blockPool.peer.id != "peer0" {
 		t.Errorf("existing peer0 (TD=1) should be set as best peer")
 	}
+	peer0.waitBlocksRequests(0)
 
 	blockPool.RemovePeer("peer0")
 	peer, best = blockPool.getPeer("peer0")
@@ -502,7 +503,7 @@ func TestAddPeer(t *testing.T) {
 	if blockPool.peer.id != "peer0" {
 		t.Errorf("peer0 (TD=1) should be set as best")
 	}
-	peer0.checkBlockHashesRequests(0, 0, 3)
+	peer0.waitBlocksRequests(3)
 
 	blockPool.Stop()
 
@@ -513,17 +514,36 @@ func TestPeerWithKnownBlock(t *testing.T) {
 	_, blockPool, blockPoolTester := newTestBlockPool(t)
 	blockPoolTester.refBlockChain[0] = nil
 	blockPoolTester.blockChain[0] = nil
-	// hashPool, blockPool, blockPoolTester := newTestBlockPool()
 	blockPool.Start()
 
 	peer0 := blockPoolTester.newPeer("0", 1, 0)
 	peer0.AddPeer()
 
+	blockPool.Wait(waitTimeout * time.Second)
 	blockPool.Stop()
 	// no request on known block
 	peer0.checkBlockHashesRequests()
 }
 
+func TestPeerWithKnownParentBlock(t *testing.T) {
+	logInit()
+	_, blockPool, blockPoolTester := newTestBlockPool(t)
+	blockPoolTester.initRefBlockChain(1)
+	blockPoolTester.blockChain[0] = nil
+	blockPool.Start()
+
+	peer0 := blockPoolTester.newPeer("0", 1, 1)
+	peer0.AddPeer()
+	peer0.AddBlocks(0, 1)
+
+	blockPool.Wait(waitTimeout * time.Second)
+	blockPool.Stop()
+	peer0.checkBlocksRequests([]int{1})
+	peer0.checkBlockHashesRequests()
+	blockPoolTester.refBlockChain[1] = []int{}
+	blockPoolTester.checkBlockChain(blockPoolTester.refBlockChain)
+}
+
 func TestSimpleChain(t *testing.T) {
 	logInit()
 	_, blockPool, blockPoolTester := newTestBlockPool(t)
@@ -534,8 +554,9 @@ func TestSimpleChain(t *testing.T) {
 
 	peer1 := blockPoolTester.newPeer("peer1", 1, 2)
 	peer1.AddPeer()
+	peer1.AddBlocks(1, 2)
 	go peer1.AddBlockHashes(2, 1, 0)
-	peer1.AddBlocks(0, 1, 2)
+	peer1.AddBlocks(0, 1)
 
 	blockPool.Wait(waitTimeout * time.Second)
 	blockPool.Stop()
@@ -543,6 +564,26 @@ func TestSimpleChain(t *testing.T) {
 	blockPoolTester.checkBlockChain(blockPoolTester.refBlockChain)
 }
 
+func TestChainConnectingWithParentHash(t *testing.T) {
+	logInit()
+	_, blockPool, blockPoolTester := newTestBlockPool(t)
+	blockPoolTester.blockChain[0] = nil
+	blockPoolTester.initRefBlockChain(3)
+
+	blockPool.Start()
+
+	peer1 := blockPoolTester.newPeer("peer1", 1, 3)
+	peer1.AddPeer()
+	go peer1.AddBlocks(2, 3)
+	go peer1.AddBlockHashes(3, 2, 1)
+	peer1.AddBlocks(0, 1, 2)
+
+	blockPool.Wait(waitTimeout * time.Second)
+	blockPool.Stop()
+	blockPoolTester.refBlockChain[3] = []int{}
+	blockPoolTester.checkBlockChain(blockPoolTester.refBlockChain)
+}
+
 func TestInvalidBlock(t *testing.T) {
 	logInit()
 	_, blockPool, blockPoolTester := newTestBlockPool(t)
@@ -554,8 +595,9 @@ func TestInvalidBlock(t *testing.T) {
 
 	peer1 := blockPoolTester.newPeer("peer1", 1, 3)
 	peer1.AddPeer()
+	go peer1.AddBlocks(2, 3)
 	go peer1.AddBlockHashes(3, 2, 1, 0)
-	peer1.AddBlocks(0, 1, 2, 3)
+	peer1.AddBlocks(0, 1, 2)
 
 	blockPool.Wait(waitTimeout * time.Second)
 	blockPool.Stop()
@@ -566,7 +608,7 @@ func TestInvalidBlock(t *testing.T) {
 			t.Errorf("wrong error, got %v, expected %v", peer1.peerErrors[0], ErrInvalidBlock)
 		}
 	} else {
-		t.Errorf("expected invalid block error, got nothing")
+		t.Errorf("expected invalid block error, got nothing %v", peer1.peerErrors)
 	}
 }
 
@@ -579,7 +621,7 @@ func TestVerifyPoW(t *testing.T) {
 	blockPoolTester.blockPool.verifyPoW = func(b pow.Block) bool {
 		bb, _ := b.(*types.Block)
 		indexes := blockPoolTester.hashPool.hashesToIndexes([][]byte{bb.Hash()})
-		if indexes[0] == 1 && !first {
+		if indexes[0] == 2 && !first {
 			first = true
 			return false
 		} else {
@@ -590,15 +632,17 @@ func TestVerifyPoW(t *testing.T) {
 
 	blockPool.Start()
 
-	peer1 := blockPoolTester.newPeer("peer1", 1, 2)
+	peer1 := blockPoolTester.newPeer("peer1", 1, 3)
 	peer1.AddPeer()
-	go peer1.AddBlockHashes(2, 1, 0)
+	go peer1.AddBlocks(2, 3)
+	go peer1.AddBlockHashes(3, 2, 1, 0)
 	peer1.AddBlocks(0, 1, 2)
-	peer1.AddBlocks(0, 1)
 
-	blockPool.Wait(waitTimeout * time.Second)
+	// blockPool.Wait(waitTimeout * time.Second)
+	time.Sleep(1 * time.Second)
 	blockPool.Stop()
-	blockPoolTester.refBlockChain[2] = []int{}
+	blockPoolTester.refBlockChain[1] = []int{}
+	delete(blockPoolTester.refBlockChain, 2)
 	blockPoolTester.checkBlockChain(blockPoolTester.refBlockChain)
 	if len(peer1.peerErrors) == 1 {
 		if peer1.peerErrors[0] != ErrInvalidPoW {
@@ -620,8 +664,9 @@ func TestMultiSectionChain(t *testing.T) {
 	peer1 := blockPoolTester.newPeer("peer1", 1, 5)
 
 	peer1.AddPeer()
+	go peer1.AddBlocks(4, 5)
 	go peer1.AddBlockHashes(5, 4, 3)
-	go peer1.AddBlocks(2, 3, 4, 5)
+	go peer1.AddBlocks(2, 3, 4)
 	go peer1.AddBlockHashes(3, 2, 1, 0)
 	peer1.AddBlocks(0, 1, 2)
 
@@ -641,14 +686,17 @@ func TestNewBlocksOnPartialChain(t *testing.T) {
 	peer1 := blockPoolTester.newPeer("peer1", 1, 5)
 
 	peer1.AddPeer()
+	go peer1.AddBlocks(4, 5) // partially complete section
 	go peer1.AddBlockHashes(5, 4, 3)
-	peer1.AddBlocks(2, 3) // partially complete section
+	peer1.AddBlocks(3, 4) // partially complete section
 	// peer1 found new blocks
 	peer1.td = 2
 	peer1.currentBlock = 7
 	peer1.AddPeer()
+	go peer1.AddBlocks(6, 7)
 	go peer1.AddBlockHashes(7, 6, 5)
-	go peer1.AddBlocks(3, 4, 5, 6, 7)
+	go peer1.AddBlocks(2, 3)
+	go peer1.AddBlocks(5, 6)
 	go peer1.AddBlockHashes(3, 2, 1, 0) // tests that hash request from known chain root is remembered
 	peer1.AddBlocks(0, 1, 2)
 
@@ -658,35 +706,37 @@ func TestNewBlocksOnPartialChain(t *testing.T) {
 	blockPoolTester.checkBlockChain(blockPoolTester.refBlockChain)
 }
 
-func TestPeerSwitch(t *testing.T) {
+func TestPeerSwitchUp(t *testing.T) {
 	logInit()
 	_, blockPool, blockPoolTester := newTestBlockPool(t)
 	blockPoolTester.blockChain[0] = nil
-	blockPoolTester.initRefBlockChain(6)
+	blockPoolTester.initRefBlockChain(7)
 
 	blockPool.Start()
 
-	peer1 := blockPoolTester.newPeer("peer1", 1, 5)
+	peer1 := blockPoolTester.newPeer("peer1", 1, 6)
-	peer2 := blockPoolTester.newPeer("peer2", 2, 6)
+	peer2 := blockPoolTester.newPeer("peer2", 2, 7)
 	peer2.blocksRequestsMap = peer1.blocksRequestsMap
 
 	peer1.AddPeer()
-	go peer1.AddBlockHashes(5, 4, 3)
+	go peer1.AddBlocks(5, 6)
+	go peer1.AddBlockHashes(6, 5, 4, 3) //
 	peer1.AddBlocks(2, 3)               // section partially complete, block 3 will be preserved after peer demoted
 	peer2.AddPeer()                     // peer2 is promoted as best peer, peer1 is demoted
-	go peer2.AddBlockHashes(6, 5)       //
+	go peer2.AddBlocks(6, 7)
-	go peer2.AddBlocks(4, 5, 6)         // tests that block request for earlier section is remembered
+	go peer2.AddBlockHashes(7, 6)       //
+	go peer2.AddBlocks(4, 5)            // tests that block request for earlier section is remembered
 	go peer1.AddBlocks(3, 4)            // tests that connecting section by demoted peer is remembered and blocks are accepted from demoted peer
 	go peer2.AddBlockHashes(3, 2, 1, 0) // tests that known chain section is activated, hash requests from 3 is remembered
 	peer2.AddBlocks(0, 1, 2)            // final blocks linking to blockchain sent
 
 	blockPool.Wait(waitTimeout * time.Second)
 	blockPool.Stop()
-	blockPoolTester.refBlockChain[6] = []int{}
+	blockPoolTester.refBlockChain[7] = []int{}
 	blockPoolTester.checkBlockChain(blockPoolTester.refBlockChain)
 }
 
-func TestPeerDownSwitch(t *testing.T) {
+func TestPeerSwitchDown(t *testing.T) {
 	logInit()
 	_, blockPool, blockPoolTester := newTestBlockPool(t)
 	blockPoolTester.blockChain[0] = nil
@@ -698,12 +748,39 @@ func TestPeerDownSwitch(t *testing.T) {
 	peer2.blocksRequestsMap = peer1.blocksRequestsMap
 
 	peer2.AddPeer()
-	go peer2.AddBlockHashes(6, 5, 4)
 	peer2.AddBlocks(5, 6)                  // partially complete, section will be preserved
+	go peer2.AddBlockHashes(6, 5, 4)       //
+	peer2.AddBlocks(4, 5)                  //
 	blockPool.RemovePeer("peer2")          // peer2 disconnects
 	peer1.AddPeer()                        // inferior peer1 is promoted as best peer
 	go peer1.AddBlockHashes(4, 3, 2, 1, 0) //
-	go peer1.AddBlocks(3, 4, 5)            // tests that section set by demoted peer is remembered and blocks are accepted
+	go peer1.AddBlocks(3, 4)               // tests that section set by demoted peer is remembered and blocks are accepted , this connects the chain sections together
+	peer1.AddBlocks(0, 1, 2, 3)
+
+	blockPool.Wait(waitTimeout * time.Second)
+	blockPool.Stop()
+	blockPoolTester.refBlockChain[6] = []int{}
+	blockPoolTester.checkBlockChain(blockPoolTester.refBlockChain)
+}
+
+func TestPeerCompleteSectionSwitchDown(t *testing.T) {
+	logInit()
+	_, blockPool, blockPoolTester := newTestBlockPool(t)
+	blockPoolTester.blockChain[0] = nil
+	blockPoolTester.initRefBlockChain(6)
+	blockPool.Start()
+
+	peer1 := blockPoolTester.newPeer("peer1", 1, 4)
+	peer2 := blockPoolTester.newPeer("peer2", 2, 6)
+	peer2.blocksRequestsMap = peer1.blocksRequestsMap
+
+	peer2.AddPeer()
+	peer2.AddBlocks(5, 6)               // partially complete, section will be preserved
+	go peer2.AddBlockHashes(6, 5, 4)    //
+	peer2.AddBlocks(3, 4, 5)            // complete section
+	blockPool.RemovePeer("peer2")       // peer2 disconnects
+	peer1.AddPeer()                     // inferior peer1 is promoted as best peer
+	peer1.AddBlockHashes(4, 3, 2, 1, 0) // tests that hash request are directly connecting if the head block exists
 	peer1.AddBlocks(0, 1, 2, 3)
 
 	blockPool.Wait(waitTimeout * time.Second)
@@ -725,11 +802,13 @@ func TestPeerSwitchBack(t *testing.T) {
 	peer2.blocksRequestsMap = peer1.blocksRequestsMap
 
 	peer2.AddPeer()
+	go peer2.AddBlocks(7, 8)
 	go peer2.AddBlockHashes(8, 7, 6)
 	go peer2.AddBlockHashes(6, 5, 4)
 	peer2.AddBlocks(4, 5)                  // section partially complete
 	peer1.AddPeer()                        // peer1 is promoted as best peer
-	go peer1.AddBlockHashes(11, 10)        // only gives useless results
+	go peer1.AddBlocks(10, 11)             //
+	peer1.AddBlockHashes(11, 10)           // only gives useless results
 	blockPool.RemovePeer("peer1")          // peer1 disconnects
 	go peer2.AddBlockHashes(4, 3, 2, 1, 0) // tests that asking for hashes from 4 is remembered
 	go peer2.AddBlocks(3, 4, 5, 6, 7, 8)   // tests that section 4, 5, 6 and 7, 8 are remembered for missing blocks
@@ -756,11 +835,13 @@ func TestForkSimple(t *testing.T) {
 	peer2.blocksRequestsMap = peer1.blocksRequestsMap
 
 	peer1.AddPeer()
+	go peer1.AddBlocks(8, 9)
 	go peer1.AddBlockHashes(9, 8, 7, 3, 2)
-	peer1.AddBlocks(1, 2, 3, 7, 8, 9)
+	peer1.AddBlocks(1, 2, 3, 7, 8)
 	peer2.AddPeer()                        // peer2 is promoted as best peer
+	go peer2.AddBlocks(5, 6)               //
 	go peer2.AddBlockHashes(6, 5, 4, 3, 2) // fork on 3 -> 4 (earlier child: 7)
-	go peer2.AddBlocks(1, 2, 3, 4, 5, 6)
+	go peer2.AddBlocks(1, 2, 3, 4, 5)
 	go peer2.AddBlockHashes(2, 1, 0)
 	peer2.AddBlocks(0, 1, 2)
 
@@ -790,23 +871,24 @@ func TestForkSwitchBackByNewBlocks(t *testing.T) {
 	peer2.blocksRequestsMap = peer1.blocksRequestsMap
 
 	peer1.AddPeer()
-	go peer1.AddBlockHashes(9, 8, 7, 3, 2)
+	peer1.AddBlocks(8, 9)                  //
-	peer1.AddBlocks(8, 9)                  // partial section
+	go peer1.AddBlockHashes(9, 8, 7, 3, 2) //
+	peer1.AddBlocks(7, 8)                  // partial section
 	peer2.AddPeer()                        //
+	peer2.AddBlocks(5, 6)                  //
 	go peer2.AddBlockHashes(6, 5, 4, 3, 2) // peer2 forks on block 3
-	peer2.AddBlocks(1, 2, 3, 4, 5, 6)      //
+	peer2.AddBlocks(1, 2, 3, 4, 5)         //
 
 	// peer1 finds new blocks
 	peer1.td = 3
 	peer1.currentBlock = 11
 	peer1.AddPeer()
+	go peer1.AddBlocks(10, 11)
 	go peer1.AddBlockHashes(11, 10, 9)
-	peer1.AddBlocks(7, 8, 9, 10, 11)
+	peer1.AddBlocks(9, 10)
-	go peer1.AddBlockHashes(7, 3) // tests that hash request from fork root is remembered
 	go peer1.AddBlocks(3, 7)         // tests that block requests on earlier fork are remembered
-	// go peer1.AddBlockHashes(1, 0) // tests that hash request from root of connecting chain section (added by demoted peer) is remembered
 	go peer1.AddBlockHashes(2, 1, 0) // tests that hash request from root of connecting chain section (added by demoted peer) is remembered
-	peer1.AddBlocks(0, 1, 2, 3)
+	peer1.AddBlocks(0, 1)
 
 	blockPool.Wait(waitTimeout * time.Second)
 	blockPool.Stop()
@@ -834,16 +916,18 @@ func TestForkSwitchBackByPeerSwitchBack(t *testing.T) {
 	peer2.blocksRequestsMap = peer1.blocksRequestsMap
 
 	peer1.AddPeer()
+	go peer1.AddBlocks(8, 9)
 	go peer1.AddBlockHashes(9, 8, 7, 3, 2)
-	peer1.AddBlocks(8, 9)
+	peer1.AddBlocks(7, 8)
-	peer2.AddPeer()                        //
+	peer2.AddPeer()
+	go peer2.AddBlocks(5, 6)               //
 	go peer2.AddBlockHashes(6, 5, 4, 3, 2) // peer2 forks on block 3
-	peer2.AddBlocks(2, 3, 4, 5, 6)         //
+	peer2.AddBlocks(2, 3, 4, 5)            //
 	blockPool.RemovePeer("peer2")          // peer2 disconnects, peer1 is promoted again as best peer
-	peer1.AddBlockHashes(7, 3)             // tests that hash request from fork root is remembered
+	go peer1.AddBlocks(3, 7)               // tests that block requests on earlier fork are remembered and orphan section relinks to existing parent block
-	go peer1.AddBlocks(3, 7, 8)            // tests that block requests on earlier fork are remembered
+	go peer1.AddBlocks(1, 2)               //
 	go peer1.AddBlockHashes(2, 1, 0)       //
-	peer1.AddBlocks(0, 1, 2, 3)
+	peer1.AddBlocks(0, 1)
 
 	blockPool.Wait(waitTimeout * time.Second)
 	blockPool.Stop()
@@ -871,16 +955,18 @@ func TestForkCompleteSectionSwitchBackByPeerSwitchBack(t *testing.T) {
 	peer2.blocksRequestsMap = peer1.blocksRequestsMap
 
 	peer1.AddPeer()
+	go peer1.AddBlocks(8, 9)
 	go peer1.AddBlockHashes(9, 8, 7)
-	peer1.AddBlocks(3, 7, 8, 9) // make sure this section is complete
+	peer1.AddBlocks(3, 7, 8) // make sure this section is complete
 	time.Sleep(1 * time.Second)
 	go peer1.AddBlockHashes(7, 3, 2)       // block 3/7 is section boundary
-	peer1.AddBlocks(2, 3)                  // partially complete sections
+	peer1.AddBlocks(2, 3)                  // partially complete sections block 2 missing
 	peer2.AddPeer()                        //
+	go peer2.AddBlocks(5, 6)               //
 	go peer2.AddBlockHashes(6, 5, 4, 3, 2) // peer2 forks on block 3
-	peer2.AddBlocks(2, 3, 4, 5, 6)         // block 2 still missing.
+	peer2.AddBlocks(2, 3, 4, 5)            // block 2 still missing.
 	blockPool.RemovePeer("peer2")          // peer2 disconnects, peer1 is promoted again as best peer
-	peer1.AddBlockHashes(7, 3)             // tests that hash request from fork root is remembered even though section process completed
+	// peer1.AddBlockHashes(7, 3)             // tests that hash request from fork root is remembered even though section process completed
 	go peer1.AddBlockHashes(2, 1, 0) //
 	peer1.AddBlocks(0, 1, 2)
 

eth/error.go

@@ -16,6 +16,7 @@ const (
 	ErrInvalidBlock
 	ErrInvalidPoW
 	ErrUnrequestedBlock
+	ErrInsufficientChainInfo
 )
 
 var errorToString = map[int]string{
@@ -30,6 +31,7 @@ var errorToString = map[int]string{
 	ErrInvalidBlock:            "Invalid block",
 	ErrInvalidPoW:              "Invalid PoW",
 	ErrUnrequestedBlock:        "Unrequested block",
+	ErrInsufficientChainInfo:   "Insufficient chain info",
 }
 
 type protocolError struct {

eth/protocol.go

@@ -211,16 +211,6 @@ func (self *ethProtocol) handle() error {
 		// uses AddPeer followed by AddHashes, AddBlock only if peer is the best peer
 		// (or selected as new best peer)
 		if self.blockPool.AddPeer(request.TD, hash, self.id, self.requestBlockHashes, self.requestBlocks, self.protoErrorDisconnect) {
-			called := true
-			iter := func() ([]byte, bool) {
-				if called {
-					called = false
-					return hash, true
-				} else {
-					return nil, false
-				}
-			}
-			self.blockPool.AddBlockHashes(iter, self.id)
 			self.blockPool.AddBlock(request.Block, self.id)
 		}
 

eth/test/tests/02.sh

@@ -12,8 +12,8 @@ EOF
 
 peer 11 01
 peer 12 02
-P13ID=$PID
+P12ID=$PID
 test_node $NAME "" -loglevel 5 $JSFILE
-sleep 0.5
+sleep 0.3
-kill $P13ID
+kill $P12ID
 

eth/test/tests/03.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
 
-TIMEOUT=35
+TIMEOUT=12
 
 cat >> $JSFILE <<EOF
 eth.addPeer("localhost:30311");
-sleep(30000);
+sleep(10000);
 eth.export("$CHAIN_TEST");
 EOF
 

eth/test/tests/common.sh

@@ -3,14 +3,14 @@
 # launched by run.sh
 function test_node {
   rm -rf $DIR/$1
-  ARGS="-datadir $DIR/$1 -debug debug -seed=false -shh=false -id test$1"
+  ARGS="-datadir $DIR/$1 -debug debug -seed=false -shh=false -id test$1 -port 303$1"
   if [ "" != "$2" ]; then
     chain="chains/$2.chain"
     echo "import chain $chain"
     $ETH $ARGS -loglevel 3 -chain $chain | grep CLI |grep import
   fi
-  echo "starting test node $1 with extra args ${@:3}"
+  echo "starting test node $1 with args $ARGS ${@:3}"
-  $ETH $ARGS -port 303$1 ${@:3} &
+  $ETH $ARGS  ${@:3} &
   PID=$!
   PIDS="$PIDS $PID"
 }

ethutil/path.go

@@ -12,7 +12,7 @@ func ExpandHomePath(p string) (path string) {
 	path = p
 
 	// Check in case of paths like "/something/~/something/"
-	if path[:2] == "~/" {
+	if len(path) > 1 && path[:2] == "~/" {
 		usr, _ := user.Current()
 		dir := usr.HomeDir
 

ethutil/rlp.go

@@ -137,7 +137,7 @@ func Encode(object interface{}) []byte {
 		case byte:
 			buff.Write(Encode(big.NewInt(int64(t))))
 		case *big.Int:
-			// Not sure how this is possible while we check for
+			// Not sure how this is possible while we check for nil
 			if t == nil {
 				buff.WriteByte(0xc0)
 			} else {

ptrie/iterator.go

@@ -1,115 +0,0 @@
-package ptrie
-
-import (
-	"bytes"
-
-	"github.com/ethereum/go-ethereum/trie"
-)
-
-type Iterator struct {
-	trie *Trie
-
-	Key   []byte
-	Value []byte
-}
-
-func NewIterator(trie *Trie) *Iterator {
-	return &Iterator{trie: trie, Key: make([]byte, 32)}
-}
-
-func (self *Iterator) Next() bool {
-	self.trie.mu.Lock()
-	defer self.trie.mu.Unlock()
-
-	key := trie.RemTerm(trie.CompactHexDecode(string(self.Key)))
-	k := self.next(self.trie.root, key)
-
-	self.Key = []byte(trie.DecodeCompact(k))
-
-	return len(k) > 0
-
-}
-
-func (self *Iterator) next(node Node, key []byte) []byte {
-	if node == nil {
-		return nil
-	}
-
-	switch node := node.(type) {
-	case *FullNode:
-		if len(key) > 0 {
-			k := self.next(node.branch(key[0]), key[1:])
-			if k != nil {
-				return append([]byte{key[0]}, k...)
-			}
-		}
-
-		var r byte
-		if len(key) > 0 {
-			r = key[0] + 1
-		}
-
-		for i := r; i < 16; i++ {
-			k := self.key(node.branch(byte(i)))
-			if k != nil {
-				return append([]byte{i}, k...)
-			}
-		}
-
-	case *ShortNode:
-		k := trie.RemTerm(node.Key())
-		if vnode, ok := node.Value().(*ValueNode); ok {
-			if bytes.Compare([]byte(k), key) > 0 {
-				self.Value = vnode.Val()
-				return k
-			}
-		} else {
-			cnode := node.Value()
-
-			var ret []byte
-			skey := key[len(k):]
-			if trie.BeginsWith(key, k) {
-				ret = self.next(cnode, skey)
-			} else if bytes.Compare(k, key[:len(k)]) > 0 {
-				ret = self.key(node)
-			}
-
-			if ret != nil {
-				return append(k, ret...)
-			}
-		}
-	}
-
-	return nil
-}
-
-func (self *Iterator) key(node Node) []byte {
-	switch node := node.(type) {
-	case *ShortNode:
-		// Leaf node
-		if vnode, ok := node.Value().(*ValueNode); ok {
-			k := trie.RemTerm(node.Key())
-			self.Value = vnode.Val()
-
-			return k
-		} else {
-			k := trie.RemTerm(node.Key())
-			return append(k, self.key(node.Value())...)
-		}
-	case *FullNode:
-		if node.Value() != nil {
-			self.Value = node.Value().(*ValueNode).Val()
-
-			return []byte{16}
-		}
-
-		for i := 0; i < 16; i++ {
-			k := self.key(node.branch(byte(i)))
-			if k != nil {
-				return append([]byte{byte(i)}, k...)
-			}
-		}
-	}
-
-	return nil
-}

ptrie/trie.go

@@ -1,335 +0,0 @@
-package ptrie
-
-import (
-	"bytes"
-	"container/list"
-	"fmt"
-	"sync"
-
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/ethutil"
-	"github.com/ethereum/go-ethereum/trie"
-)
-
-func ParanoiaCheck(t1 *Trie, backend Backend) (bool, *Trie) {
-	t2 := New(nil, backend)
-
-	it := t1.Iterator()
-	for it.Next() {
-		t2.Update(it.Key, it.Value)
-	}
-
-	return bytes.Equal(t2.Hash(), t1.Hash()), t2
-}
-
-type Trie struct {
-	mu       sync.Mutex
-	root     Node
-	roothash []byte
-	cache    *Cache
-
-	revisions *list.List
-}
-
-func New(root []byte, backend Backend) *Trie {
-	trie := &Trie{}
-	trie.revisions = list.New()
-	trie.roothash = root
-	trie.cache = NewCache(backend)
-
-	if root != nil {
-		value := ethutil.NewValueFromBytes(trie.cache.Get(root))
-		trie.root = trie.mknode(value)
-	}
-
-	return trie
-}
-
-func (self *Trie) Iterator() *Iterator {
-	return NewIterator(self)
-}
-
-func (self *Trie) Copy() *Trie {
-	return New(self.roothash, self.cache.backend)
-}
-
-// Legacy support
-func (self *Trie) Root() []byte { return self.Hash() }
-func (self *Trie) Hash() []byte {
-	var hash []byte
-	if self.root != nil {
-		t := self.root.Hash()
-		if byts, ok := t.([]byte); ok && len(byts) > 0 {
-			hash = byts
-		} else {
-			hash = crypto.Sha3(ethutil.Encode(self.root.RlpData()))
-		}
-	} else {
-		hash = crypto.Sha3(ethutil.Encode(""))
-	}
-
-	if !bytes.Equal(hash, self.roothash) {
-		self.revisions.PushBack(self.roothash)
-		self.roothash = hash
-	}
-
-	return hash
-}
-func (self *Trie) Commit() {
-	self.mu.Lock()
-	defer self.mu.Unlock()
-
-	// Hash first
-	self.Hash()
-
-	self.cache.Flush()
-}
-
-// Reset should only be called if the trie has been hashed
-func (self *Trie) Reset() {
-	self.mu.Lock()
-	defer self.mu.Unlock()
-
-	self.cache.Reset()
-
-	if self.revisions.Len() > 0 {
-		revision := self.revisions.Remove(self.revisions.Back()).([]byte)
-		self.roothash = revision
-	}
-	value := ethutil.NewValueFromBytes(self.cache.Get(self.roothash))
-	self.root = self.mknode(value)
-}
-
-func (self *Trie) UpdateString(key, value string) Node { return self.Update([]byte(key), []byte(value)) }
-func (self *Trie) Update(key, value []byte) Node {
-	self.mu.Lock()
-	defer self.mu.Unlock()
-
-	k := trie.CompactHexDecode(string(key))
-
-	if len(value) != 0 {
-		self.root = self.insert(self.root, k, &ValueNode{self, value})
-	} else {
-		self.root = self.delete(self.root, k)
-	}
-
-	return self.root
-}
-
-func (self *Trie) GetString(key string) []byte { return self.Get([]byte(key)) }
-func (self *Trie) Get(key []byte) []byte {
-	self.mu.Lock()
-	defer self.mu.Unlock()
-
-	k := trie.CompactHexDecode(string(key))
-
-	n := self.get(self.root, k)
-	if n != nil {
-		return n.(*ValueNode).Val()
-	}
-
-	return nil
-}
-
-func (self *Trie) DeleteString(key string) Node { return self.Delete([]byte(key)) }
-func (self *Trie) Delete(key []byte) Node {
-	self.mu.Lock()
-	defer self.mu.Unlock()
-
-	k := trie.CompactHexDecode(string(key))
-	self.root = self.delete(self.root, k)
-
-	return self.root
-}
-
-func (self *Trie) insert(node Node, key []byte, value Node) Node {
-	if len(key) == 0 {
-		return value
-	}
-
-	if node == nil {
-		return NewShortNode(self, key, value)
-	}
-
-	switch node := node.(type) {
-	case *ShortNode:
-		k := node.Key()
-		cnode := node.Value()
-		if bytes.Equal(k, key) {
-			return NewShortNode(self, key, value)
-		}
-
-		var n Node
-		matchlength := trie.MatchingNibbleLength(key, k)
-		if matchlength == len(k) {
-			n = self.insert(cnode, key[matchlength:], value)
-		} else {
-			pnode := self.insert(nil, k[matchlength+1:], cnode)
-			nnode := self.insert(nil, key[matchlength+1:], value)
-			fulln := NewFullNode(self)
-			fulln.set(k[matchlength], pnode)
-			fulln.set(key[matchlength], nnode)
-			n = fulln
-		}
-		if matchlength == 0 {
-			return n
-		}
-
-		return NewShortNode(self, key[:matchlength], n)
-
-	case *FullNode:
-		cpy := node.Copy().(*FullNode)
-		cpy.set(key[0], self.insert(node.branch(key[0]), key[1:], value))
-
-		return cpy
-
-	default:
-		panic(fmt.Sprintf("%T: invalid node: %v", node, node))
-	}
-}
-
-func (self *Trie) get(node Node, key []byte) Node {
-	if len(key) == 0 {
-		return node
-	}
-
-	if node == nil {
-		return nil
-	}
-
-	switch node := node.(type) {
-	case *ShortNode:
-		k := node.Key()
-		cnode := node.Value()
-
-		if len(key) >= len(k) && bytes.Equal(k, key[:len(k)]) {
-			return self.get(cnode, key[len(k):])
-		}
-
-		return nil
-	case *FullNode:
-		return self.get(node.branch(key[0]), key[1:])
-	default:
-		panic(fmt.Sprintf("%T: invalid node: %v", node, node))
-	}
-}
-
-func (self *Trie) delete(node Node, key []byte) Node {
-	if len(key) == 0 && node == nil {
-		return nil
-	}
-
-	switch node := node.(type) {
-	case *ShortNode:
-		k := node.Key()
-		cnode := node.Value()
-		if bytes.Equal(key, k) {
-			return nil
-		} else if bytes.Equal(key[:len(k)], k) {
-			child := self.delete(cnode, key[len(k):])
-
-			var n Node
-			switch child := child.(type) {
-			case *ShortNode:
-				nkey := append(k, child.Key()...)
-				n = NewShortNode(self, nkey, child.Value())
-			case *FullNode:
-				sn := NewShortNode(self, node.Key(), child)
-				sn.key = node.key
-				n = sn
-			}
-
-			return n
-		} else {
-			return node
-		}
-
-	case *FullNode:
-		n := node.Copy().(*FullNode)
-		n.set(key[0], self.delete(n.branch(key[0]), key[1:]))
-
-		pos := -1
-		for i := 0; i < 17; i++ {
-			if n.branch(byte(i)) != nil {
-				if pos == -1 {
-					pos = i
-				} else {
-					pos = -2
-				}
-			}
-		}
-
-		var nnode Node
-		if pos == 16 {
-			nnode = NewShortNode(self, []byte{16}, n.branch(byte(pos)))
-		} else if pos >= 0 {
-			cnode := n.branch(byte(pos))
-			switch cnode := cnode.(type) {
-			case *ShortNode:
-				// Stitch keys
-				k := append([]byte{byte(pos)}, cnode.Key()...)
-				nnode = NewShortNode(self, k, cnode.Value())
-			case *FullNode:
-				nnode = NewShortNode(self, []byte{byte(pos)}, n.branch(byte(pos)))
-			}
-		} else {
-			nnode = n
-		}
-
-		return nnode
-	case nil:
-		return nil
-	default:
-		panic(fmt.Sprintf("%T: invalid node: %v (%v)", node, node, key))
-	}
-}
-
-// casting functions and cache storing
-func (self *Trie) mknode(value *ethutil.Value) Node {
-	l := value.Len()
-	switch l {
-	case 0:
-		return nil
-	case 2:
-		// A value node may consists of 2 bytes.
-		if value.Get(0).Len() != 0 {
-			return NewShortNode(self, trie.CompactDecode(string(value.Get(0).Bytes())), self.mknode(value.Get(1)))
-		}
-	case 17:
-		fnode := NewFullNode(self)
-		for i := 0; i < l; i++ {
-			fnode.set(byte(i), self.mknode(value.Get(i)))
-		}
-		return fnode
-	case 32:
-		return &HashNode{value.Bytes()}
-	}
-
-	return &ValueNode{self, value.Bytes()}
-}
-
-func (self *Trie) trans(node Node) Node {
-	switch node := node.(type) {
-	case *HashNode:
-		value := ethutil.NewValueFromBytes(self.cache.Get(node.key))
-		return self.mknode(value)
-	default:
-		return node
-	}
-}
-
-func (self *Trie) store(node Node) interface{} {
-	data := ethutil.Encode(node)
-	if len(data) >= 32 {
-		key := crypto.Sha3(data)
-		self.cache.Put(key, data)
-
-		return key
-	}
-
-	return node.RlpData()
-}
-
-func (self *Trie) PrintRoot() {
-	fmt.Println(self.root)
-}

ptrie/trie_test.go

@@ -1,259 +0,0 @@
-package ptrie
-
-import (
-	"bytes"
-	"fmt"
-	"testing"
-
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/ethutil"
-)
-
-type Db map[string][]byte
-
-func (self Db) Get(k []byte) ([]byte, error) { return self[string(k)], nil }
-func (self Db) Put(k, v []byte)              { self[string(k)] = v }
-
-// Used for testing
-func NewEmpty() *Trie {
-	return New(nil, make(Db))
-}
-
-func TestEmptyTrie(t *testing.T) {
-	trie := NewEmpty()
-	res := trie.Hash()
-	exp := crypto.Sha3(ethutil.Encode(""))
-	if !bytes.Equal(res, exp) {
-		t.Errorf("expected %x got %x", exp, res)
-	}
-}
-
-func TestInsert(t *testing.T) {
-	trie := NewEmpty()
-
-	trie.UpdateString("doe", "reindeer")
-	trie.UpdateString("dog", "puppy")
-	trie.UpdateString("dogglesworth", "cat")
-
-	exp := ethutil.Hex2Bytes("8aad789dff2f538bca5d8ea56e8abe10f4c7ba3a5dea95fea4cd6e7c3a1168d3")
-	root := trie.Hash()
-	if !bytes.Equal(root, exp) {
-		t.Errorf("exp %x got %x", exp, root)
-	}
-
-	trie = NewEmpty()
-	trie.UpdateString("A", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
-
-	exp = ethutil.Hex2Bytes("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab")
-	root = trie.Hash()
-	if !bytes.Equal(root, exp) {
-		t.Errorf("exp %x got %x", exp, root)
-	}
-}
-
-func TestGet(t *testing.T) {
-	trie := NewEmpty()
-
-	trie.UpdateString("doe", "reindeer")
-	trie.UpdateString("dog", "puppy")
-	trie.UpdateString("dogglesworth", "cat")
-
-	res := trie.GetString("dog")
-	if !bytes.Equal(res, []byte("puppy")) {
-		t.Errorf("expected puppy got %x", res)
-	}
-
-	unknown := trie.GetString("unknown")
-	if unknown != nil {
-		t.Errorf("expected nil got %x", unknown)
-	}
-}
-
-func TestDelete(t *testing.T) {
-	trie := NewEmpty()
-
-	vals := []struct{ k, v string }{
-		{"do", "verb"},
-		{"ether", "wookiedoo"},
-		{"horse", "stallion"},
-		{"shaman", "horse"},
-		{"doge", "coin"},
-		{"ether", ""},
-		{"dog", "puppy"},
-		{"shaman", ""},
-	}
-	for _, val := range vals {
-		if val.v != "" {
-			trie.UpdateString(val.k, val.v)
-		} else {
-			trie.DeleteString(val.k)
-		}
-	}
-
-	hash := trie.Hash()
-	exp := ethutil.Hex2Bytes("5991bb8c6514148a29db676a14ac506cd2cd5775ace63c30a4fe457715e9ac84")
-	if !bytes.Equal(hash, exp) {
-		t.Errorf("expected %x got %x", exp, hash)
-	}
-}
-
-func TestEmptyValues(t *testing.T) {
-	trie := NewEmpty()
-
-	vals := []struct{ k, v string }{
-		{"do", "verb"},
-		{"ether", "wookiedoo"},
-		{"horse", "stallion"},
-		{"shaman", "horse"},
-		{"doge", "coin"},
-		{"ether", ""},
-		{"dog", "puppy"},
-		{"shaman", ""},
-	}
-	for _, val := range vals {
-		trie.UpdateString(val.k, val.v)
-	}
-
-	hash := trie.Hash()
-	exp := ethutil.Hex2Bytes("5991bb8c6514148a29db676a14ac506cd2cd5775ace63c30a4fe457715e9ac84")
-	if !bytes.Equal(hash, exp) {
-		t.Errorf("expected %x got %x", exp, hash)
-	}
-}
-
-func TestReplication(t *testing.T) {
-	trie := NewEmpty()
-	vals := []struct{ k, v string }{
-		{"do", "verb"},
-		{"ether", "wookiedoo"},
-		{"horse", "stallion"},
-		{"shaman", "horse"},
-		{"doge", "coin"},
-		{"ether", ""},
-		{"dog", "puppy"},
-		{"shaman", ""},
-		{"somethingveryoddindeedthis is", "myothernodedata"},
-	}
-	for _, val := range vals {
-		trie.UpdateString(val.k, val.v)
-	}
-	trie.Commit()
-
-	trie2 := New(trie.roothash, trie.cache.backend)
-	if string(trie2.GetString("horse")) != "stallion" {
-		t.Error("expected to have horse => stallion")
-	}
-
-	hash := trie2.Hash()
-	exp := trie.Hash()
-	if !bytes.Equal(hash, exp) {
-		t.Errorf("root failure. expected %x got %x", exp, hash)
-	}
-
-}
-
-func TestReset(t *testing.T) {
-	trie := NewEmpty()
-	vals := []struct{ k, v string }{
-		{"do", "verb"},
-		{"ether", "wookiedoo"},
-		{"horse", "stallion"},
-	}
-	for _, val := range vals {
-		trie.UpdateString(val.k, val.v)
-	}
-	trie.Commit()
-
-	before := ethutil.CopyBytes(trie.roothash)
-	trie.UpdateString("should", "revert")
-	trie.Hash()
-	// Should have no effect
-	trie.Hash()
-	trie.Hash()
-	// ###
-
-	trie.Reset()
-	after := ethutil.CopyBytes(trie.roothash)
-
-	if !bytes.Equal(before, after) {
-		t.Errorf("expected roots to be equal. %x - %x", before, after)
-	}
-}
-
-func TestParanoia(t *testing.T) {
-	t.Skip()
-	trie := NewEmpty()
-
-	vals := []struct{ k, v string }{
-		{"do", "verb"},
-		{"ether", "wookiedoo"},
-		{"horse", "stallion"},
-		{"shaman", "horse"},
-		{"doge", "coin"},
-		{"ether", ""},
-		{"dog", "puppy"},
-		{"shaman", ""},
-		{"somethingveryoddindeedthis is", "myothernodedata"},
-	}
-	for _, val := range vals {
-		trie.UpdateString(val.k, val.v)
-	}
-	trie.Commit()
-
-	ok, t2 := ParanoiaCheck(trie, trie.cache.backend)
-	if !ok {
-		t.Errorf("trie paranoia check failed %x %x", trie.roothash, t2.roothash)
-	}
-}
-
-// Not an actual test
-func TestOutput(t *testing.T) {
-	t.Skip()
-
-	base := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
-	trie := NewEmpty()
-	for i := 0; i < 50; i++ {
-		trie.UpdateString(fmt.Sprintf("%s%d", base, i), "valueeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee")
-	}
-	fmt.Println("############################## FULL ################################")
-	fmt.Println(trie.root)
-
-	trie.Commit()
-	fmt.Println("############################## SMALL ################################")
-	trie2 := New(trie.roothash, trie.cache.backend)
-	trie2.GetString(base + "20")
-	fmt.Println(trie2.root)
-}
-
-func BenchmarkGets(b *testing.B) {
-	trie := NewEmpty()
-	vals := []struct{ k, v string }{
-		{"do", "verb"},
-		{"ether", "wookiedoo"},
-		{"horse", "stallion"},
-		{"shaman", "horse"},
-		{"doge", "coin"},
-		{"ether", ""},
-		{"dog", "puppy"},
-		{"shaman", ""},
-		{"somethingveryoddindeedthis is", "myothernodedata"},
-	}
-	for _, val := range vals {
-		trie.UpdateString(val.k, val.v)
-	}
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		trie.Get([]byte("horse"))
-	}
-}
-
-func BenchmarkUpdate(b *testing.B) {
-	trie := NewEmpty()
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		trie.UpdateString(fmt.Sprintf("aaaaaaaaa%d", i), "value")
-	}
-	trie.Hash()
-}

state/state_object.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethutil"
-	"github.com/ethereum/go-ethereum/ptrie"
+	"github.com/ethereum/go-ethereum/trie"
 )
 
 type Code []byte
@@ -152,7 +152,7 @@ func (self *StateObject) Sync() {
 	}
 
 	/*
-		valid, t2 := ptrie.ParanoiaCheck(self.State.trie, ethutil.Config.Db)
+		valid, t2 := trie.ParanoiaCheck(self.State.trie, ethutil.Config.Db)
 		if !valid {
 			statelogger.Infof("Warn: PARANOIA: Different state storage root during copy %x vs %x\n", self.State.Root(), t2.Root())
 
@@ -273,7 +273,7 @@ func (c *StateObject) Init() Code {
 	return c.InitCode
 }
 
-func (self *StateObject) Trie() *ptrie.Trie {
+func (self *StateObject) Trie() *trie.Trie {
 	return self.State.trie
 }
 

state/statedb.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/ethutil"
 	"github.com/ethereum/go-ethereum/logger"
-	"github.com/ethereum/go-ethereum/ptrie"
+	"github.com/ethereum/go-ethereum/trie"
 )
 
 var statelogger = logger.NewLogger("STATE")
@@ -18,7 +18,7 @@ var statelogger = logger.NewLogger("STATE")
 // * Accounts
 type StateDB struct {
 	db   ethutil.Database
-	trie *ptrie.Trie
+	trie *trie.Trie
 
 	stateObjects map[string]*StateObject
 
@@ -30,9 +30,8 @@ type StateDB struct {
 }
 
 // Create a new state from a given trie
-//func New(trie *ptrie.Trie) *StateDB {
 func New(root []byte, db ethutil.Database) *StateDB {
-	trie := ptrie.New(root, db)
+	trie := trie.New(root, db)
 	return &StateDB{db: db, trie: trie, stateObjects: make(map[string]*StateObject), manifest: NewManifest(), refund: make(map[string]*big.Int)}
 }
 
@@ -308,7 +307,7 @@ func (self *StateDB) Update(gasUsed *big.Int) {
 
 	// FIXME trie delete is broken
 	if deleted {
-		valid, t2 := ptrie.ParanoiaCheck(self.trie, self.db)
+		valid, t2 := trie.ParanoiaCheck(self.trie, self.db)
 		if !valid {
 			statelogger.Infof("Warn: PARANOIA: Different state root during copy %x vs %x\n", self.trie.Root(), t2.Root())
 

tests/helper/trie.go

@@ -1,6 +1,6 @@
 package helper
 
-import "github.com/ethereum/go-ethereum/ptrie"
+import "github.com/ethereum/go-ethereum/trie"
 
 type MemDatabase struct {
 	db map[string][]byte
@@ -24,8 +24,8 @@ func (db *MemDatabase) Print()              {}
 func (db *MemDatabase) Close()              {}
 func (db *MemDatabase) LastKnownTD() []byte { return nil }
 
-func NewTrie() *ptrie.Trie {
+func NewTrie() *trie.Trie {
 	db, _ := NewMemDatabase()
 
-	return ptrie.New(nil, db)
+	return trie.New(nil, db)
 }

trie/cache.go

@@ -1,4 +1,4 @@
-package ptrie
+package trie
 
 type Backend interface {
 	Get([]byte) ([]byte, error)

trie/fullnode.go

@@ -1,4 +1,4 @@
-package ptrie
+package trie
 
 import "fmt"
 

trie/hashnode.go

@@ -1,4 +1,4 @@
-package ptrie
+package trie
 
 type HashNode struct {
 	key []byte

trie/iterator.go

@@ -1,124 +1,73 @@
 package trie
 
-/*
+import "bytes"
-import (
-	"bytes"
-
-	"github.com/ethereum/go-ethereum/ethutil"
-)
-
-type NodeType byte
-
-const (
-	EmptyNode NodeType = iota
-	BranchNode
-	LeafNode
-	ExtNode
-)
-
-func getType(node *ethutil.Value) NodeType {
-	if node.Len() == 0 {
-		return EmptyNode
-	}
-
-	if node.Len() == 2 {
-		k := CompactDecode(node.Get(0).Str())
-		if HasTerm(k) {
-			return LeafNode
-		}
-
-		return ExtNode
-	}
-
-	return BranchNode
-}
 
 type Iterator struct {
-	Path [][]byte
 	trie *Trie
 
 	Key   []byte
-	Value *ethutil.Value
+	Value []byte
 }
 
 func NewIterator(trie *Trie) *Iterator {
-	return &Iterator{trie: trie}
+	return &Iterator{trie: trie, Key: make([]byte, 32)}
 }
 
-func (self *Iterator) key(node *ethutil.Value, path [][]byte) []byte {
+func (self *Iterator) Next() bool {
-	switch getType(node) {
+	self.trie.mu.Lock()
-	case LeafNode:
+	defer self.trie.mu.Unlock()
-		k := RemTerm(CompactDecode(node.Get(0).Str()))
 
-		self.Path = append(path, k)
+	key := RemTerm(CompactHexDecode(string(self.Key)))
-		self.Value = node.Get(1)
+	k := self.next(self.trie.root, key)
 
-		return k
+	self.Key = []byte(DecodeCompact(k))
-	case BranchNode:
-		if node.Get(16).Len() > 0 {
-			return []byte{16}
-		}
 
-		for i := byte(0); i < 16; i++ {
+	return len(k) > 0
-			o := self.key(self.trie.getNode(node.Get(int(i)).Raw()), append(path, []byte{i}))
-			if o != nil {
-				return append([]byte{i}, o...)
-			}
-		}
-	case ExtNode:
-		currKey := node.Get(0).Bytes()
 
-		return self.key(self.trie.getNode(node.Get(1).Raw()), append(path, currKey))
-	}
-
-	return nil
 }
 
-func (self *Iterator) next(node *ethutil.Value, key []byte, path [][]byte) []byte {
+func (self *Iterator) next(node Node, key []byte) []byte {
-	switch typ := getType(node); typ {
+	if node == nil {
-	case EmptyNode:
 		return nil
-	case BranchNode:
+	}
+
+	switch node := node.(type) {
+	case *FullNode:
 		if len(key) > 0 {
-			subNode := self.trie.getNode(node.Get(int(key[0])).Raw())
+			k := self.next(node.branch(key[0]), key[1:])
-
+			if k != nil {
-			o := self.next(subNode, key[1:], append(path, key[:1]))
+				return append([]byte{key[0]}, k...)
-			if o != nil {
-				return append([]byte{key[0]}, o...)
 			}
 		}
 
-		var r byte = 0
+		var r byte
 		if len(key) > 0 {
 			r = key[0] + 1
 		}
 
 		for i := r; i < 16; i++ {
-			subNode := self.trie.getNode(node.Get(int(i)).Raw())
+			k := self.key(node.branch(byte(i)))
-			o := self.key(subNode, append(path, []byte{i}))
+			if k != nil {
-			if o != nil {
+				return append([]byte{i}, k...)
-				return append([]byte{i}, o...)
 			}
 		}
-	case LeafNode, ExtNode:
-		k := RemTerm(CompactDecode(node.Get(0).Str()))
-		if typ == LeafNode {
-			if bytes.Compare([]byte(k), []byte(key)) > 0 {
-				self.Value = node.Get(1)
-				self.Path = append(path, k)
 
+	case *ShortNode:
+		k := RemTerm(node.Key())
+		if vnode, ok := node.Value().(*ValueNode); ok {
+			if bytes.Compare([]byte(k), key) > 0 {
+				self.Value = vnode.Val()
 				return k
 			}
 		} else {
-			subNode := self.trie.getNode(node.Get(1).Raw())
+			cnode := node.Value()
-			subKey := key[len(k):]
+
 			var ret []byte
+			skey := key[len(k):]
 			if BeginsWith(key, k) {
-				ret = self.next(subNode, subKey, append(path, k))
+				ret = self.next(cnode, skey)
 			} else if bytes.Compare(k, key[:len(k)]) > 0 {
-				ret = self.key(node, append(path, k))
+				ret = self.key(node)
-			} else {
-				ret = nil
 			}
 
 			if ret != nil {
@@ -130,16 +79,33 @@ func (self *Iterator) next(node *ethutil.Value, key []byte, path [][]byte) []byt
 	return nil
 }
 
-// Get the next in keys
+func (self *Iterator) key(node Node) []byte {
-func (self *Iterator) Next(key string) []byte {
+	switch node := node.(type) {
-	self.trie.mut.Lock()
+	case *ShortNode:
-	defer self.trie.mut.Unlock()
+		// Leaf node
+		if vnode, ok := node.Value().(*ValueNode); ok {
+			k := RemTerm(node.Key())
+			self.Value = vnode.Val()
 
-	k := RemTerm(CompactHexDecode(key))
+			return k
-	n := self.next(self.trie.getNode(self.trie.Root), k, nil)
+		} else {
+			k := RemTerm(node.Key())
+			return append(k, self.key(node.Value())...)
+		}
+	case *FullNode:
+		if node.Value() != nil {
+			self.Value = node.Value().(*ValueNode).Val()
 
-	self.Key = []byte(DecodeCompact(n))
+			return []byte{16}
+		}
 
-	return self.Key
+		for i := 0; i < 16; i++ {
+			k := self.key(node.branch(byte(i)))
+			if k != nil {
+				return append([]byte{byte(i)}, k...)
+			}
+		}
+	}
+
+	return nil
 }
-*/

trie/iterator_test.go

@@ -1,4 +1,4 @@
-package ptrie
+package trie
 
 import "testing"
 

trie/main_test.go

@@ -1,9 +0,0 @@
-package trie
-
-import (
-	"testing"
-
-	checker "gopkg.in/check.v1"
-)
-
-func Test(t *testing.T) { checker.TestingT(t) }

trie/node.go

@@ -1,4 +1,4 @@
-package ptrie
+package trie
 
 import "fmt"
 

trie/shortnode.go

@@ -1,6 +1,4 @@
-package ptrie
+package trie
-
-import "github.com/ethereum/go-ethereum/trie"
 
 type ShortNode struct {
 	trie  *Trie
@@ -9,7 +7,7 @@ type ShortNode struct {
 }
 
 func NewShortNode(t *Trie, key []byte, value Node) *ShortNode {
-	return &ShortNode{t, []byte(trie.CompactEncode(key)), value}
+	return &ShortNode{t, []byte(CompactEncode(key)), value}
 }
 func (self *ShortNode) Value() Node {
 	self.value = self.trie.trans(self.value)
@@ -27,5 +25,5 @@ func (self *ShortNode) Hash() interface{} {
 }
 
 func (self *ShortNode) Key() []byte {
-	return trie.CompactDecode(string(self.key))
+	return CompactDecode(string(self.key))
 }

trie/trie.go

@@ -1,8 +1,8 @@
 package trie
 
-/*
 import (
 	"bytes"
+	"container/list"
 	"fmt"
 	"sync"
 
@@ -10,618 +10,325 @@ import (
 	"github.com/ethereum/go-ethereum/ethutil"
 )
 
-func ParanoiaCheck(t1 *Trie) (bool, *Trie) {
+func ParanoiaCheck(t1 *Trie, backend Backend) (bool, *Trie) {
-	t2 := New(ethutil.Config.Db, "")
+	t2 := New(nil, backend)
 
-	t1.NewIterator().Each(func(key string, v *ethutil.Value) {
+	it := t1.Iterator()
-		t2.Update(key, v.Str())
+	for it.Next() {
-	})
+		t2.Update(it.Key, it.Value)
-
-	return bytes.Compare(t2.GetRoot(), t1.GetRoot()) == 0, t2
-}
-
-func (s *Cache) Len() int {
-	return len(s.nodes)
-}
-
-// TODO
-// A StateObject is an object that has a state root
-// This is goig to be the object for the second level caching (the caching of object which have a state such as contracts)
-type StateObject interface {
-	State() *Trie
-	Sync()
-	Undo()
-}
-
-type Node struct {
-	Key   []byte
-	Value *ethutil.Value
-	Dirty bool
-}
-
-func NewNode(key []byte, val *ethutil.Value, dirty bool) *Node {
-	return &Node{Key: key, Value: val, Dirty: dirty}
-}
-
-func (n *Node) Copy() *Node {
-	return NewNode(n.Key, n.Value, n.Dirty)
-}
-
-type Cache struct {
-	nodes   map[string]*Node
-	db      ethutil.Database
-	IsDirty bool
-}
-
-func NewCache(db ethutil.Database) *Cache {
-	return &Cache{db: db, nodes: make(map[string]*Node)}
-}
-
-func (cache *Cache) PutValue(v interface{}, force bool) interface{} {
-	value := ethutil.NewValue(v)
-
-	enc := value.Encode()
-	if len(enc) >= 32 || force {
-		sha := crypto.Sha3(enc)
-
-		cache.nodes[string(sha)] = NewNode(sha, value, true)
-		cache.IsDirty = true
-
-		return sha
 	}
 
-	return v
+	return bytes.Equal(t2.Hash(), t1.Hash()), t2
 }
 
-func (cache *Cache) Put(v interface{}) interface{} {
-	return cache.PutValue(v, false)
-}
-
-func (cache *Cache) Get(key []byte) *ethutil.Value {
-	// First check if the key is the cache
-	if cache.nodes[string(key)] != nil {
-		return cache.nodes[string(key)].Value
-	}
-
-	// Get the key of the database instead and cache it
-	data, _ := cache.db.Get(key)
-	// Create the cached value
-	value := ethutil.NewValueFromBytes(data)
-
-	defer func() {
-		if r := recover(); r != nil {
-			fmt.Println("RECOVER GET", cache, cache.nodes)
-			panic("bye")
-		}
-	}()
-	// Create caching node
-	cache.nodes[string(key)] = NewNode(key, value, true)
-
-	return value
-}
-
-func (cache *Cache) Delete(key []byte) {
-	delete(cache.nodes, string(key))
-
-	cache.db.Delete(key)
-}
-
-func (cache *Cache) Commit() {
-	// Don't try to commit if it isn't dirty
-	if !cache.IsDirty {
-		return
-	}
-
-	for key, node := range cache.nodes {
-		if node.Dirty {
-			cache.db.Put([]byte(key), node.Value.Encode())
-			node.Dirty = false
-		}
-	}
-	cache.IsDirty = false
-
-	// If the nodes grows beyond the 200 entries we simple empty it
-	// FIXME come up with something better
-	if len(cache.nodes) > 200 {
-		cache.nodes = make(map[string]*Node)
-	}
-}
-
-func (cache *Cache) Undo() {
-	for key, node := range cache.nodes {
-		if node.Dirty {
-			delete(cache.nodes, key)
-		}
-	}
-	cache.IsDirty = false
-}
-
-// A (modified) Radix Trie implementation. The Trie implements
-// a caching mechanism and will used cached values if they are
-// present. If a node is not present in the cache it will try to
-// fetch it from the database and store the cached value.
-// Please note that the data isn't persisted unless `Sync` is
-// explicitly called.
 type Trie struct {
-	mut      sync.RWMutex
+	mu       sync.Mutex
-	prevRoot interface{}
+	root     Node
-	Root     interface{}
+	roothash []byte
-	//db   Database
 	cache    *Cache
+
+	revisions *list.List
 }
 
-func copyRoot(root interface{}) interface{} {
+func New(root []byte, backend Backend) *Trie {
-	var prevRootCopy interface{}
+	trie := &Trie{}
-	if b, ok := root.([]byte); ok {
+	trie.revisions = list.New()
-		prevRootCopy = ethutil.CopyBytes(b)
+	trie.roothash = root
-	} else {
+	trie.cache = NewCache(backend)
-		prevRootCopy = root
-	}
 
-	return prevRootCopy
+	if root != nil {
-}
+		value := ethutil.NewValueFromBytes(trie.cache.Get(root))
-
+		trie.root = trie.mknode(value)
-func New(db ethutil.Database, Root interface{}) *Trie {
-	// Make absolute sure the root is copied
-	r := copyRoot(Root)
-	p := copyRoot(Root)
-
-	trie := &Trie{cache: NewCache(db), Root: r, prevRoot: p}
-	trie.setRoot(Root)
-
-	return trie
-}
-
-func (self *Trie) setRoot(root interface{}) {
-	switch t := root.(type) {
-	case string:
-		//if t == "" {
-		//	root = crypto.Sha3(ethutil.Encode(""))
-		//}
-		self.Root = []byte(t)
-	case []byte:
-		self.Root = root
-	default:
-		self.Root = self.cache.PutValue(root, true)
-	}
-}
-
-func (t *Trie) Update(key, value string) {
-	t.mut.Lock()
-	defer t.mut.Unlock()
-
-	k := CompactHexDecode(key)
-
-	var root interface{}
-	if value != "" {
-		root = t.UpdateState(t.Root, k, value)
-	} else {
-		root = t.deleteState(t.Root, k)
-	}
-	t.setRoot(root)
-}
-
-func (t *Trie) Get(key string) string {
-	t.mut.Lock()
-	defer t.mut.Unlock()
-
-	k := CompactHexDecode(key)
-	c := ethutil.NewValue(t.getState(t.Root, k))
-
-	return c.Str()
-}
-
-func (t *Trie) Delete(key string) {
-	t.mut.Lock()
-	defer t.mut.Unlock()
-
-	k := CompactHexDecode(key)
-
-	root := t.deleteState(t.Root, k)
-	t.setRoot(root)
-}
-
-func (self *Trie) GetRoot() []byte {
-	switch t := self.Root.(type) {
-	case string:
-		if t == "" {
-			return crypto.Sha3(ethutil.Encode(""))
-		}
-		return []byte(t)
-	case []byte:
-		if len(t) == 0 {
-			return crypto.Sha3(ethutil.Encode(""))
-		}
-
-		return t
-	default:
-		panic(fmt.Sprintf("invalid root type %T (%v)", self.Root, self.Root))
-	}
-}
-
-// Simple compare function which creates a rlp value out of the evaluated objects
-func (t *Trie) Cmp(trie *Trie) bool {
-	return ethutil.NewValue(t.Root).Cmp(ethutil.NewValue(trie.Root))
-}
-
-// Returns a copy of this trie
-func (t *Trie) Copy() *Trie {
-	trie := New(t.cache.db, t.Root)
-	for key, node := range t.cache.nodes {
-		trie.cache.nodes[key] = node.Copy()
 	}
 
 	return trie
 }
 
-// Save the cached value to the database.
-func (t *Trie) Sync() {
-	t.cache.Commit()
-	t.prevRoot = copyRoot(t.Root)
-}
-
-func (t *Trie) Undo() {
-	t.cache.Undo()
-	t.Root = t.prevRoot
-}
-
-func (t *Trie) Cache() *Cache {
-	return t.cache
-}
-
-func (t *Trie) getState(node interface{}, key []byte) interface{} {
-	n := ethutil.NewValue(node)
-	// Return the node if key is empty (= found)
-	if len(key) == 0 || n.IsNil() || n.Len() == 0 {
-		return node
-	}
-
-	currentNode := t.getNode(node)
-	length := currentNode.Len()
-
-	if length == 0 {
-		return ""
-	} else if length == 2 {
-		// Decode the key
-		k := CompactDecode(currentNode.Get(0).Str())
-		v := currentNode.Get(1).Raw()
-
-		if len(key) >= len(k) && bytes.Equal(k, key[:len(k)]) { //CompareIntSlice(k, key[:len(k)]) {
-			return t.getState(v, key[len(k):])
-		} else {
-			return ""
-		}
-	} else if length == 17 {
-		return t.getState(currentNode.Get(int(key[0])).Raw(), key[1:])
-	}
-
-	// It shouldn't come this far
-	panic("unexpected return")
-}
-
-func (t *Trie) getNode(node interface{}) *ethutil.Value {
-	n := ethutil.NewValue(node)
-
-	if !n.Get(0).IsNil() {
-		return n
-	}
-
-	str := n.Str()
-	if len(str) == 0 {
-		return n
-	} else if len(str) < 32 {
-		return ethutil.NewValueFromBytes([]byte(str))
-	}
-
-	data := t.cache.Get(n.Bytes())
-
-	return data
-}
-
-func (t *Trie) UpdateState(node interface{}, key []byte, value string) interface{} {
-	return t.InsertState(node, key, value)
-}
-
-func (t *Trie) Put(node interface{}) interface{} {
-	return t.cache.Put(node)
-
-}
-
-func EmptyStringSlice(l int) []interface{} {
-	slice := make([]interface{}, l)
-	for i := 0; i < l; i++ {
-		slice[i] = ""
-	}
-	return slice
-}
-
-func (t *Trie) InsertState(node interface{}, key []byte, value interface{}) interface{} {
-	if len(key) == 0 {
-		return value
-	}
-
-	// New node
-	n := ethutil.NewValue(node)
-	if node == nil || n.Len() == 0 {
-		newNode := []interface{}{CompactEncode(key), value}
-
-		return t.Put(newNode)
-	}
-
-	currentNode := t.getNode(node)
-	// Check for "special" 2 slice type node
-	if currentNode.Len() == 2 {
-		// Decode the key
-
-		k := CompactDecode(currentNode.Get(0).Str())
-		v := currentNode.Get(1).Raw()
-
-		// Matching key pair (ie. there's already an object with this key)
-		if bytes.Equal(k, key) { //CompareIntSlice(k, key) {
-			newNode := []interface{}{CompactEncode(key), value}
-			return t.Put(newNode)
-		}
-
-		var newHash interface{}
-		matchingLength := MatchingNibbleLength(key, k)
-		if matchingLength == len(k) {
-			// Insert the hash, creating a new node
-			newHash = t.InsertState(v, key[matchingLength:], value)
-		} else {
-			// Expand the 2 length slice to a 17 length slice
-			oldNode := t.InsertState("", k[matchingLength+1:], v)
-			newNode := t.InsertState("", key[matchingLength+1:], value)
-			// Create an expanded slice
-			scaledSlice := EmptyStringSlice(17)
-			// Set the copied and new node
-			scaledSlice[k[matchingLength]] = oldNode
-			scaledSlice[key[matchingLength]] = newNode
-
-			newHash = t.Put(scaledSlice)
-		}
-
-		if matchingLength == 0 {
-			// End of the chain, return
-			return newHash
-		} else {
-			newNode := []interface{}{CompactEncode(key[:matchingLength]), newHash}
-			return t.Put(newNode)
-		}
-	} else {
-
-		// Copy the current node over to the new node and replace the first nibble in the key
-		newNode := EmptyStringSlice(17)
-
-		for i := 0; i < 17; i++ {
-			cpy := currentNode.Get(i).Raw()
-			if cpy != nil {
-				newNode[i] = cpy
-			}
-		}
-
-		newNode[key[0]] = t.InsertState(currentNode.Get(int(key[0])).Raw(), key[1:], value)
-
-		return t.Put(newNode)
-	}
-
-	panic("unexpected end")
-}
-
-func (t *Trie) deleteState(node interface{}, key []byte) interface{} {
-	if len(key) == 0 {
-		return ""
-	}
-
-	// New node
-	n := ethutil.NewValue(node)
-	//if node == nil || (n.Type() == reflect.String && (n.Str() == "" || n.Get(0).IsNil())) || n.Len() == 0 {
-	if node == nil || n.Len() == 0 {
-		//return nil
-		//fmt.Printf("<empty ret> %x %d\n", n, len(n.Bytes()))
-
-		return ""
-	}
-
-	currentNode := t.getNode(node)
-	// Check for "special" 2 slice type node
-	if currentNode.Len() == 2 {
-		// Decode the key
-		k := CompactDecode(currentNode.Get(0).Str())
-		v := currentNode.Get(1).Raw()
-
-		// Matching key pair (ie. there's already an object with this key)
-		if bytes.Equal(k, key) { //CompareIntSlice(k, key) {
-			//fmt.Printf("<delete ret> %x\n", v)
-
-			return ""
-		} else if bytes.Equal(key[:len(k)], k) { //CompareIntSlice(key[:len(k)], k) {
-			hash := t.deleteState(v, key[len(k):])
-			child := t.getNode(hash)
-
-			var newNode []interface{}
-			if child.Len() == 2 {
-				newKey := append(k, CompactDecode(child.Get(0).Str())...)
-				newNode = []interface{}{CompactEncode(newKey), child.Get(1).Raw()}
-			} else {
-				newNode = []interface{}{currentNode.Get(0).Str(), hash}
-			}
-
-			//fmt.Printf("%x\n", newNode)
-
-			return t.Put(newNode)
-		} else {
-			return node
-		}
-	} else {
-		// Copy the current node over to the new node and replace the first nibble in the key
-		n := EmptyStringSlice(17)
-		var newNode []interface{}
-
-		for i := 0; i < 17; i++ {
-			cpy := currentNode.Get(i).Raw()
-			if cpy != nil {
-				n[i] = cpy
-			}
-		}
-
-		n[key[0]] = t.deleteState(n[key[0]], key[1:])
-		amount := -1
-		for i := 0; i < 17; i++ {
-			if n[i] != "" {
-				if amount == -1 {
-					amount = i
-				} else {
-					amount = -2
-				}
-			}
-		}
-		if amount == 16 {
-			newNode = []interface{}{CompactEncode([]byte{16}), n[amount]}
-		} else if amount >= 0 {
-			child := t.getNode(n[amount])
-			if child.Len() == 17 {
-				newNode = []interface{}{CompactEncode([]byte{byte(amount)}), n[amount]}
-			} else if child.Len() == 2 {
-				key := append([]byte{byte(amount)}, CompactDecode(child.Get(0).Str())...)
-				newNode = []interface{}{CompactEncode(key), child.Get(1).Str()}
-			}
-
-		} else {
-			newNode = n
-		}
-
-		//fmt.Printf("%x\n", newNode)
-		return t.Put(newNode)
-	}
-
-	panic("unexpected return")
-}
-
-type TrieIterator struct {
-	trie  *Trie
-	key   string
-	value string
-
-	shas   [][]byte
-	values []string
-
-	lastNode []byte
-}
-
-func (t *Trie) NewIterator() *TrieIterator {
-	return &TrieIterator{trie: t}
-}
-
 func (self *Trie) Iterator() *Iterator {
 	return NewIterator(self)
 }
 
-// Some time in the near future this will need refactoring :-)
+func (self *Trie) Copy() *Trie {
-// XXX Note to self, IsSlice == inline node. Str == sha3 to node
+	return New(self.roothash, self.cache.backend)
-func (it *TrieIterator) workNode(currentNode *ethutil.Value) {
+}
-	if currentNode.Len() == 2 {
-		k := CompactDecode(currentNode.Get(0).Str())
 
-		if currentNode.Get(1).Str() == "" {
+// Legacy support
-			it.workNode(currentNode.Get(1))
+func (self *Trie) Root() []byte { return self.Hash() }
+func (self *Trie) Hash() []byte {
+	var hash []byte
+	if self.root != nil {
+		t := self.root.Hash()
+		if byts, ok := t.([]byte); ok && len(byts) > 0 {
+			hash = byts
 		} else {
-			if k[len(k)-1] == 16 {
+			hash = crypto.Sha3(ethutil.Encode(self.root.RlpData()))
-				it.values = append(it.values, currentNode.Get(1).Str())
-			} else {
-				it.shas = append(it.shas, currentNode.Get(1).Bytes())
-				it.getNode(currentNode.Get(1).Bytes())
-			}
 		}
 	} else {
-		for i := 0; i < currentNode.Len(); i++ {
+		hash = crypto.Sha3(ethutil.Encode(""))
-			if i == 16 && currentNode.Get(i).Len() != 0 {
+	}
-				it.values = append(it.values, currentNode.Get(i).Str())
+
+	if !bytes.Equal(hash, self.roothash) {
+		self.revisions.PushBack(self.roothash)
+		self.roothash = hash
+	}
+
+	return hash
+}
+func (self *Trie) Commit() {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+
+	// Hash first
+	self.Hash()
+
+	self.cache.Flush()
+}
+
+// Reset should only be called if the trie has been hashed
+func (self *Trie) Reset() {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+
+	self.cache.Reset()
+
+	if self.revisions.Len() > 0 {
+		revision := self.revisions.Remove(self.revisions.Back()).([]byte)
+		self.roothash = revision
+	}
+	value := ethutil.NewValueFromBytes(self.cache.Get(self.roothash))
+	self.root = self.mknode(value)
+}
+
+func (self *Trie) UpdateString(key, value string) Node { return self.Update([]byte(key), []byte(value)) }
+func (self *Trie) Update(key, value []byte) Node {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+
+	k := CompactHexDecode(string(key))
+
+	if len(value) != 0 {
+		self.root = self.insert(self.root, k, &ValueNode{self, value})
 	} else {
-				if currentNode.Get(i).Str() == "" {
+		self.root = self.delete(self.root, k)
-					it.workNode(currentNode.Get(i))
+	}
+
+	return self.root
+}
+
+func (self *Trie) GetString(key string) []byte { return self.Get([]byte(key)) }
+func (self *Trie) Get(key []byte) []byte {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+
+	k := CompactHexDecode(string(key))
+
+	n := self.get(self.root, k)
+	if n != nil {
+		return n.(*ValueNode).Val()
+	}
+
+	return nil
+}
+
+func (self *Trie) DeleteString(key string) Node { return self.Delete([]byte(key)) }
+func (self *Trie) Delete(key []byte) Node {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+
+	k := CompactHexDecode(string(key))
+	self.root = self.delete(self.root, k)
+
+	return self.root
+}
+
+func (self *Trie) insert(node Node, key []byte, value Node) Node {
+	if len(key) == 0 {
+		return value
+	}
+
+	if node == nil {
+		return NewShortNode(self, key, value)
+	}
+
+	switch node := node.(type) {
+	case *ShortNode:
+		k := node.Key()
+		cnode := node.Value()
+		if bytes.Equal(k, key) {
+			return NewShortNode(self, key, value)
+		}
+
+		var n Node
+		matchlength := MatchingNibbleLength(key, k)
+		if matchlength == len(k) {
+			n = self.insert(cnode, key[matchlength:], value)
 		} else {
-					val := currentNode.Get(i).Str()
+			pnode := self.insert(nil, k[matchlength+1:], cnode)
-					if val != "" {
+			nnode := self.insert(nil, key[matchlength+1:], value)
-						it.shas = append(it.shas, currentNode.Get(1).Bytes())
+			fulln := NewFullNode(self)
-						it.getNode([]byte(val))
+			fulln.set(k[matchlength], pnode)
-					}
+			fulln.set(key[matchlength], nnode)
-				}
+			n = fulln
 		}
+		if matchlength == 0 {
+			return n
 		}
+
+		return NewShortNode(self, key[:matchlength], n)
+
+	case *FullNode:
+		cpy := node.Copy().(*FullNode)
+		cpy.set(key[0], self.insert(node.branch(key[0]), key[1:], value))
+
+		return cpy
+
+	default:
+		panic(fmt.Sprintf("%T: invalid node: %v", node, node))
 	}
 }
 
-func (it *TrieIterator) getNode(node []byte) {
+func (self *Trie) get(node Node, key []byte) Node {
-	currentNode := it.trie.cache.Get(node)
+	if len(key) == 0 {
-	it.workNode(currentNode)
+		return node
-}
+	}
 
-func (it *TrieIterator) Collect() [][]byte {
+	if node == nil {
-	if it.trie.Root == "" {
 		return nil
 	}
 
-	it.getNode(ethutil.NewValue(it.trie.Root).Bytes())
+	switch node := node.(type) {
+	case *ShortNode:
+		k := node.Key()
+		cnode := node.Value()
 
-	return it.shas
+		if len(key) >= len(k) && bytes.Equal(k, key[:len(k)]) {
-}
+			return self.get(cnode, key[len(k):])
-
-func (it *TrieIterator) Purge() int {
-	shas := it.Collect()
-	for _, sha := range shas {
-		it.trie.cache.Delete(sha)
 		}
-	return len(it.values)
-}
 
-func (it *TrieIterator) Key() string {
+		return nil
-	return ""
+	case *FullNode:
-}
+		return self.get(node.branch(key[0]), key[1:])
-
+	default:
-func (it *TrieIterator) Value() string {
+		panic(fmt.Sprintf("%T: invalid node: %v", node, node))
-	return ""
-}
-
-type EachCallback func(key string, node *ethutil.Value)
-
-func (it *TrieIterator) Each(cb EachCallback) {
-	it.fetchNode(nil, ethutil.NewValue(it.trie.Root).Bytes(), cb)
-}
-
-func (it *TrieIterator) fetchNode(key []byte, node []byte, cb EachCallback) {
-	it.iterateNode(key, it.trie.cache.Get(node), cb)
-}
-
-func (it *TrieIterator) iterateNode(key []byte, currentNode *ethutil.Value, cb EachCallback) {
-	if currentNode.Len() == 2 {
-		k := CompactDecode(currentNode.Get(0).Str())
-
-		pk := append(key, k...)
-		if currentNode.Get(1).Len() != 0 && currentNode.Get(1).Str() == "" {
-			it.iterateNode(pk, currentNode.Get(1), cb)
-		} else {
-			if k[len(k)-1] == 16 {
-				cb(DecodeCompact(pk), currentNode.Get(1))
-			} else {
-				it.fetchNode(pk, currentNode.Get(1).Bytes(), cb)
 	}
+}
+
+func (self *Trie) delete(node Node, key []byte) Node {
+	if len(key) == 0 && node == nil {
+		return nil
+	}
+
+	switch node := node.(type) {
+	case *ShortNode:
+		k := node.Key()
+		cnode := node.Value()
+		if bytes.Equal(key, k) {
+			return nil
+		} else if bytes.Equal(key[:len(k)], k) {
+			child := self.delete(cnode, key[len(k):])
+
+			var n Node
+			switch child := child.(type) {
+			case *ShortNode:
+				nkey := append(k, child.Key()...)
+				n = NewShortNode(self, nkey, child.Value())
+			case *FullNode:
+				sn := NewShortNode(self, node.Key(), child)
+				sn.key = node.key
+				n = sn
+			}
+
+			return n
+		} else {
+			return node
+		}
+
+	case *FullNode:
+		n := node.Copy().(*FullNode)
+		n.set(key[0], self.delete(n.branch(key[0]), key[1:]))
+
+		pos := -1
+		for i := 0; i < 17; i++ {
+			if n.branch(byte(i)) != nil {
+				if pos == -1 {
+					pos = i
+				} else {
+					pos = -2
+				}
+			}
+		}
+
+		var nnode Node
+		if pos == 16 {
+			nnode = NewShortNode(self, []byte{16}, n.branch(byte(pos)))
+		} else if pos >= 0 {
+			cnode := n.branch(byte(pos))
+			switch cnode := cnode.(type) {
+			case *ShortNode:
+				// Stitch keys
+				k := append([]byte{byte(pos)}, cnode.Key()...)
+				nnode = NewShortNode(self, k, cnode.Value())
+			case *FullNode:
+				nnode = NewShortNode(self, []byte{byte(pos)}, n.branch(byte(pos)))
 			}
 		} else {
-		for i := 0; i < currentNode.Len(); i++ {
+			nnode = n
-			pk := append(key, byte(i))
-			if i == 16 && currentNode.Get(i).Len() != 0 {
-				cb(DecodeCompact(pk), currentNode.Get(i))
-			} else {
-				if currentNode.Get(i).Len() != 0 && currentNode.Get(i).Str() == "" {
-					it.iterateNode(pk, currentNode.Get(i), cb)
-				} else {
-					val := currentNode.Get(i).Str()
-					if val != "" {
-						it.fetchNode(pk, []byte(val), cb)
-					}
-				}
-			}
 		}
+
+		return nnode
+	case nil:
+		return nil
+	default:
+		panic(fmt.Sprintf("%T: invalid node: %v (%v)", node, node, key))
 	}
 }
-*/
+
+// casting functions and cache storing
+func (self *Trie) mknode(value *ethutil.Value) Node {
+	l := value.Len()
+	switch l {
+	case 0:
+		return nil
+	case 2:
+		// A value node may consists of 2 bytes.
+		if value.Get(0).Len() != 0 {
+			return NewShortNode(self, CompactDecode(string(value.Get(0).Bytes())), self.mknode(value.Get(1)))
+		}
+	case 17:
+		fnode := NewFullNode(self)
+		for i := 0; i < l; i++ {
+			fnode.set(byte(i), self.mknode(value.Get(i)))
+		}
+		return fnode
+	case 32:
+		return &HashNode{value.Bytes()}
+	}
+
+	return &ValueNode{self, value.Bytes()}
+}
+
+func (self *Trie) trans(node Node) Node {
+	switch node := node.(type) {
+	case *HashNode:
+		value := ethutil.NewValueFromBytes(self.cache.Get(node.key))
+		return self.mknode(value)
+	default:
+		return node
+	}
+}
+
+func (self *Trie) store(node Node) interface{} {
+	data := ethutil.Encode(node)
+	if len(data) >= 32 {
+		key := crypto.Sha3(data)
+		self.cache.Put(key, data)
+
+		return key
+	}
+
+	return node.RlpData()
+}
+
+func (self *Trie) PrintRoot() {
+	fmt.Println(self.root)
+}

trie/trie_test.go

@@ -1,345 +1,76 @@
 package trie
 
-/*
 import (
 	"bytes"
-	"encoding/hex"
-	"encoding/json"
 	"fmt"
-	"io/ioutil"
-	"math/rand"
-	"net/http"
 	"testing"
-	"time"
-
-	checker "gopkg.in/check.v1"
 
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethutil"
 )
 
-const LONG_WORD = "1234567890abcdefghijklmnopqrstuvwxxzABCEFGHIJKLMNOPQRSTUVWXYZ"
+type Db map[string][]byte
 
-type TrieSuite struct {
+func (self Db) Get(k []byte) ([]byte, error) { return self[string(k)], nil }
-	db   *MemDatabase
+func (self Db) Put(k, v []byte)              { self[string(k)] = v }
-	trie *Trie
+
+// Used for testing
+func NewEmpty() *Trie {
+	return New(nil, make(Db))
 }
 
-type MemDatabase struct {
+func TestEmptyTrie(t *testing.T) {
-	db map[string][]byte
+	trie := NewEmpty()
-}
+	res := trie.Hash()
-
+	exp := crypto.Sha3(ethutil.Encode(""))
-func NewMemDatabase() (*MemDatabase, error) {
+	if !bytes.Equal(res, exp) {
-	db := &MemDatabase{db: make(map[string][]byte)}
+		t.Errorf("expected %x got %x", exp, res)
-	return db, nil
-}
-func (db *MemDatabase) Put(key []byte, value []byte) {
-	db.db[string(key)] = value
-}
-func (db *MemDatabase) Get(key []byte) ([]byte, error) {
-	return db.db[string(key)], nil
-}
-func (db *MemDatabase) Delete(key []byte) error {
-	delete(db.db, string(key))
-	return nil
-}
-func (db *MemDatabase) Print()              {}
-func (db *MemDatabase) Close()              {}
-func (db *MemDatabase) LastKnownTD() []byte { return nil }
-
-func NewTrie() (*MemDatabase, *Trie) {
-	db, _ := NewMemDatabase()
-	return db, New(db, "")
-}
-
-func (s *TrieSuite) SetUpTest(c *checker.C) {
-	s.db, s.trie = NewTrie()
-}
-
-func (s *TrieSuite) TestTrieSync(c *checker.C) {
-	s.trie.Update("dog", LONG_WORD)
-	c.Assert(s.db.db, checker.HasLen, 0, checker.Commentf("Expected no data in database"))
-	s.trie.Sync()
-	c.Assert(s.db.db, checker.HasLen, 3)
-}
-
-func (s *TrieSuite) TestTrieDirtyTracking(c *checker.C) {
-	s.trie.Update("dog", LONG_WORD)
-	c.Assert(s.trie.cache.IsDirty, checker.Equals, true, checker.Commentf("Expected no data in database"))
-
-	s.trie.Sync()
-	c.Assert(s.trie.cache.IsDirty, checker.Equals, false, checker.Commentf("Expected trie to be dirty"))
-
-	s.trie.Update("test", LONG_WORD)
-	s.trie.cache.Undo()
-	c.Assert(s.trie.cache.IsDirty, checker.Equals, false)
-}
-
-func (s *TrieSuite) TestTrieReset(c *checker.C) {
-	s.trie.Update("cat", LONG_WORD)
-	c.Assert(s.trie.cache.nodes, checker.HasLen, 1, checker.Commentf("Expected cached nodes"))
-
-	s.trie.cache.Undo()
-	c.Assert(s.trie.cache.nodes, checker.HasLen, 0, checker.Commentf("Expected no nodes after undo"))
-}
-
-func (s *TrieSuite) TestTrieGet(c *checker.C) {
-	s.trie.Update("cat", LONG_WORD)
-	x := s.trie.Get("cat")
-	c.Assert(x, checker.DeepEquals, LONG_WORD)
-}
-
-func (s *TrieSuite) TestTrieUpdating(c *checker.C) {
-	s.trie.Update("cat", LONG_WORD)
-	s.trie.Update("cat", LONG_WORD+"1")
-	x := s.trie.Get("cat")
-	c.Assert(x, checker.DeepEquals, LONG_WORD+"1")
-}
-
-func (s *TrieSuite) TestTrieCmp(c *checker.C) {
-	_, trie1 := NewTrie()
-	_, trie2 := NewTrie()
-
-	trie1.Update("doge", LONG_WORD)
-	trie2.Update("doge", LONG_WORD)
-	c.Assert(trie1, checker.DeepEquals, trie2)
-
-	trie1.Update("dog", LONG_WORD)
-	trie2.Update("cat", LONG_WORD)
-	c.Assert(trie1, checker.Not(checker.DeepEquals), trie2)
-}
-
-func (s *TrieSuite) TestTrieDelete(c *checker.C) {
-	s.trie.Update("cat", LONG_WORD)
-	exp := s.trie.Root
-	s.trie.Update("dog", LONG_WORD)
-	s.trie.Delete("dog")
-	c.Assert(s.trie.Root, checker.DeepEquals, exp)
-
-	s.trie.Update("dog", LONG_WORD)
-	exp = s.trie.Root
-	s.trie.Update("dude", LONG_WORD)
-	s.trie.Delete("dude")
-	c.Assert(s.trie.Root, checker.DeepEquals, exp)
-}
-
-func (s *TrieSuite) TestTrieDeleteWithValue(c *checker.C) {
-	s.trie.Update("c", LONG_WORD)
-	exp := s.trie.Root
-	s.trie.Update("ca", LONG_WORD)
-	s.trie.Update("cat", LONG_WORD)
-	s.trie.Delete("ca")
-	s.trie.Delete("cat")
-	c.Assert(s.trie.Root, checker.DeepEquals, exp)
-}
-
-func (s *TrieSuite) TestTriePurge(c *checker.C) {
-	s.trie.Update("c", LONG_WORD)
-	s.trie.Update("ca", LONG_WORD)
-	s.trie.Update("cat", LONG_WORD)
-
-	lenBefore := len(s.trie.cache.nodes)
-	it := s.trie.NewIterator()
-	num := it.Purge()
-	c.Assert(num, checker.Equals, 3)
-	c.Assert(len(s.trie.cache.nodes), checker.Equals, lenBefore)
-}
-
-func h(str string) string {
-	d, err := hex.DecodeString(str)
-	if err != nil {
-		panic(err)
-	}
-
-	return string(d)
-}
-
-func get(in string) (out string) {
-	if len(in) > 2 && in[:2] == "0x" {
-		out = h(in[2:])
-	} else {
-		out = in
-	}
-
-	return
-}
-
-type TrieTest struct {
-	Name string
-	In   map[string]string
-	Root string
-}
-
-func CreateTest(name string, data []byte) (TrieTest, error) {
-	t := TrieTest{Name: name}
-	err := json.Unmarshal(data, &t)
-	if err != nil {
-		return TrieTest{}, fmt.Errorf("%v", err)
-	}
-
-	return t, nil
-}
-
-func CreateTests(uri string, cb func(TrieTest)) map[string]TrieTest {
-	resp, err := http.Get(uri)
-	if err != nil {
-		panic(err)
-	}
-	defer resp.Body.Close()
-
-	data, err := ioutil.ReadAll(resp.Body)
-
-	var objmap map[string]*json.RawMessage
-	err = json.Unmarshal(data, &objmap)
-	if err != nil {
-		panic(err)
-	}
-
-	tests := make(map[string]TrieTest)
-	for name, testData := range objmap {
-		test, err := CreateTest(name, *testData)
-		if err != nil {
-			panic(err)
-		}
-
-		if cb != nil {
-			cb(test)
-		}
-		tests[name] = test
-	}
-
-	return tests
-}
-
-func RandomData() [][]string {
-	data := [][]string{
-		{"0x000000000000000000000000ec4f34c97e43fbb2816cfd95e388353c7181dab1", "0x4e616d6552656700000000000000000000000000000000000000000000000000"},
-		{"0x0000000000000000000000000000000000000000000000000000000000000045", "0x22b224a1420a802ab51d326e29fa98e34c4f24ea"},
-		{"0x0000000000000000000000000000000000000000000000000000000000000046", "0x67706c2076330000000000000000000000000000000000000000000000000000"},
-		{"0x000000000000000000000000697c7b8c961b56f675d570498424ac8de1a918f6", "0x6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000"},
-		{"0x0000000000000000000000007ef9e639e2733cb34e4dfc576d4b23f72db776b2", "0x4655474156000000000000000000000000000000000000000000000000000000"},
-		{"0x6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000", "0x697c7b8c961b56f675d570498424ac8de1a918f6"},
-		{"0x4655474156000000000000000000000000000000000000000000000000000000", "0x7ef9e639e2733cb34e4dfc576d4b23f72db776b2"},
-		{"0x4e616d6552656700000000000000000000000000000000000000000000000000", "0xec4f34c97e43fbb2816cfd95e388353c7181dab1"},
-	}
-
-	var c [][]string
-	for len(data) != 0 {
-		e := rand.Intn(len(data))
-		c = append(c, data[e])
-
-		copy(data[e:], data[e+1:])
-		data[len(data)-1] = nil
-		data = data[:len(data)-1]
-	}
-
-	return c
-}
-
-const MaxTest = 1000
-
-// This test insert data in random order and seeks to find indifferences between the different tries
-func (s *TrieSuite) TestRegression(c *checker.C) {
-	rand.Seed(time.Now().Unix())
-
-	roots := make(map[string]int)
-	for i := 0; i < MaxTest; i++ {
-		_, trie := NewTrie()
-		data := RandomData()
-
-		for _, test := range data {
-			trie.Update(test[0], test[1])
-		}
-		trie.Delete("0x4e616d6552656700000000000000000000000000000000000000000000000000")
-
-		roots[string(trie.Root.([]byte))] += 1
-	}
-
-	c.Assert(len(roots) <= 1, checker.Equals, true)
-	// if len(roots) > 1 {
-	// 	for root, num := range roots {
-	// 		t.Errorf("%x => %d\n", root, num)
-	// 	}
-	// }
-}
-
-func (s *TrieSuite) TestDelete(c *checker.C) {
-	s.trie.Update("a", "jeffreytestlongstring")
-	s.trie.Update("aa", "otherstring")
-	s.trie.Update("aaa", "othermorestring")
-	s.trie.Update("aabbbbccc", "hithere")
-	s.trie.Update("abbcccdd", "hstanoehutnaheoustnh")
-	s.trie.Update("rnthaoeuabbcccdd", "hstanoehutnaheoustnh")
-	s.trie.Update("rneuabbcccdd", "hstanoehutnaheoustnh")
-	s.trie.Update("rneuabboeusntahoeucccdd", "hstanoehutnaheoustnh")
-	s.trie.Update("rnxabboeusntahoeucccdd", "hstanoehutnaheoustnh")
-	s.trie.Delete("aaboaestnuhbccc")
-	s.trie.Delete("a")
-	s.trie.Update("a", "nthaonethaosentuh")
-	s.trie.Update("c", "shtaosntehua")
-	s.trie.Delete("a")
-	s.trie.Update("aaaa", "testmegood")
-
-	_, t2 := NewTrie()
-	s.trie.NewIterator().Each(func(key string, v *ethutil.Value) {
-		if key == "aaaa" {
-			t2.Update(key, v.Str())
-		} else {
-			t2.Update(key, v.Str())
-		}
-	})
-
-	a := ethutil.NewValue(s.trie.Root).Bytes()
-	b := ethutil.NewValue(t2.Root).Bytes()
-
-	c.Assert(a, checker.DeepEquals, b)
-}
-
-func (s *TrieSuite) TestTerminator(c *checker.C) {
-	key := CompactDecode("hello")
-	c.Assert(HasTerm(key), checker.Equals, true, checker.Commentf("Expected %v to have a terminator", key))
-}
-
-func (s *TrieSuite) TestIt(c *checker.C) {
-	s.trie.Update("cat", "cat")
-	s.trie.Update("doge", "doge")
-	s.trie.Update("wallace", "wallace")
-	it := s.trie.Iterator()
-
-	inputs := []struct {
-		In, Out string
-	}{
-		{"", "cat"},
-		{"bobo", "cat"},
-		{"c", "cat"},
-		{"car", "cat"},
-		{"catering", "doge"},
-		{"w", "wallace"},
-		{"wallace123", ""},
-	}
-
-	for _, test := range inputs {
-		res := string(it.Next(test.In))
-		c.Assert(res, checker.Equals, test.Out)
 	}
 }
 
-func (s *TrieSuite) TestBeginsWith(c *checker.C) {
+func TestInsert(t *testing.T) {
-	a := CompactDecode("hello")
+	trie := NewEmpty()
-	b := CompactDecode("hel")
 
-	c.Assert(BeginsWith(a, b), checker.Equals, false)
+	trie.UpdateString("doe", "reindeer")
-	c.Assert(BeginsWith(b, a), checker.Equals, true)
+	trie.UpdateString("dog", "puppy")
+	trie.UpdateString("dogglesworth", "cat")
+
+	exp := ethutil.Hex2Bytes("8aad789dff2f538bca5d8ea56e8abe10f4c7ba3a5dea95fea4cd6e7c3a1168d3")
+	root := trie.Hash()
+	if !bytes.Equal(root, exp) {
+		t.Errorf("exp %x got %x", exp, root)
+	}
+
+	trie = NewEmpty()
+	trie.UpdateString("A", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
+
+	exp = ethutil.Hex2Bytes("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab")
+	root = trie.Hash()
+	if !bytes.Equal(root, exp) {
+		t.Errorf("exp %x got %x", exp, root)
+	}
 }
 
-func (s *TrieSuite) TestItems(c *checker.C) {
+func TestGet(t *testing.T) {
-	s.trie.Update("A", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
+	trie := NewEmpty()
-	exp := "d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab"
 
-	c.Assert(s.trie.GetRoot(), checker.DeepEquals, ethutil.Hex2Bytes(exp))
+	trie.UpdateString("doe", "reindeer")
+	trie.UpdateString("dog", "puppy")
+	trie.UpdateString("dogglesworth", "cat")
+
+	res := trie.GetString("dog")
+	if !bytes.Equal(res, []byte("puppy")) {
+		t.Errorf("expected puppy got %x", res)
+	}
+
+	unknown := trie.GetString("unknown")
+	if unknown != nil {
+		t.Errorf("expected nil got %x", unknown)
+	}
 }
 
-func TestOtherSomething(t *testing.T) {
+func TestDelete(t *testing.T) {
-	_, trie := NewTrie()
+	trie := NewEmpty()
 
 	vals := []struct{ k, v string }{
 		{"do", "verb"},
@@ -352,18 +83,46 @@ func TestOtherSomething(t *testing.T) {
 		{"shaman", ""},
 	}
 	for _, val := range vals {
-		trie.Update(val.k, val.v)
+		if val.v != "" {
+			trie.UpdateString(val.k, val.v)
+		} else {
+			trie.DeleteString(val.k)
+		}
 	}
 
+	hash := trie.Hash()
 	exp := ethutil.Hex2Bytes("5991bb8c6514148a29db676a14ac506cd2cd5775ace63c30a4fe457715e9ac84")
-	hash := trie.Root.([]byte)
 	if !bytes.Equal(hash, exp) {
 		t.Errorf("expected %x got %x", exp, hash)
 	}
 }
 
-func BenchmarkGets(b *testing.B) {
+func TestEmptyValues(t *testing.T) {
-	_, trie := NewTrie()
+	trie := NewEmpty()
+
+	vals := []struct{ k, v string }{
+		{"do", "verb"},
+		{"ether", "wookiedoo"},
+		{"horse", "stallion"},
+		{"shaman", "horse"},
+		{"doge", "coin"},
+		{"ether", ""},
+		{"dog", "puppy"},
+		{"shaman", ""},
+	}
+	for _, val := range vals {
+		trie.UpdateString(val.k, val.v)
+	}
+
+	hash := trie.Hash()
+	exp := ethutil.Hex2Bytes("5991bb8c6514148a29db676a14ac506cd2cd5775ace63c30a4fe457715e9ac84")
+	if !bytes.Equal(hash, exp) {
+		t.Errorf("expected %x got %x", exp, hash)
+	}
+}
+
+func TestReplication(t *testing.T) {
+	trie := NewEmpty()
 	vals := []struct{ k, v string }{
 		{"do", "verb"},
 		{"ether", "wookiedoo"},
@@ -376,21 +135,125 @@ func BenchmarkGets(b *testing.B) {
 		{"somethingveryoddindeedthis is", "myothernodedata"},
 	}
 	for _, val := range vals {
-		trie.Update(val.k, val.v)
+		trie.UpdateString(val.k, val.v)
+	}
+	trie.Commit()
+
+	trie2 := New(trie.roothash, trie.cache.backend)
+	if string(trie2.GetString("horse")) != "stallion" {
+		t.Error("expected to have horse => stallion")
+	}
+
+	hash := trie2.Hash()
+	exp := trie.Hash()
+	if !bytes.Equal(hash, exp) {
+		t.Errorf("root failure. expected %x got %x", exp, hash)
+	}
+
+}
+
+func TestReset(t *testing.T) {
+	trie := NewEmpty()
+	vals := []struct{ k, v string }{
+		{"do", "verb"},
+		{"ether", "wookiedoo"},
+		{"horse", "stallion"},
+	}
+	for _, val := range vals {
+		trie.UpdateString(val.k, val.v)
+	}
+	trie.Commit()
+
+	before := ethutil.CopyBytes(trie.roothash)
+	trie.UpdateString("should", "revert")
+	trie.Hash()
+	// Should have no effect
+	trie.Hash()
+	trie.Hash()
+	// ###
+
+	trie.Reset()
+	after := ethutil.CopyBytes(trie.roothash)
+
+	if !bytes.Equal(before, after) {
+		t.Errorf("expected roots to be equal. %x - %x", before, after)
+	}
+}
+
+func TestParanoia(t *testing.T) {
+	t.Skip()
+	trie := NewEmpty()
+
+	vals := []struct{ k, v string }{
+		{"do", "verb"},
+		{"ether", "wookiedoo"},
+		{"horse", "stallion"},
+		{"shaman", "horse"},
+		{"doge", "coin"},
+		{"ether", ""},
+		{"dog", "puppy"},
+		{"shaman", ""},
+		{"somethingveryoddindeedthis is", "myothernodedata"},
+	}
+	for _, val := range vals {
+		trie.UpdateString(val.k, val.v)
+	}
+	trie.Commit()
+
+	ok, t2 := ParanoiaCheck(trie, trie.cache.backend)
+	if !ok {
+		t.Errorf("trie paranoia check failed %x %x", trie.roothash, t2.roothash)
+	}
+}
+
+// Not an actual test
+func TestOutput(t *testing.T) {
+	t.Skip()
+
+	base := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+	trie := NewEmpty()
+	for i := 0; i < 50; i++ {
+		trie.UpdateString(fmt.Sprintf("%s%d", base, i), "valueeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee")
+	}
+	fmt.Println("############################## FULL ################################")
+	fmt.Println(trie.root)
+
+	trie.Commit()
+	fmt.Println("############################## SMALL ################################")
+	trie2 := New(trie.roothash, trie.cache.backend)
+	trie2.GetString(base + "20")
+	fmt.Println(trie2.root)
+}
+
+func BenchmarkGets(b *testing.B) {
+	trie := NewEmpty()
+	vals := []struct{ k, v string }{
+		{"do", "verb"},
+		{"ether", "wookiedoo"},
+		{"horse", "stallion"},
+		{"shaman", "horse"},
+		{"doge", "coin"},
+		{"ether", ""},
+		{"dog", "puppy"},
+		{"shaman", ""},
+		{"somethingveryoddindeedthis is", "myothernodedata"},
+	}
+	for _, val := range vals {
+		trie.UpdateString(val.k, val.v)
 	}
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		trie.Get("horse")
+		trie.Get([]byte("horse"))
 	}
 }
 
 func BenchmarkUpdate(b *testing.B) {
-	_, trie := NewTrie()
+	trie := NewEmpty()
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		trie.Update(fmt.Sprintf("aaaaaaaaaaaaaaa%d", i), "value")
+		trie.UpdateString(fmt.Sprintf("aaaaaaaaa%d", i), "value")
 	}
+	trie.Hash()
 }
-*/

trie/valuenode.go

@@ -1,4 +1,4 @@
-package ptrie
+package trie
 
 type ValueNode struct {
 	trie *Trie

types/ethereum.go

@@ -1 +0,0 @@
-package types

ui/qt/qwhisper/whisper.go

@@ -6,10 +6,13 @@ import (
 
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethutil"
+	"github.com/ethereum/go-ethereum/logger"
 	"github.com/ethereum/go-ethereum/whisper"
 	"gopkg.in/qml.v1"
 )
 
+var qlogger = logger.NewLogger("QSHH")
+
 func fromHex(s string) []byte {
 	if len(s) > 1 {
 		return ethutil.Hex2Bytes(s[2:])
@@ -36,9 +39,10 @@ func (self *Whisper) SetView(view qml.Object) {
 func (self *Whisper) Post(payload []string, to, from string, topics []string, priority, ttl uint32) {
 	var data []byte
 	for _, d := range payload {
-		data = append(data, fromHex(d)...)
+		data = append(data, ethutil.Hex2Bytes(d)...)
 	}
 
+	fmt.Println(payload, data, "from", from, fromHex(from), crypto.ToECDSA(fromHex(from)))
 	msg := whisper.NewMessage(data)
 	envelope, err := msg.Seal(time.Duration(priority*100000), whisper.Opts{
 		Ttl:    time.Duration(ttl),
@@ -47,13 +51,13 @@ func (self *Whisper) Post(payload []string, to, from string, topics []string, pr
 		Topics: whisper.TopicsFromString(topics...),
 	})
 	if err != nil {
-		fmt.Println(err)
+		qlogger.Infoln(err)
 		// handle error
 		return
 	}
 
 	if err := self.Whisper.Send(envelope); err != nil {
-		fmt.Println(err)
+		qlogger.Infoln(err)
 		// handle error
 		return
 	}

whisper/message.go

@@ -2,6 +2,7 @@ package whisper
 
 import (
 	"crypto/ecdsa"
+	"fmt"
 	"time"
 
 	"github.com/ethereum/go-ethereum/crypto"
@@ -53,6 +54,7 @@ type Opts struct {
 }
 
 func (self *Message) Seal(pow time.Duration, opts Opts) (*Envelope, error) {
+	fmt.Println(opts)
 	if opts.From != nil {
 		err := self.sign(opts.From)
 		if err != nil {

whisper/peer.go

@@ -55,7 +55,7 @@ out:
 		case <-relay.C:
 			err := self.broadcast(self.host.envelopes())
 			if err != nil {
-				self.peer.Infoln(err)
+				self.peer.Infoln("broadcast err:", err)
 				break out
 			}
 

whisper/whisper.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"crypto/ecdsa"
 	"errors"
+	"fmt"
 	"sync"
 	"time"
 
@@ -143,6 +144,7 @@ func (self *Whisper) msgHandler(peer *p2p.Peer, ws p2p.MsgReadWriter) error {
 		if err != nil {
 			return err
 		}
+		fmt.Println("reading message")
 
 		envelope, err := NewEnvelopeFromReader(msg.Payload)
 		if err != nil {
@@ -160,6 +162,7 @@ func (self *Whisper) msgHandler(peer *p2p.Peer, ws p2p.MsgReadWriter) error {
 
 // takes care of adding envelopes to the messages pool. At this moment no sanity checks are being performed.
 func (self *Whisper) add(envelope *Envelope) error {
+	fmt.Println("adding")
 	if !envelope.valid() {
 		return errors.New("invalid pow provided for envelope")
 	}
@@ -229,11 +232,11 @@ func (self *Whisper) envelopes() (envelopes []*Envelope) {
 func (self *Whisper) postEvent(envelope *Envelope) {
 	for _, key := range self.keys {
 		if message, err := envelope.Open(key); err == nil || (err != nil && err == ecies.ErrInvalidPublicKey) {
-			// Create a custom filter?
 			self.filters.Notify(filter.Generic{
 				Str1: string(crypto.FromECDSA(key)), Str2: string(crypto.FromECDSAPub(message.Recover())),
 				Data: bytesToMap(envelope.Topics),
 			}, message)
+			break
 		} else {
 			wlogger.Infoln(err)
 		}