Merge pull request #2 from ethereum/master

Merge Upstream

.travis.yml

@@ -1,16 +1,30 @@
 language: go
 go_import_path: github.com/ethereum/go-ethereum
 sudo: false
-matrix:
+jobs:
   include:
-    - os: linux
+        # This builder only tests code linters on latest version of Go
+    - stage: lint
+      os: linux
+      dist: xenial
+      go: 1.12.x
+      env:
+        - lint
+      git:
+        submodules: false # avoid cloning ethereum/tests
+      script:
+        - go run build/ci.go lint
+
+    - stage: build
+      os: linux
       dist: xenial
       go: 1.10.x
       script:
       - go run build/ci.go install
       - go run build/ci.go test -coverage $TEST_PACKAGES
 
-    - os: linux
+    - stage: build
+      os: linux
       dist: xenial
       go: 1.11.x
       script:
@@ -18,14 +32,16 @@ matrix:
         - go run build/ci.go test -coverage $TEST_PACKAGES
 
     # These are the latest Go versions.
-    - os: linux
+    - stage: build
+      os: linux
       dist: xenial
       go: 1.12.x
       script:
         - go run build/ci.go install
         - go run build/ci.go test -coverage $TEST_PACKAGES
 
-    - os: osx
+    - stage: build
+      os: osx
       go: 1.12.x
       script:
         - echo "Increase the maximum number of open file descriptors on macOS"
@@ -40,19 +56,9 @@ matrix:
         - go run build/ci.go install
         - go run build/ci.go test -coverage $TEST_PACKAGES
 
-    # This builder only tests code linters on latest version of Go
-    - os: linux
-      dist: xenial
-      go: 1.12.x
-      env:
-        - lint
-      git:
-        submodules: false # avoid cloning ethereum/tests
-      script:
-        - go run build/ci.go lint
-
     # This builder does the Ubuntu PPA upload
-    - if: type = push
+    - stage: build
+      if: type = push
       os: linux
       dist: xenial
       go: 1.12.x
@@ -74,7 +80,8 @@ matrix:
         - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
 
     # This builder does the Linux Azure uploads
-    - if: type = push
+    - stage: build
+      if: type = push
       os: linux
       dist: xenial
       sudo: required
@@ -108,7 +115,8 @@ matrix:
         - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
 
     # This builder does the Linux Azure MIPS xgo uploads
-    - if: type = push
+    - stage: build
+      if: type = push
       os: linux
       dist: xenial
       services:
@@ -136,7 +144,8 @@ matrix:
         - go run build/ci.go archive -arch mips64le -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
 
     # This builder does the Android Maven and Azure uploads
-    - if: type = push
+    - stage: build
+      if: type = push
       os: linux
       dist: xenial
       addons:
@@ -173,7 +182,8 @@ matrix:
         - go run build/ci.go aar -signer ANDROID_SIGNING_KEY -deploy https://oss.sonatype.org -upload gethstore/builds
 
     # This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads
-    - if: type = push
+    - stage: build
+      if: type = push
       os: osx
       go: 1.12.x
       env:
@@ -202,7 +212,8 @@ matrix:
         - go run build/ci.go xcode -signer IOS_SIGNING_KEY -deploy trunk -upload gethstore/builds
 
     # This builder does the Azure archive purges to avoid accumulating junk
-    - if: type = cron
+    - stage: build
+      if: type = cron
       os: linux
       dist: xenial
       go: 1.12.x

AUTHORS

@@ -1,5 +1,11 @@
 # This is the official list of go-ethereum authors for copyright purposes.
 
+a e r t h <aerth@users.noreply.github.com>
+Abel Nieto <abel.nieto90@gmail.com>
+Abel Nieto <anietoro@uwaterloo.ca>
+Adam Babik <a.babik@designfortress.com>
+Aditya <adityasripal@gmail.com>
+Adrià Cidre <adria.cidre@gmail.com>
 Afri Schoedon <5chdn@users.noreply.github.com>
 Agustin Armellini Fischer <armellini13@gmail.com>
 Airead <fgh1987168@gmail.com>
@@ -10,165 +16,354 @@ Alex Leverington <alex@ethdev.com>
 Alex Wu <wuyiding@gmail.com>
 Alexandre Van de Sande <alex.vandesande@ethdev.com>
 Ali Hajimirza <Ali92hm@users.noreply.github.com>
+am2rican5 <am2rican5@gmail.com>
+Andrea Franz <andrea@gravityblast.com>
+Andrey Petrov <andrey.petrov@shazow.net>
+Andrey Petrov <shazow@gmail.com>
+ANOTHEL <anothel1@naver.com>
+Antoine Rondelet <rondelet.antoine@gmail.com>
 Anton Evangelatov <anton.evangelatov@gmail.com>
+Antonio Salazar Cardozo <savedfastcool@gmail.com>
 Arba Sasmoyo <arba.sasmoyo@gmail.com>
 Armani Ferrante <armaniferrante@berkeley.edu>
 Armin Braun <me@obrown.io>
 Aron Fischer <github@aron.guru>
+atsushi-ishibashi <atsushi.ishibashi@finatext.com>
+ayeowch <ayeowch@gmail.com>
+b00ris <b00ris@mail.ru>
+bailantaotao <Edwin@maicoin.com>
+baizhenxuan <nkbai@163.com>
+Balint Gabor <balint.g@gmail.com>
 Bas van Kervel <bas@ethdev.com>
 Benjamin Brent <benjamin@benjaminbrent.com>
+benma <mbencun@gmail.com>
 Benoit Verkindt <benoit.verkindt@gmail.com>
+bloonfield <bloonfield@163.com>
 Bo <bohende@gmail.com>
 Bo Ye <boy.e.computer.1982@outlook.com>
 Bob Glickstein <bobg@users.noreply.github.com>
+Brent <bmperrea@gmail.com>
 Brian Schroeder <bts@gmail.com>
+Bruno Škvorc <bruno@skvorc.me>
+C. Brown <hackdom@majoolr.io>
+Caesar Chad <BLUE.WEB.GEEK@gmail.com>
 Casey Detrio <cdetrio@gmail.com>
+CDsigma <cdsigma271@gmail.com>
+changhong <changhong.yu@shanbay.com>
 Chase Wright <mysticryuujin@gmail.com>
+Chen Quan <terasum@163.com>
+chenyufeng <yufengcode@gmail.com>
+Christian Muehlhaeuser <muesli@gmail.com>
 Christoph Jentzsch <jentzsch.software@gmail.com>
+cong <ackratos@users.noreply.github.com>
+Corey Lin <514971757@qq.com>
+cpusoft <cpusoft@live.com>
+Crispin Flowerday <crispin@bitso.com>
+croath <croathliu@gmail.com>
+cui <523516579@qq.com>
+Dan Kinsley <dan@joincivil.com>
 Daniel A. Nagy <nagy.da@gmail.com>
 Daniel Sloof <goapsychadelic@gmail.com>
 Darrel Herbst <dherbst@gmail.com>
 Dave Appleton <calistralabs@gmail.com>
+Dave McGregor <dave.s.mcgregor@gmail.com>
+David Huie <dahuie@gmail.com>
+Derek Gottfrid <derek@codecubed.com>
 Diego Siqueira <DiSiqueira@users.noreply.github.com>
+Diep Pham <mrfavadi@gmail.com>
+dipingxian2 <39109351+dipingxian2@users.noreply.github.com>
+dm4 <sunrisedm4@gmail.com>
+Dmitrij Koniajev <dimchansky@gmail.com>
 Dmitry Shulyak <yashulyak@gmail.com>
+Domino Valdano <dominoplural@gmail.com>
+Domino Valdano <jeff@okcupid.com>
+Dragan Milic <dragan@netice9.com>
+dragonvslinux <35779158+dragononcrypto@users.noreply.github.com>
 Egon Elbre <egonelbre@gmail.com>
+Elad <theman@elad.im>
+Eli <elihanover@yahoo.com>
 Elias Naur <elias.naur@gmail.com>
 Elliot Shepherd <elliot@identitii.com>
+Emil <mursalimovemeel@gmail.com>
+emile <emile@users.noreply.github.com>
 Enrique Fynn <enriquefynn@gmail.com>
+Enrique Fynn <me@enriquefynn.com>
+EOS Classic <info@eos-classic.io>
+Erichin <erichinbato@gmail.com>
 Ernesto del Toro <ernesto.deltoro@gmail.com>
 Ethan Buchman <ethan@coinculture.info>
+ethersphere <thesw@rm.eth>
 Eugene Valeyev <evgen.povt@gmail.com>
 Evangelos Pappas <epappas@evalonlabs.com>
+Evgeny <awesome.observer@yandex.com>
 Evgeny Danilenko <6655321@bk.ru>
+evgk <evgeniy.kamyshev@gmail.com>
 Fabian Vogelsteller <fabian@frozeman.de>
 Fabio Barone <fabio.barone.co@gmail.com>
 Fabio Berger <fabioberger1991@gmail.com>
 FaceHo <facehoshi@gmail.com>
 Felix Lange <fjl@twurst.com>
+Ferenc Szabo <frncmx@gmail.com>
+ferhat elmas <elmas.ferhat@gmail.com>
 Fiisio <liangcszzu@163.com>
+Frank Szendzielarz <33515470+FrankSzendzielarz@users.noreply.github.com>
 Frank Wang <eternnoir@gmail.com>
+Franklin <mr_franklin@126.com>
 Furkan KAMACI <furkankamaci@gmail.com>
+GagziW <leon.stanko@rwth-aachen.de>
 Gary Rong <garyrong0905@gmail.com>
 George Ornbo <george@shapeshed.com>
 Gregg Dourgarian <greggd@tempworks.com>
+Guilherme Salgado <gsalgado@gmail.com>
 Guillaume Ballet <gballet@gmail.com>
 Guillaume Nicolas <guin56@gmail.com>
+GuiltyMorishita <morilliantblue@gmail.com>
+Gus <yo@soygus.com>
 Gustav Simonsson <gustav.simonsson@gmail.com>
+Gísli Kristjánsson <gislik@hamstur.is>
+Ha ĐANG <dvietha@gmail.com>
+HackyMiner <hackyminer@gmail.com>
+hadv <dvietha@gmail.com>
 Hao Bryan Cheng <haobcheng@gmail.com>
+HAOYUatHZ <37070449+HAOYUatHZ@users.noreply.github.com>
 Henning Diedrich <hd@eonblast.com>
+holisticode <holistic.computing@gmail.com>
+Hongbin Mao <hello2mao@gmail.com>
+Hsien-Tang Kao <htkao@pm.me>
+Husam Ibrahim <39692071+HusamIbrahim@users.noreply.github.com>
+hydai <z54981220@gmail.com>
+Hyung-Kyu Hqueue Choi <hyungkyu.choi@gmail.com>
+Ian Macalinao <me@ian.pw>
+Ian Norden <iannordenn@gmail.com>
 Isidoro Ghezzi <isidoro.ghezzi@icloud.com>
+Iskander (Alex) Sharipov <quasilyte@gmail.com>
 Ivan Daniluk <ivan.daniluk@gmail.com>
+Ivo Georgiev <ivo@strem.io>
 Jae Kwon <jkwon.work@gmail.com>
 Jamie Pitts <james.pitts@gmail.com>
+Janos Guljas <janos@resenje.org>
 Janoš Guljaš <janos@users.noreply.github.com>
 Jason Carver <jacarver@linkedin.com>
+Javier Peletier <jm@epiclabs.io>
+Javier Peletier <jpeletier@users.noreply.github.com>
+Javier Sagredo <jasataco@gmail.com>
+Jay <codeholic.arena@gmail.com>
 Jay Guo <guojiannan1101@gmail.com>
+Jaynti Kanani <jdkanani@gmail.com>
+Jeff Prestes <jeffprestes@gmail.com>
 Jeff R. Allen <jra@nella.org>
+Jeffery Robert Walsh <rlxrlps@gmail.com>
 Jeffrey Wilcke <jeffrey@ethereum.org>
 Jens Agerberg <github@agerberg.me>
+Jeremy McNevin <jeremy.mcnevin@optum.com>
+Jeremy Schlatter <jeremy.schlatter@gmail.com>
+Jerzy Lasyk <jerzylasyk@gmail.com>
 Jia Chenhui <jiachenhui1989@gmail.com>
 Jim McDonald <Jim@mcdee.net>
+jkcomment <jkcomment@gmail.com>
 Joel Burget <joelburget@gmail.com>
+John C. Vernaleo <john@netpurgatory.com>
+Johns Beharry <johns@peakshift.com>
+Jonas <felberj@users.noreply.github.com>
 Jonathan Brown <jbrown@bluedroplet.com>
+JoranHonig <JoranHonig@users.noreply.github.com>
+Jordan Krage <jmank88@gmail.com>
 Joseph Chow <ethereum@outlook.com>
+jtakalai <juuso.takalainen@streamr.com>
+JU HYEONG PARK <dkdkajej@gmail.com>
 Justin Clark-Casey <justincc@justincc.org>
 Justin Drake <drakefjustin@gmail.com>
+jwasinger <j-wasinger@hotmail.com>
+ken10100147 <sunhongping@kanjian.com>
 Kenji Siu <kenji@isuntv.com>
+Kenso Trabing <kenso.trabing@bloomwebsite.com>
+Kenso Trabing <ktrabing@acm.org>
+Kevin <denk.kevin@web.de>
+kevin.xu <cming.xu@gmail.com>
+kiel barry <kiel.j.barry@gmail.com>
+kimmylin <30611210+kimmylin@users.noreply.github.com>
+Kitten King <53072918+kittenking@users.noreply.github.com>
+knarfeh <hejun1874@gmail.com>
 Kobi Gurkan <kobigurk@gmail.com>
 Konrad Feldmeier <konrad@brainbot.com>
+Kris Shinn <raggamuffin.music@gmail.com>
 Kurkó Mihály <kurkomisi@users.noreply.github.com>
+Kushagra Sharma <ksharm01@gmail.com>
+Kwuaint <34888408+kwuaint@users.noreply.github.com>
 Kyuntae Ethan Kim <ethan.kyuntae.kim@gmail.com>
+ledgerwatch <akhounov@gmail.com>
 Lefteris Karapetsas <lefteris@refu.co>
 Leif Jurvetson <leijurv@gmail.com>
 Leo Shklovskii <leo@thermopylae.net>
+LeoLiao <leofantast@gmail.com>
 Lewis Marshall <lewis@lmars.net>
+lhendre <lhendre2@gmail.com>
+Liang Ma <liangma.ul@gmail.com>
+Liang Ma <liangma@liangbit.com>
+Liang ZOU <liang.d.zou@gmail.com>
+libotony <liboliqi@gmail.com>
+ligi <ligi@ligi.de>
 Lio李欧 <lionello@users.noreply.github.com>
+Lorenzo Manacorda <lorenzo@kinvolk.io>
 Louis Holbrook <dev@holbrook.no>
 Luca Zeug <luclu@users.noreply.github.com>
 Magicking <s@6120.eu>
+manlio <manlio.poltronieri@gmail.com>
 Maran Hidskes <maran.hidskes@gmail.com>
 Marek Kotewicz <marek.kotewicz@gmail.com>
+Marius van der Wijden <m.vanderwijden@live.de>
 Mark <markya0616@gmail.com>
+Mark Rushakoff <mark.rushakoff@gmail.com>
+mark.lin <mark@maicoin.com>
+Martin Alex Philip Dawson <u1356770@gmail.com>
 Martin Holst Swende <martin@swende.se>
+Martin Klepsch <martinklepsch@googlemail.com>
+Mats Julian Olsen <mats@plysjbyen.net>
+Matt K <1036969+mkrump@users.noreply.github.com>
 Matthew Di Ferrante <mattdf@users.noreply.github.com>
+Matthew Halpern <matthalp@gmail.com>
+Matthew Halpern <matthalp@google.com>
 Matthew Wampler-Doty <matthew.wampler.doty@gmail.com>
+Max Sistemich <mafrasi2@googlemail.com>
 Maximilian Meister <mmeister@suse.de>
 Micah Zoltu <micah@zoltu.net>
 Michael Ruminer <michael.ruminer+github@gmail.com>
 Miguel Mota <miguelmota2@gmail.com>
 Miya Chen <miyatlchen@gmail.com>
+Mohanson <mohanson@outlook.com>
+mr_franklin <mr_franklin@126.com>
+Mymskmkt <1847234666@qq.com>
+Nalin Bhardwaj <nalinbhardwaj@nibnalin.me>
 Nchinda Nchinda <nchinda2@gmail.com>
+necaremus <necaremus@gmail.com>
+needkane <604476380@qq.com>
+Nguyen Kien Trung <trung.n.k@gmail.com>
+Nguyen Sy Thanh Son <thanhson1085@gmail.com>
 Nick Dodson <silentcicero@outlook.com>
 Nick Johnson <arachnid@notdot.net>
 Nicolas Guillaume <gunicolas@sqli.com>
+Nilesh Trivedi <nilesh@hypertrack.io>
+Nimrod Gutman <nimrod.gutman@gmail.com>
+njupt-moon <1015041018@njupt.edu.cn>
+nkbai <nkbai@163.com>
+nobody <ddean2009@163.com>
 Noman <noman@noman.land>
+Oleg Kovalov <iamolegkovalov@gmail.com>
 Oli Bye <olibye@users.noreply.github.com>
+Osuke <arget-fee.free.dgm@hotmail.co.jp>
+Paul Berg <hello@paulrberg.com>
 Paul Litvak <litvakpol@012.net.il>
 Paulo L F Casaretto <pcasaretto@gmail.com>
 Paweł Bylica <chfast@gmail.com>
+Pedro Pombeiro <PombeirP@users.noreply.github.com>
+Peter Broadhurst <peter@themumbles.net>
 Peter Pratscher <pratscher@gmail.com>
 Petr Mikusek <petr@mikusek.info>
+Philip Schlump <pschlump@gmail.com>
+Pierre Neter <pierreneter@gmail.com>
+PilkyuJung <anothel1@naver.com>
+protolambda <proto@protolambda.com>
 Péter Szilágyi <peterke@gmail.com>
-RJ Catalano <catalanor0220@gmail.com>
+qd-ethan <31876119+qdgogogo@users.noreply.github.com>
+Raghav Sood <raghavsood@gmail.com>
+Ralph Caraveo <deckarep@gmail.com>
+Ralph Caraveo III <deckarep@gmail.com>
 Ramesh Nair <ram@hiddentao.com>
+reinerRubin <tolstov.georgij@gmail.com>
+rhaps107 <dod-source@yandex.ru>
 Ricardo Catalinas Jiménez <r@untroubled.be>
 Ricardo Domingos <ricardohsd@gmail.com>
 Richard Hart <richardhart92@gmail.com>
+RJ Catalano <catalanor0220@gmail.com>
 Rob <robert@rojotek.com>
+Rob Mulholand <rmulholand@8thlight.com>
 Robert Zaremba <robert.zaremba@scale-it.pl>
+Roc Yu <rociiu0112@gmail.com>
+Runchao Han <elvisage941102@gmail.com>
 Russ Cox <rsc@golang.org>
+Ryan Schneider <ryanleeschneider@gmail.com>
 Rémy Roy <remyroy@remyroy.com>
 S. Matthew English <s-matthew-english@users.noreply.github.com>
+salanfe <salanfe@users.noreply.github.com>
+Samuel Marks <samuelmarks@gmail.com>
+Sarlor <kinsleer@outlook.com>
+Sasuke1964 <neilperry1964@gmail.com>
+Saulius Grigaitis <saulius@necolt.com>
+Sean <darcys22@gmail.com>
+Sheldon <11510383@mail.sustc.edu.cn>
+Sheldon <374662347@qq.com>
 Shintaro Kaneko <kaneshin0120@gmail.com>
+Shuai Qi <qishuai231@gmail.com>
+Shunsuke Watanabe <ww.shunsuke@gmail.com>
+silence <wangsai.silence@qq.com>
+Simon Jentzsch <simon@slock.it>
+slumber1122 <slumber1122@gmail.com>
+Smilenator <yurivanenko@yandex.ru>
 Sorin Neacsu <sorin.neacsu@gmail.com>
 Stein Dekker <dekker.stein@gmail.com>
+Steve Gattuso <steve@stevegattuso.me>
+Steve Ruckdashel <steve.ruckdashel@gmail.com>
 Steve Waldman <swaldman@mchange.com>
 Steven Roose <stevenroose@gmail.com>
+stompesi <stompesi@gmail.com>
+stormpang <jialinpeng@vip.qq.com>
+sunxiaojun2014 <sunxiaojun-xy@360.cn>
+tamirms <tamir@trello.com>
 Taylor Gerring <taylor.gerring@gmail.com>
+TColl <38299499+TColl@users.noreply.github.com>
+terasum <terasum@163.com>
 Thomas Bocek <tom@tomp2p.net>
+thomasmodeneis <thomas.modeneis@gmail.com>
+thumb8432 <thumb8432@gmail.com>
 Ti Zhou <tizhou1986@gmail.com>
 Tosh Camille <tochecamille@gmail.com>
+tsarpaul <Litvakpol@012.net.il>
+tzapu <alex@tzapu.com>
+ult-bobonovski <alex@ultiledger.io>
 Valentin Wüstholz <wuestholz@gmail.com>
+Vedhavyas Singareddi <vedhavyas.singareddi@gmail.com>
 Victor Farazdagi <simple.square@gmail.com>
 Victor Tran <vu.tran54@gmail.com>
+Vie <yangchenzhong@gmail.com>
 Viktor Trón <viktor.tron@gmail.com>
 Ville Sundell <github@solarius.fi>
+vim88 <vim88vim88@gmail.com>
 Vincent G <caktux@gmail.com>
+Vincent Serpoul <vincent@serpoul.com>
 Vitalik Buterin <v@buterin.com>
+Vitaly Bogdanov <vsbogd@gmail.com>
 Vitaly V <vvelikodny@gmail.com>
 Vivek Anand <vivekanand1101@users.noreply.github.com>
+Vlad <gluk256@gmail.com>
+Vlad Bokov <razum2um@mail.ru>
 Vlad Gluhovsky <gluk256@users.noreply.github.com>
+weimumu <934657014@qq.com>
+Wenbiao Zheng <delweng@gmail.com>
+William Setzer <bootstrapsetzer@gmail.com>
+williambannas <wrschwartz@wpi.edu>
+Wuxiang <wuxiangzhou2010@gmail.com>
+xiekeyang <xiekeyang@users.noreply.github.com>
+xincaosu <xincaosu@126.com>
+yahtoo <yahtoo.ma@gmail.com>
+YaoZengzeng <yaozengzeng@zju.edu.cn>
+YH-Zhou <yanhong.zhou05@gmail.com>
 Yohann Léon <sybiload@gmail.com>
 Yoichi Hirai <i@yoichihirai.com>
 Yondon Fu <yondon.fu@gmail.com>
+YOSHIDA Masanori <masanori.yoshida@gmail.com>
+yoza <yoza.is12s@gmail.com>
+Yusup <awklsgrep@gmail.com>
 Zach <zach.ramsay@gmail.com>
+zah <zahary@gmail.com>
 Zahoor Mohamed <zahoor@zahoor.in>
+Zak Cole <zak@beattiecole.com>
+zer0to0ne <36526113+zer0to0ne@users.noreply.github.com>
+Zhenguo Niu <Niu.ZGlinux@gmail.com>
 Zoe Nolan <github@zoenolan.org>
 Zsolt Felföldi <zsfelfoldi@gmail.com>
-am2rican5 <am2rican5@gmail.com>
+Łukasz Kurowski <crackcomm@users.noreply.github.com>
-ayeowch <ayeowch@gmail.com>
-b00ris <b00ris@mail.ru>
-bailantaotao <Edwin@maicoin.com>
-baizhenxuan <nkbai@163.com>
-bloonfield <bloonfield@163.com>
-changhong <changhong.yu@shanbay.com>
-evgk <evgeniy.kamyshev@gmail.com>
-ferhat elmas <elmas.ferhat@gmail.com>
-holisticode <holistic.computing@gmail.com>
-jtakalai <juuso.takalainen@streamr.com>
-ken10100147 <sunhongping@kanjian.com>
-ligi <ligi@ligi.de>
-mark.lin <mark@maicoin.com>
-necaremus <necaremus@gmail.com>
-njupt-moon <1015041018@njupt.edu.cn>
-nkbai <nkbai@163.com>
-rhaps107 <dod-source@yandex.ru>
-slumber1122 <slumber1122@gmail.com>
-sunxiaojun2014 <sunxiaojun-xy@360.cn>
-terasum <terasum@163.com>
-tsarpaul <Litvakpol@012.net.il>
-xiekeyang <xiekeyang@users.noreply.github.com>
-yoza <yoza.is12s@gmail.com>
 ΞTHΞЯSPHΞЯΞ <{viktor.tron,nagydani,zsfelfoldi}@gmail.com>
 Максим Чусовлянов <mchusovlianov@gmail.com>
-Ralph Caraveo <deckarep@gmail.com>
+大彬 <hz_stb@163.com>
+贺鹏飞 <hpf@hackerful.cn>
+유용환 <33824408+eric-yoo@users.noreply.github.com>

README.md

@@ -98,7 +98,7 @@ Specifying the `--testnet` flag, however, will reconfigure your `geth` instance
    this.
  * Instead of connecting the main Ethereum network, the client will connect to the test
    network, which uses different P2P bootnodes, different network IDs and genesis states.
-   
+
 *Note: Although there are some internal protective measures to prevent transactions from
 crossing over between the main network and test network, you should make sure to always
 use separate accounts for play-money and real-money. Unless you manually move
@@ -212,8 +212,12 @@ aware of and agree upon. This consists of a small JSON file (e.g. call it `genes
   "config": {
     "chainId": 0,
     "homesteadBlock": 0,
+    "eip150Block": 0,
     "eip155Block": 0,
-    "eip158Block": 0
+    "eip158Block": 0,
+    "byzantiumBlock": 0,
+    "constantinopleBlock": 0,
+    "petersburgBlock": 0
   },
   "alloc": {},
   "coinbase": "0x0000000000000000000000000000000000000000",

accounts/abi/abi.go

@@ -21,6 +21,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+
+	"github.com/ethereum/go-ethereum/common"
 )
 
 // The ABI holds information about a contract's context and available
@@ -68,7 +70,7 @@ func (abi ABI) Pack(name string, args ...interface{}) ([]byte, error) {
 		return nil, err
 	}
 	// Pack up the method ID too if not a constructor and return
-	return append(method.Id(), arguments...), nil
+	return append(method.ID(), arguments...), nil
 }
 
 // Unpack output in v according to the abi specification
@@ -119,11 +121,9 @@ func (abi *ABI) UnmarshalJSON(data []byte) error {
 		Inputs    []Argument
 		Outputs   []Argument
 	}
-
 	if err := json.Unmarshal(data, &fields); err != nil {
 		return err
 	}
-
 	abi.Methods = make(map[string]Method)
 	abi.Events = make(map[string]Event)
 	for _, field := range fields {
@@ -134,15 +134,29 @@ func (abi *ABI) UnmarshalJSON(data []byte) error {
 			}
 		// empty defaults to function according to the abi spec
 		case "function", "":
-			abi.Methods[field.Name] = Method{
+			name := field.Name
-				Name:    field.Name,
+			_, ok := abi.Methods[name]
+			for idx := 0; ok; idx++ {
+				name = fmt.Sprintf("%s%d", field.Name, idx)
+				_, ok = abi.Methods[name]
+			}
+			abi.Methods[name] = Method{
+				Name:    name,
+				RawName: field.Name,
 				Const:   field.Constant,
 				Inputs:  field.Inputs,
 				Outputs: field.Outputs,
 			}
 		case "event":
-			abi.Events[field.Name] = Event{
+			name := field.Name
-				Name:      field.Name,
+			_, ok := abi.Events[name]
+			for idx := 0; ok; idx++ {
+				name = fmt.Sprintf("%s%d", field.Name, idx)
+				_, ok = abi.Events[name]
+			}
+			abi.Events[name] = Event{
+				Name:      name,
+				RawName:   field.Name,
 				Anonymous: field.Anonymous,
 				Inputs:    field.Inputs,
 			}
@@ -159,9 +173,20 @@ func (abi *ABI) MethodById(sigdata []byte) (*Method, error) {
 		return nil, fmt.Errorf("data too short (%d bytes) for abi method lookup", len(sigdata))
 	}
 	for _, method := range abi.Methods {
-		if bytes.Equal(method.Id(), sigdata[:4]) {
+		if bytes.Equal(method.ID(), sigdata[:4]) {
 			return &method, nil
 		}
 	}
 	return nil, fmt.Errorf("no method with id: %#x", sigdata[:4])
 }
+
+// EventByID looks an event up by its topic hash in the
+// ABI and returns nil if none found.
+func (abi *ABI) EventByID(topic common.Hash) (*Event, error) {
+	for _, event := range abi.Events {
+		if bytes.Equal(event.ID().Bytes(), topic.Bytes()) {
+			return &event, nil
+		}
+	}
+	return nil, fmt.Errorf("no event with id: %#x", topic.Hex())
+}

accounts/abi/abi_test.go

@@ -20,7 +20,6 @@ import (
 	"bytes"
 	"encoding/hex"
 	"fmt"
-	"log"
 	"math/big"
 	"reflect"
 	"strings"
@@ -62,10 +61,10 @@ func TestReader(t *testing.T) {
 	exp := ABI{
 		Methods: map[string]Method{
 			"balance": {
-				"balance", true, nil, nil,
+				"balance", "balance", true, nil, nil,
 			},
 			"send": {
-				"send", false, []Argument{
+				"send", "send", false, []Argument{
 					{"amount", Uint256, false},
 				}, nil,
 			},
@@ -102,8 +101,7 @@ func TestReader(t *testing.T) {
 func TestTestNumbers(t *testing.T) {
 	abi, err := JSON(strings.NewReader(jsondata2))
 	if err != nil {
-		t.Error(err)
+		t.Fatal(err)
-		t.FailNow()
 	}
 
 	if _, err := abi.Pack("balance"); err != nil {
@@ -140,8 +138,7 @@ func TestTestNumbers(t *testing.T) {
 func TestTestString(t *testing.T) {
 	abi, err := JSON(strings.NewReader(jsondata2))
 	if err != nil {
-		t.Error(err)
+		t.Fatal(err)
-		t.FailNow()
 	}
 
 	if _, err := abi.Pack("string", "hello world"); err != nil {
@@ -152,8 +149,7 @@ func TestTestString(t *testing.T) {
 func TestTestBool(t *testing.T) {
 	abi, err := JSON(strings.NewReader(jsondata2))
 	if err != nil {
-		t.Error(err)
+		t.Fatal(err)
-		t.FailNow()
 	}
 
 	if _, err := abi.Pack("bool", true); err != nil {
@@ -164,15 +160,12 @@ func TestTestBool(t *testing.T) {
 func TestTestSlice(t *testing.T) {
 	abi, err := JSON(strings.NewReader(jsondata2))
 	if err != nil {
-		t.Error(err)
+		t.Fatal(err)
-		t.FailNow()
 	}
-
 	slice := make([]uint64, 2)
 	if _, err := abi.Pack("uint64[2]", slice); err != nil {
 		t.Error(err)
 	}
-
 	if _, err := abi.Pack("uint64[]", slice); err != nil {
 		t.Error(err)
 	}
@@ -180,19 +173,19 @@ func TestTestSlice(t *testing.T) {
 
 func TestMethodSignature(t *testing.T) {
 	String, _ := NewType("string", nil)
-	m := Method{"foo", false, []Argument{{"bar", String, false}, {"baz", String, false}}, nil}
+	m := Method{"foo", "foo", false, []Argument{{"bar", String, false}, {"baz", String, false}}, nil}
 	exp := "foo(string,string)"
 	if m.Sig() != exp {
 		t.Error("signature mismatch", exp, "!=", m.Sig())
 	}
 
 	idexp := crypto.Keccak256([]byte(exp))[:4]
-	if !bytes.Equal(m.Id(), idexp) {
+	if !bytes.Equal(m.ID(), idexp) {
-		t.Errorf("expected ids to match %x != %x", m.Id(), idexp)
+		t.Errorf("expected ids to match %x != %x", m.ID(), idexp)
 	}
 
 	uintt, _ := NewType("uint256", nil)
-	m = Method{"foo", false, []Argument{{"bar", uintt, false}}, nil}
+	m = Method{"foo", "foo", false, []Argument{{"bar", uintt, false}}, nil}
 	exp = "foo(uint256)"
 	if m.Sig() != exp {
 		t.Error("signature mismatch", exp, "!=", m.Sig())
@@ -211,18 +204,40 @@ func TestMethodSignature(t *testing.T) {
 			{Name: "y", Type: "int256"},
 		}},
 	})
-	m = Method{"foo", false, []Argument{{"s", s, false}, {"bar", String, false}}, nil}
+	m = Method{"foo", "foo", false, []Argument{{"s", s, false}, {"bar", String, false}}, nil}
 	exp = "foo((int256,int256[],(int256,int256)[],(int256,int256)[2]),string)"
 	if m.Sig() != exp {
 		t.Error("signature mismatch", exp, "!=", m.Sig())
 	}
 }
 
+func TestOverloadedMethodSignature(t *testing.T) {
+	json := `[{"constant":true,"inputs":[{"name":"i","type":"uint256"},{"name":"j","type":"uint256"}],"name":"foo","outputs":[],"payable":false,"stateMutability":"pure","type":"function"},{"constant":true,"inputs":[{"name":"i","type":"uint256"}],"name":"foo","outputs":[],"payable":false,"stateMutability":"pure","type":"function"},{"anonymous":false,"inputs":[{"indexed":false,"name":"i","type":"uint256"}],"name":"bar","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"i","type":"uint256"},{"indexed":false,"name":"j","type":"uint256"}],"name":"bar","type":"event"}]`
+	abi, err := JSON(strings.NewReader(json))
+	if err != nil {
+		t.Fatal(err)
+	}
+	check := func(name string, expect string, method bool) {
+		if method {
+			if abi.Methods[name].Sig() != expect {
+				t.Fatalf("The signature of overloaded method mismatch, want %s, have %s", expect, abi.Methods[name].Sig())
+			}
+		} else {
+			if abi.Events[name].Sig() != expect {
+				t.Fatalf("The signature of overloaded event mismatch, want %s, have %s", expect, abi.Events[name].Sig())
+			}
+		}
+	}
+	check("foo", "foo(uint256,uint256)", true)
+	check("foo0", "foo(uint256)", true)
+	check("bar", "bar(uint256)", false)
+	check("bar0", "bar(uint256,uint256)", false)
+}
+
 func TestMultiPack(t *testing.T) {
 	abi, err := JSON(strings.NewReader(jsondata2))
 	if err != nil {
-		t.Error(err)
+		t.Fatal(err)
-		t.FailNow()
 	}
 
 	sig := crypto.Keccak256([]byte("bar(uint32,uint16)"))[:4]
@@ -232,10 +247,8 @@ func TestMultiPack(t *testing.T) {
 
 	packed, err := abi.Pack("bar", uint32(10), uint16(11))
 	if err != nil {
-		t.Error(err)
+		t.Fatal(err)
-		t.FailNow()
 	}
-
 	if !bytes.Equal(packed, sig) {
 		t.Errorf("expected %x got %x", sig, packed)
 	}
@@ -246,11 +259,11 @@ func ExampleJSON() {
 
 	abi, err := JSON(strings.NewReader(definition))
 	if err != nil {
-		log.Fatalln(err)
+		panic(err)
 	}
 	out, err := abi.Pack("isBar", common.HexToAddress("01"))
 	if err != nil {
-		log.Fatalln(err)
+		panic(err)
 	}
 
 	fmt.Printf("%x\n", out)
@@ -832,9 +845,6 @@ func TestUnpackIntoMapNamingConflict(t *testing.T) {
 	if err = abi.UnpackIntoMap(receivedMap, "received", data); err != nil {
 		t.Error("naming conflict between two events; no error expected")
 	}
-	if len(receivedMap) != 1 {
-		t.Error("naming conflict between two events; event defined latest in the abi expected to be used")
-	}
 
 	// Method and event have the same name
 	abiJSON = `[{"constant":false,"inputs":[{"name":"memo","type":"bytes"}],"name":"received","outputs":[],"payable":true,"stateMutability":"payable","type":"function"},{"anonymous":false,"inputs":[{"indexed":false,"name":"sender","type":"address"},{"indexed":false,"name":"amount","type":"uint256"},{"indexed":false,"name":"memo","type":"bytes"}],"name":"received","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"sender","type":"address"}],"name":"receivedAddr","type":"event"}]`
@@ -911,13 +921,13 @@ func TestABI_MethodById(t *testing.T) {
 	}
 	for name, m := range abi.Methods {
 		a := fmt.Sprintf("%v", m)
-		m2, err := abi.MethodById(m.Id())
+		m2, err := abi.MethodById(m.ID())
 		if err != nil {
 			t.Fatalf("Failed to look up ABI method: %v", err)
 		}
 		b := fmt.Sprintf("%v", m2)
 		if a != b {
-			t.Errorf("Method %v (id %v) not 'findable' by id in ABI", name, common.ToHex(m.Id()))
+			t.Errorf("Method %v (id %v) not 'findable' by id in ABI", name, common.ToHex(m.ID()))
 		}
 	}
 	// Also test empty
@@ -931,3 +941,113 @@ func TestABI_MethodById(t *testing.T) {
 		t.Errorf("Expected error, nil is short to decode data")
 	}
 }
+
+func TestABI_EventById(t *testing.T) {
+	tests := []struct {
+		name  string
+		json  string
+		event string
+	}{
+		{
+			name: "",
+			json: `[
+			{"type":"event","name":"received","anonymous":false,"inputs":[
+				{"indexed":false,"name":"sender","type":"address"},
+				{"indexed":false,"name":"amount","type":"uint256"},
+				{"indexed":false,"name":"memo","type":"bytes"}
+				]
+			}]`,
+			event: "received(address,uint256,bytes)",
+		}, {
+			name: "",
+			json: `[
+				{ "constant": true, "inputs": [], "name": "name", "outputs": [ { "name": "", "type": "string" } ], "payable": false, "stateMutability": "view", "type": "function" },
+				{ "constant": false, "inputs": [ { "name": "_spender", "type": "address" }, { "name": "_value", "type": "uint256" } ], "name": "approve", "outputs": [ { "name": "", "type": "bool" } ], "payable": false, "stateMutability": "nonpayable", "type": "function" },
+				{ "constant": true, "inputs": [], "name": "totalSupply", "outputs": [ { "name": "", "type": "uint256" } ], "payable": false, "stateMutability": "view", "type": "function" },
+				{ "constant": false, "inputs": [ { "name": "_from", "type": "address" }, { "name": "_to", "type": "address" }, { "name": "_value", "type": "uint256" } ], "name": "transferFrom", "outputs": [ { "name": "", "type": "bool" } ], "payable": false, "stateMutability": "nonpayable", "type": "function" },
+				{ "constant": true, "inputs": [], "name": "decimals", "outputs": [ { "name": "", "type": "uint8" } ], "payable": false, "stateMutability": "view", "type": "function" },
+				{ "constant": true, "inputs": [ { "name": "_owner", "type": "address" } ], "name": "balanceOf", "outputs": [ { "name": "balance", "type": "uint256" } ], "payable": false, "stateMutability": "view", "type": "function" },
+				{ "constant": true, "inputs": [], "name": "symbol", "outputs": [ { "name": "", "type": "string" } ], "payable": false, "stateMutability": "view", "type": "function" },
+				{ "constant": false, "inputs": [ { "name": "_to", "type": "address" }, { "name": "_value", "type": "uint256" } ], "name": "transfer", "outputs": [ { "name": "", "type": "bool" } ], "payable": false, "stateMutability": "nonpayable", "type": "function" },
+				{ "constant": true, "inputs": [ { "name": "_owner", "type": "address" }, { "name": "_spender", "type": "address" } ], "name": "allowance", "outputs": [ { "name": "", "type": "uint256" } ], "payable": false, "stateMutability": "view", "type": "function" },
+				{ "payable": true, "stateMutability": "payable", "type": "fallback" },
+				{ "anonymous": false, "inputs": [ { "indexed": true, "name": "owner", "type": "address" }, { "indexed": true, "name": "spender", "type": "address" }, { "indexed": false, "name": "value", "type": "uint256" } ], "name": "Approval", "type": "event" },
+				{ "anonymous": false, "inputs": [ { "indexed": true, "name": "from", "type": "address" }, { "indexed": true, "name": "to", "type": "address" }, { "indexed": false, "name": "value", "type": "uint256" } ], "name": "Transfer", "type": "event" }
+			]`,
+			event: "Transfer(address,address,uint256)",
+		},
+	}
+
+	for testnum, test := range tests {
+		abi, err := JSON(strings.NewReader(test.json))
+		if err != nil {
+			t.Error(err)
+		}
+
+		topic := test.event
+		topicID := crypto.Keccak256Hash([]byte(topic))
+
+		event, err := abi.EventByID(topicID)
+		if err != nil {
+			t.Fatalf("Failed to look up ABI method: %v, test #%d", err, testnum)
+		}
+		if event == nil {
+			t.Errorf("We should find a event for topic %s, test #%d", topicID.Hex(), testnum)
+		}
+
+		if event.ID() != topicID {
+			t.Errorf("Event id %s does not match topic %s, test #%d", event.ID().Hex(), topicID.Hex(), testnum)
+		}
+
+		unknowntopicID := crypto.Keccak256Hash([]byte("unknownEvent"))
+		unknownEvent, err := abi.EventByID(unknowntopicID)
+		if err == nil {
+			t.Errorf("EventByID should return an error if a topic is not found, test #%d", testnum)
+		}
+		if unknownEvent != nil {
+			t.Errorf("We should not find any event for topic %s, test #%d", unknowntopicID.Hex(), testnum)
+		}
+	}
+}
+
+func TestDuplicateMethodNames(t *testing.T) {
+	abiJSON := `[{"constant":false,"inputs":[{"name":"to","type":"address"},{"name":"value","type":"uint256"}],"name":"transfer","outputs":[{"name":"ok","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"to","type":"address"},{"name":"value","type":"uint256"},{"name":"data","type":"bytes"}],"name":"transfer","outputs":[{"name":"ok","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"to","type":"address"},{"name":"value","type":"uint256"},{"name":"data","type":"bytes"},{"name":"customFallback","type":"string"}],"name":"transfer","outputs":[{"name":"ok","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"}]`
+	contractAbi, err := JSON(strings.NewReader(abiJSON))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := contractAbi.Methods["transfer"]; !ok {
+		t.Fatalf("Could not find original method")
+	}
+	if _, ok := contractAbi.Methods["transfer0"]; !ok {
+		t.Fatalf("Could not find duplicate method")
+	}
+	if _, ok := contractAbi.Methods["transfer1"]; !ok {
+		t.Fatalf("Could not find duplicate method")
+	}
+	if _, ok := contractAbi.Methods["transfer2"]; ok {
+		t.Fatalf("Should not have found extra method")
+	}
+}
+
+// TestDoubleDuplicateMethodNames checks that if transfer0 already exists, there won't be a name
+// conflict and that the second transfer method will be renamed transfer1.
+func TestDoubleDuplicateMethodNames(t *testing.T) {
+	abiJSON := `[{"constant":false,"inputs":[{"name":"to","type":"address"},{"name":"value","type":"uint256"}],"name":"transfer","outputs":[{"name":"ok","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"to","type":"address"},{"name":"value","type":"uint256"},{"name":"data","type":"bytes"}],"name":"transfer0","outputs":[{"name":"ok","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"to","type":"address"},{"name":"value","type":"uint256"},{"name":"data","type":"bytes"},{"name":"customFallback","type":"string"}],"name":"transfer","outputs":[{"name":"ok","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"}]`
+	contractAbi, err := JSON(strings.NewReader(abiJSON))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := contractAbi.Methods["transfer"]; !ok {
+		t.Fatalf("Could not find original method")
+	}
+	if _, ok := contractAbi.Methods["transfer0"]; !ok {
+		t.Fatalf("Could not find duplicate method")
+	}
+	if _, ok := contractAbi.Methods["transfer1"]; !ok {
+		t.Fatalf("Could not find duplicate method")
+	}
+	if _, ok := contractAbi.Methods["transfer2"]; ok {
+		t.Fatalf("Should not have found extra method")
+	}
+}

accounts/abi/argument.go

@@ -119,11 +119,22 @@ func unpack(t *Type, dst interface{}, src interface{}) error {
 		dstVal = reflect.ValueOf(dst).Elem()
 		srcVal = reflect.ValueOf(src)
 	)
-
+	tuple, typ := false, t
-	if t.T != TupleTy && !((t.T == SliceTy || t.T == ArrayTy) && t.Elem.T == TupleTy) {
+	for {
+		if typ.T == SliceTy || typ.T == ArrayTy {
+			typ = typ.Elem
+			continue
+		}
+		tuple = typ.T == TupleTy
+		break
+	}
+	if !tuple {
 		return set(dstVal, srcVal)
 	}
 
+	// Dereferences interface or pointer wrapper
+	dstVal = indirectInterfaceOrPtr(dstVal)
+
 	switch t.T {
 	case TupleTy:
 		if dstVal.Kind() != reflect.Struct {
@@ -191,7 +202,7 @@ func (arguments Arguments) unpackAtomic(v interface{}, marshalledValues interfac
 	argument := arguments.NonIndexed()[0]
 	elem := reflect.ValueOf(v).Elem()
 
-	if elem.Kind() == reflect.Struct {
+	if elem.Kind() == reflect.Struct && argument.Type.T != TupleTy {
 		fieldmap, err := mapArgNamesToStructFields([]string{argument.Name}, elem)
 		if err != nil {
 			return err

accounts/abi/bind/auth.go

@@ -23,6 +23,7 @@ import (
 	"io/ioutil"
 
 	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/accounts/external"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -43,7 +44,7 @@ func NewTransactor(keyin io.Reader, passphrase string) (*TransactOpts, error) {
 	return NewKeyedTransactor(key.PrivateKey), nil
 }
 
-// NewKeystoreTransactor is a utility method to easily create a transaction signer from
+// NewKeyStoreTransactor is a utility method to easily create a transaction signer from
 // an decrypted key from a keystore
 func NewKeyStoreTransactor(keystore *keystore.KeyStore, account accounts.Account) (*TransactOpts, error) {
 	return &TransactOpts{
@@ -79,3 +80,17 @@ func NewKeyedTransactor(key *ecdsa.PrivateKey) *TransactOpts {
 		},
 	}
 }
+
+// NewClefTransactor is a utility method to easily create a transaction signer
+// with a clef backend.
+func NewClefTransactor(clef *external.ExternalSigner, account accounts.Account) *TransactOpts {
+	return &TransactOpts{
+		From: account.Address,
+		Signer: func(signer types.Signer, address common.Address, transaction *types.Transaction) (*types.Transaction, error) {
+			if address != account.Address {
+				return nil, errors.New("not authorized to sign this account")
+			}
+			return clef.SignTx(account, transaction, nil) // Clef enforces its own chain id
+		},
+	}
+}

accounts/abi/bind/backends/simulated.go

@@ -45,8 +45,10 @@ import (
 // This nil assignment ensures compile time that SimulatedBackend implements bind.ContractBackend.
 var _ bind.ContractBackend = (*SimulatedBackend)(nil)
 
-var errBlockNumberUnsupported = errors.New("SimulatedBackend cannot access blocks other than the latest block")
+var (
-var errGasEstimationFailed = errors.New("gas required exceeds allowance or always failing transaction")
+	errBlockNumberUnsupported = errors.New("simulatedBackend cannot access blocks other than the latest block")
+	errGasEstimationFailed    = errors.New("gas required exceeds allowance or always failing transaction")
+)
 
 // SimulatedBackend implements bind.ContractBackend, simulating a blockchain in
 // the background. Its main purpose is to allow easily testing contract bindings.
@@ -63,10 +65,9 @@ type SimulatedBackend struct {
 	config *params.ChainConfig
 }
 
-// NewSimulatedBackend creates a new binding backend using a simulated blockchain
+// NewSimulatedBackendWithDatabase creates a new binding backend based on the given database
-// for testing purposes.
+// and uses a simulated blockchain for testing purposes.
-func NewSimulatedBackend(alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend {
+func NewSimulatedBackendWithDatabase(database ethdb.Database, alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend {
-	database := rawdb.NewMemoryDatabase()
 	genesis := core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: gasLimit, Alloc: alloc}
 	genesis.MustCommit(database)
 	blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}, nil)
@@ -81,6 +82,18 @@ func NewSimulatedBackend(alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBac
 	return backend
 }
 
+// NewSimulatedBackend creates a new binding backend using a simulated blockchain
+// for testing purposes.
+func NewSimulatedBackend(alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend {
+	return NewSimulatedBackendWithDatabase(rawdb.NewMemoryDatabase(), alloc, gasLimit)
+}
+
+// Close terminates the underlying blockchain's update loop.
+func (b *SimulatedBackend) Close() error {
+	b.blockchain.Stop()
+	return nil
+}
+
 // Commit imports all the pending transactions as a single block and starts a
 // fresh new state.
 func (b *SimulatedBackend) Commit() {
@@ -316,7 +329,7 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
-	sender, err := types.Sender(types.HomesteadSigner{}, tx)
+	sender, err := types.Sender(types.NewEIP155Signer(b.config.ChainID), tx)
 	if err != nil {
 		panic(fmt.Errorf("invalid transaction: %v", err))
 	}
@@ -424,6 +437,11 @@ func (b *SimulatedBackend) AdjustTime(adjustment time.Duration) error {
 	return nil
 }
 
+// Blockchain returns the underlying blockchain.
+func (b *SimulatedBackend) Blockchain() *core.BlockChain {
+	return b.blockchain
+}
+
 // callmsg implements core.Message to allow passing it as a transaction simulator.
 type callmsg struct {
 	ethereum.CallMsg

accounts/abi/bind/backends/simulated_test.go

@@ -38,6 +38,7 @@ func TestSimulatedBackend(t *testing.T) {
 	genAlloc[auth.From] = core.GenesisAccount{Balance: big.NewInt(9223372036854775807)}
 
 	sim := backends.NewSimulatedBackend(genAlloc, gasLimit)
+	defer sim.Close()
 
 	// should return an error if the tx is not found
 	txHash := common.HexToHash("2")

accounts/abi/bind/base.go

@@ -252,7 +252,7 @@ func (c *BoundContract) FilterLogs(opts *FilterOpts, name string, query ...[]int
 		opts = new(FilterOpts)
 	}
 	// Append the event selector to the query parameters and construct the topic set
-	query = append([][]interface{}{{c.abi.Events[name].Id()}}, query...)
+	query = append([][]interface{}{{c.abi.Events[name].ID()}}, query...)
 
 	topics, err := makeTopics(query...)
 	if err != nil {
@@ -301,7 +301,7 @@ func (c *BoundContract) WatchLogs(opts *WatchOpts, name string, query ...[]inter
 		opts = new(WatchOpts)
 	}
 	// Append the event selector to the query parameters and construct the topic set
-	query = append([][]interface{}{{c.abi.Events[name].Id()}}, query...)
+	query = append([][]interface{}{{c.abi.Events[name].ID()}}, query...)
 
 	topics, err := makeTopics(query...)
 	if err != nil {

accounts/abi/bind/base_test.go

@@ -345,29 +345,3 @@ func TestUnpackIndexedBytesTyLogIntoMap(t *testing.T) {
 		t.Error("unpacked map does not match expected map")
 	}
 }
-
-func TestUnpackIntoMapNamingConflict(t *testing.T) {
-	hash := crypto.Keccak256Hash([]byte("testName"))
-	mockLog := types.Log{
-		Address: common.HexToAddress("0x0"),
-		Topics: []common.Hash{
-			common.HexToHash("0x0"),
-			hash,
-		},
-		Data:        hexutil.MustDecode(hexData),
-		BlockNumber: uint64(26),
-		TxHash:      common.HexToHash("0x0"),
-		TxIndex:     111,
-		BlockHash:   common.BytesToHash([]byte{1, 2, 3, 4, 5}),
-		Index:       7,
-		Removed:     false,
-	}
-
-	abiString := `[{"anonymous":false,"inputs":[{"indexed":true,"name":"name","type":"string"},{"indexed":false,"name":"sender","type":"address"},{"indexed":false,"name":"amount","type":"uint256"},{"indexed":false,"name":"memo","type":"bytes"}],"name":"received","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"sender","type":"address"}],"name":"received","type":"event"}]`
-	parsedAbi, _ := abi.JSON(strings.NewReader(abiString))
-	bc := bind.NewBoundContract(common.HexToAddress("0x0"), parsedAbi, nil, nil, nil)
-	receivedMap := make(map[string]interface{})
-	if err := bc.UnpackLogIntoMap(receivedMap, "received", mockLog); err == nil {
-		t.Error("naming conflict between two events; error expected")
-	}
-}

accounts/abi/bind/bind.go

@@ -22,6 +22,7 @@ package bind
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"go/format"
 	"regexp"
@@ -30,6 +31,7 @@ import (
 	"unicode"
 
 	"github.com/ethereum/go-ethereum/accounts/abi"
+	"github.com/ethereum/go-ethereum/log"
 )
 
 // Lang is a target programming language selector to generate bindings for.
@@ -45,10 +47,13 @@ const (
 // to be used as is in client code, but rather as an intermediate struct which
 // enforces compile time type safety and naming convention opposed to having to
 // manually maintain hard coded strings that break on runtime.
-func Bind(types []string, abis []string, bytecodes []string, pkg string, lang Lang) (string, error) {
+func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string]string, pkg string, lang Lang, libs map[string]string) (string, error) {
 	// Process each individual contract requested binding
 	contracts := make(map[string]*tmplContract)
 
+	// Map used to flag each encountered library as such
+	isLib := make(map[string]struct{})
+
 	for i := 0; i < len(types); i++ {
 		// Parse the actual ABI to generate the binding for
 		evmABI, err := abi.JSON(strings.NewReader(abis[i]))
@@ -63,11 +68,12 @@ func Bind(types []string, abis []string, bytecodes []string, pkg string, lang La
 			return r
 		}, abis[i])
 
-		// Extract the call and transact methods; events; and sort them alphabetically
+		// Extract the call and transact methods; events, struct definitions; and sort them alphabetically
 		var (
 			calls     = make(map[string]*tmplMethod)
 			transacts = make(map[string]*tmplMethod)
 			events    = make(map[string]*tmplEvent)
+			structs   = make(map[string]*tmplStruct)
 		)
 		for _, original := range evmABI.Methods {
 			// Normalize the method for capital cases and non-anonymous inputs/outputs
@@ -80,6 +86,9 @@ func Bind(types []string, abis []string, bytecodes []string, pkg string, lang La
 				if input.Name == "" {
 					normalized.Inputs[j].Name = fmt.Sprintf("arg%d", j)
 				}
+				if _, exist := structs[input.Type.String()]; input.Type.T == abi.TupleTy && !exist {
+					bindStructType[lang](input.Type, structs)
+				}
 			}
 			normalized.Outputs = make([]abi.Argument, len(original.Outputs))
 			copy(normalized.Outputs, original.Outputs)
@@ -87,6 +96,9 @@ func Bind(types []string, abis []string, bytecodes []string, pkg string, lang La
 				if output.Name != "" {
 					normalized.Outputs[j].Name = capitalise(output.Name)
 				}
+				if _, exist := structs[output.Type.String()]; output.Type.T == abi.TupleTy && !exist {
+					bindStructType[lang](output.Type, structs)
+				}
 			}
 			// Append the methods to the call or transact lists
 			if original.Const {
@@ -112,25 +124,61 @@ func Bind(types []string, abis []string, bytecodes []string, pkg string, lang La
 					if input.Name == "" {
 						normalized.Inputs[j].Name = fmt.Sprintf("arg%d", j)
 					}
+					if _, exist := structs[input.Type.String()]; input.Type.T == abi.TupleTy && !exist {
+						bindStructType[lang](input.Type, structs)
+					}
 				}
 			}
 			// Append the event to the accumulator list
 			events[original.Name] = &tmplEvent{Original: original, Normalized: normalized}
 		}
+
+		// There is no easy way to pass arbitrary java objects to the Go side.
+		if len(structs) > 0 && lang == LangJava {
+			return "", errors.New("java binding for tuple arguments is not supported yet")
+		}
+
 		contracts[types[i]] = &tmplContract{
 			Type:        capitalise(types[i]),
 			InputABI:    strings.Replace(strippedABI, "\"", "\\\"", -1),
-			InputBin:    strings.TrimSpace(bytecodes[i]),
+			InputBin:    strings.TrimPrefix(strings.TrimSpace(bytecodes[i]), "0x"),
 			Constructor: evmABI.Constructor,
 			Calls:       calls,
 			Transacts:   transacts,
 			Events:      events,
+			Libraries:   make(map[string]string),
+			Structs:     structs,
 		}
+		// Function 4-byte signatures are stored in the same sequence
+		// as types, if available.
+		if len(fsigs) > i {
+			contracts[types[i]].FuncSigs = fsigs[i]
+		}
+		// Parse library references.
+		for pattern, name := range libs {
+			matched, err := regexp.Match("__\\$"+pattern+"\\$__", []byte(contracts[types[i]].InputBin))
+			if err != nil {
+				log.Error("Could not search for pattern", "pattern", pattern, "contract", contracts[types[i]], "err", err)
+			}
+			if matched {
+				contracts[types[i]].Libraries[pattern] = name
+				// keep track that this type is a library
+				if _, ok := isLib[name]; !ok {
+					isLib[name] = struct{}{}
+				}
+			}
+		}
+	}
+	// Check if that type has already been identified as a library
+	for i := 0; i < len(types); i++ {
+		_, ok := isLib[types[i]]
+		contracts[types[i]].Library = ok
 	}
 	// Generate the contract template data content and render it
 	data := &tmplData{
 		Package:   pkg,
 		Contracts: contracts,
+		Libraries: libs,
 	}
 	buffer := new(bytes.Buffer)
 
@@ -138,6 +186,8 @@ func Bind(types []string, abis []string, bytecodes []string, pkg string, lang La
 		"bindtype":      bindType[lang],
 		"bindtopictype": bindTopicType[lang],
 		"namedtype":     namedType[lang],
+		"formatmethod":  formatMethod,
+		"formatevent":   formatEvent,
 		"capitalise":    capitalise,
 		"decapitalise":  decapitalise,
 	}
@@ -159,129 +209,67 @@ func Bind(types []string, abis []string, bytecodes []string, pkg string, lang La
 
 // bindType is a set of type binders that convert Solidity types to some supported
 // programming language types.
-var bindType = map[Lang]func(kind abi.Type) string{
+var bindType = map[Lang]func(kind abi.Type, structs map[string]*tmplStruct) string{
 	LangGo:   bindTypeGo,
 	LangJava: bindTypeJava,
 }
 
-// Helper function for the binding generators.
+// bindBasicTypeGo converts basic solidity types(except array, slice and tuple) to Go one.
-// It reads the unmatched characters after the inner type-match,
+func bindBasicTypeGo(kind abi.Type) string {
-//  (since the inner type is a prefix of the total type declaration),
+	switch kind.T {
-//  looks for valid arrays (possibly a dynamic one) wrapping the inner type,
+	case abi.AddressTy:
-//  and returns the sizes of these arrays.
+		return "common.Address"
-//
+	case abi.IntTy, abi.UintTy:
-// Returned array sizes are in the same order as solidity signatures; inner array size first.
+		parts := regexp.MustCompile(`(u)?int([0-9]*)`).FindStringSubmatch(kind.String())
-// Array sizes may also be "", indicating a dynamic array.
-func wrapArray(stringKind string, innerLen int, innerMapping string) (string, []string) {
-	remainder := stringKind[innerLen:]
-	//find all the sizes
-	matches := regexp.MustCompile(`\[(\d*)\]`).FindAllStringSubmatch(remainder, -1)
-	parts := make([]string, 0, len(matches))
-	for _, match := range matches {
-		//get group 1 from the regex match
-		parts = append(parts, match[1])
-	}
-	return innerMapping, parts
-}
-
-// Translates the array sizes to a Go-lang declaration of a (nested) array of the inner type.
-// Simply returns the inner type if arraySizes is empty.
-func arrayBindingGo(inner string, arraySizes []string) string {
-	out := ""
-	//prepend all array sizes, from outer (end arraySizes) to inner (start arraySizes)
-	for i := len(arraySizes) - 1; i >= 0; i-- {
-		out += "[" + arraySizes[i] + "]"
-	}
-	out += inner
-	return out
-}
-
-// bindTypeGo converts a Solidity type to a Go one. Since there is no clear mapping
-// from all Solidity types to Go ones (e.g. uint17), those that cannot be exactly
-// mapped will use an upscaled type (e.g. *big.Int).
-func bindTypeGo(kind abi.Type) string {
-	stringKind := kind.String()
-	innerLen, innerMapping := bindUnnestedTypeGo(stringKind)
-	return arrayBindingGo(wrapArray(stringKind, innerLen, innerMapping))
-}
-
-// The inner function of bindTypeGo, this finds the inner type of stringKind.
-// (Or just the type itself if it is not an array or slice)
-// The length of the matched part is returned, with the translated type.
-func bindUnnestedTypeGo(stringKind string) (int, string) {
-
-	switch {
-	case strings.HasPrefix(stringKind, "address"):
-		return len("address"), "common.Address"
-
-	case strings.HasPrefix(stringKind, "bytes"):
-		parts := regexp.MustCompile(`bytes([0-9]*)`).FindStringSubmatch(stringKind)
-		return len(parts[0]), fmt.Sprintf("[%s]byte", parts[1])
-
-	case strings.HasPrefix(stringKind, "int") || strings.HasPrefix(stringKind, "uint"):
-		parts := regexp.MustCompile(`(u)?int([0-9]*)`).FindStringSubmatch(stringKind)
 		switch parts[2] {
 		case "8", "16", "32", "64":
-			return len(parts[0]), fmt.Sprintf("%sint%s", parts[1], parts[2])
+			return fmt.Sprintf("%sint%s", parts[1], parts[2])
 		}
-		return len(parts[0]), "*big.Int"
+		return "*big.Int"
-
+	case abi.FixedBytesTy:
-	case strings.HasPrefix(stringKind, "bool"):
+		return fmt.Sprintf("[%d]byte", kind.Size)
-		return len("bool"), "bool"
+	case abi.BytesTy:
-
+		return "[]byte"
-	case strings.HasPrefix(stringKind, "string"):
+	case abi.FunctionTy:
-		return len("string"), "string"
+		return "[24]byte"
-
 	default:
-		return len(stringKind), stringKind
+		// string, bool types
+		return kind.String()
 	}
 }
 
-// Translates the array sizes to a Java declaration of a (nested) array of the inner type.
+// bindTypeGo converts solidity types to Go ones. Since there is no clear mapping
-// Simply returns the inner type if arraySizes is empty.
+// from all Solidity types to Go ones (e.g. uint17), those that cannot be exactly
-func arrayBindingJava(inner string, arraySizes []string) string {
-	// Java array type declarations do not include the length.
-	return inner + strings.Repeat("[]", len(arraySizes))
-}
-
-// bindTypeJava converts a Solidity type to a Java one. Since there is no clear mapping
-// from all Solidity types to Java ones (e.g. uint17), those that cannot be exactly
 // mapped will use an upscaled type (e.g. BigDecimal).
-func bindTypeJava(kind abi.Type) string {
+func bindTypeGo(kind abi.Type, structs map[string]*tmplStruct) string {
-	stringKind := kind.String()
+	switch kind.T {
-	innerLen, innerMapping := bindUnnestedTypeJava(stringKind)
+	case abi.TupleTy:
-	return arrayBindingJava(wrapArray(stringKind, innerLen, innerMapping))
+		return structs[kind.String()].Name
+	case abi.ArrayTy:
+		return fmt.Sprintf("[%d]", kind.Size) + bindTypeGo(*kind.Elem, structs)
+	case abi.SliceTy:
+		return "[]" + bindTypeGo(*kind.Elem, structs)
+	default:
+		return bindBasicTypeGo(kind)
+	}
 }
 
-// The inner function of bindTypeJava, this finds the inner type of stringKind.
+// bindBasicTypeJava converts basic solidity types(except array, slice and tuple) to Java one.
-// (Or just the type itself if it is not an array or slice)
+func bindBasicTypeJava(kind abi.Type) string {
-// The length of the matched part is returned, with the translated type.
+	switch kind.T {
-func bindUnnestedTypeJava(stringKind string) (int, string) {
+	case abi.AddressTy:
-
+		return "Address"
-	switch {
+	case abi.IntTy, abi.UintTy:
-	case strings.HasPrefix(stringKind, "address"):
+		// Note that uint and int (without digits) are also matched,
-		parts := regexp.MustCompile(`address(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
-		if len(parts) != 2 {
-			return len(stringKind), stringKind
-		}
-		if parts[1] == "" {
-			return len("address"), "Address"
-		}
-		return len(parts[0]), "Addresses"
-
-	case strings.HasPrefix(stringKind, "bytes"):
-		parts := regexp.MustCompile(`bytes([0-9]*)`).FindStringSubmatch(stringKind)
-		if len(parts) != 2 {
-			return len(stringKind), stringKind
-		}
-		return len(parts[0]), "byte[]"
-
-	case strings.HasPrefix(stringKind, "int") || strings.HasPrefix(stringKind, "uint"):
-		//Note that uint and int (without digits) are also matched,
 		// these are size 256, and will translate to BigInt (the default).
-		parts := regexp.MustCompile(`(u)?int([0-9]*)`).FindStringSubmatch(stringKind)
+		parts := regexp.MustCompile(`(u)?int([0-9]*)`).FindStringSubmatch(kind.String())
 		if len(parts) != 3 {
-			return len(stringKind), stringKind
+			return kind.String()
+		}
+		// All unsigned integers should be translated to BigInt since gomobile doesn't
+		// support them.
+		if parts[1] == "u" {
+			return "BigInt"
 		}
 
 		namedSize := map[string]string{
@@ -291,50 +279,146 @@ func bindUnnestedTypeJava(stringKind string) (int, string) {
 			"64": "long",
 		}[parts[2]]
 
-		//default to BigInt
+		// default to BigInt
 		if namedSize == "" {
 			namedSize = "BigInt"
 		}
-		return len(parts[0]), namedSize
+		return namedSize
-
+	case abi.FixedBytesTy, abi.BytesTy:
-	case strings.HasPrefix(stringKind, "bool"):
+		return "byte[]"
-		return len("bool"), "boolean"
+	case abi.BoolTy:
-
+		return "boolean"
-	case strings.HasPrefix(stringKind, "string"):
+	case abi.StringTy:
-		return len("string"), "String"
+		return "String"
-
+	case abi.FunctionTy:
+		return "byte[24]"
 	default:
-		return len(stringKind), stringKind
+		return kind.String()
+	}
+}
+
+// pluralizeJavaType explicitly converts multidimensional types to predefined
+// type in go side.
+func pluralizeJavaType(typ string) string {
+	switch typ {
+	case "boolean":
+		return "Bools"
+	case "String":
+		return "Strings"
+	case "Address":
+		return "Addresses"
+	case "byte[]":
+		return "Binaries"
+	case "BigInt":
+		return "BigInts"
+	}
+	return typ + "[]"
+}
+
+// bindTypeJava converts a Solidity type to a Java one. Since there is no clear mapping
+// from all Solidity types to Java ones (e.g. uint17), those that cannot be exactly
+// mapped will use an upscaled type (e.g. BigDecimal).
+func bindTypeJava(kind abi.Type, structs map[string]*tmplStruct) string {
+	switch kind.T {
+	case abi.TupleTy:
+		return structs[kind.String()].Name
+	case abi.ArrayTy, abi.SliceTy:
+		return pluralizeJavaType(bindTypeJava(*kind.Elem, structs))
+	default:
+		return bindBasicTypeJava(kind)
 	}
 }
 
 // bindTopicType is a set of type binders that convert Solidity types to some
 // supported programming language topic types.
-var bindTopicType = map[Lang]func(kind abi.Type) string{
+var bindTopicType = map[Lang]func(kind abi.Type, structs map[string]*tmplStruct) string{
 	LangGo:   bindTopicTypeGo,
 	LangJava: bindTopicTypeJava,
 }
 
-// bindTypeGo converts a Solidity topic type to a Go one. It is almost the same
+// bindTopicTypeGo converts a Solidity topic type to a Go one. It is almost the same
 // funcionality as for simple types, but dynamic types get converted to hashes.
-func bindTopicTypeGo(kind abi.Type) string {
+func bindTopicTypeGo(kind abi.Type, structs map[string]*tmplStruct) string {
-	bound := bindTypeGo(kind)
+	bound := bindTypeGo(kind, structs)
 	if bound == "string" || bound == "[]byte" {
 		bound = "common.Hash"
 	}
 	return bound
 }
 
-// bindTypeGo converts a Solidity topic type to a Java one. It is almost the same
+// bindTopicTypeJava converts a Solidity topic type to a Java one. It is almost the same
 // funcionality as for simple types, but dynamic types get converted to hashes.
-func bindTopicTypeJava(kind abi.Type) string {
+func bindTopicTypeJava(kind abi.Type, structs map[string]*tmplStruct) string {
-	bound := bindTypeJava(kind)
+	bound := bindTypeJava(kind, structs)
-	if bound == "String" || bound == "Bytes" {
+	if bound == "String" || bound == "byte[]" {
 		bound = "Hash"
 	}
 	return bound
 }
 
+// bindStructType is a set of type binders that convert Solidity tuple types to some supported
+// programming language struct definition.
+var bindStructType = map[Lang]func(kind abi.Type, structs map[string]*tmplStruct) string{
+	LangGo:   bindStructTypeGo,
+	LangJava: bindStructTypeJava,
+}
+
+// bindStructTypeGo converts a Solidity tuple type to a Go one and records the mapping
+// in the given map.
+// Notably, this function will resolve and record nested struct recursively.
+func bindStructTypeGo(kind abi.Type, structs map[string]*tmplStruct) string {
+	switch kind.T {
+	case abi.TupleTy:
+		if s, exist := structs[kind.String()]; exist {
+			return s.Name
+		}
+		var fields []*tmplField
+		for i, elem := range kind.TupleElems {
+			field := bindStructTypeGo(*elem, structs)
+			fields = append(fields, &tmplField{Type: field, Name: capitalise(kind.TupleRawNames[i]), SolKind: *elem})
+		}
+		name := fmt.Sprintf("Struct%d", len(structs))
+		structs[kind.String()] = &tmplStruct{
+			Name:   name,
+			Fields: fields,
+		}
+		return name
+	case abi.ArrayTy:
+		return fmt.Sprintf("[%d]", kind.Size) + bindStructTypeGo(*kind.Elem, structs)
+	case abi.SliceTy:
+		return "[]" + bindStructTypeGo(*kind.Elem, structs)
+	default:
+		return bindBasicTypeGo(kind)
+	}
+}
+
+// bindStructTypeJava converts a Solidity tuple type to a Java one and records the mapping
+// in the given map.
+// Notably, this function will resolve and record nested struct recursively.
+func bindStructTypeJava(kind abi.Type, structs map[string]*tmplStruct) string {
+	switch kind.T {
+	case abi.TupleTy:
+		if s, exist := structs[kind.String()]; exist {
+			return s.Name
+		}
+		var fields []*tmplField
+		for i, elem := range kind.TupleElems {
+			field := bindStructTypeJava(*elem, structs)
+			fields = append(fields, &tmplField{Type: field, Name: decapitalise(kind.TupleRawNames[i]), SolKind: *elem})
+		}
+		name := fmt.Sprintf("Class%d", len(structs))
+		structs[kind.String()] = &tmplStruct{
+			Name:   name,
+			Fields: fields,
+		}
+		return name
+	case abi.ArrayTy, abi.SliceTy:
+		return pluralizeJavaType(bindStructTypeJava(*kind.Elem, structs))
+	default:
+		return bindBasicTypeJava(kind)
+	}
+}
+
 // namedType is a set of functions that transform language specific types to
 // named versions that my be used inside method names.
 var namedType = map[Lang]func(string, abi.Type) string{
@@ -348,18 +432,8 @@ func namedTypeJava(javaKind string, solKind abi.Type) string {
 	switch javaKind {
 	case "byte[]":
 		return "Binary"
-	case "byte[][]":
-		return "Binaries"
-	case "string":
-		return "String"
-	case "string[]":
-		return "Strings"
 	case "boolean":
 		return "Bool"
-	case "boolean[]":
-		return "Bools"
-	case "BigInt[]":
-		return "BigInts"
 	default:
 		parts := regexp.MustCompile(`(u)?int([0-9]*)(\[[0-9]*\])?`).FindStringSubmatch(solKind.String())
 		if len(parts) != 4 {
@@ -422,3 +496,63 @@ func structured(args abi.Arguments) bool {
 	}
 	return true
 }
+
+// resolveArgName converts a raw argument representation into a user friendly format.
+func resolveArgName(arg abi.Argument, structs map[string]*tmplStruct) string {
+	var (
+		prefix   string
+		embedded string
+		typ      = &arg.Type
+	)
+loop:
+	for {
+		switch typ.T {
+		case abi.SliceTy:
+			prefix += "[]"
+		case abi.ArrayTy:
+			prefix += fmt.Sprintf("[%d]", typ.Size)
+		default:
+			embedded = typ.String()
+			break loop
+		}
+		typ = typ.Elem
+	}
+	if s, exist := structs[embedded]; exist {
+		return prefix + s.Name
+	} else {
+		return arg.Type.String()
+	}
+}
+
+// formatMethod transforms raw method representation into a user friendly one.
+func formatMethod(method abi.Method, structs map[string]*tmplStruct) string {
+	inputs := make([]string, len(method.Inputs))
+	for i, input := range method.Inputs {
+		inputs[i] = fmt.Sprintf("%v %v", resolveArgName(input, structs), input.Name)
+	}
+	outputs := make([]string, len(method.Outputs))
+	for i, output := range method.Outputs {
+		outputs[i] = resolveArgName(output, structs)
+		if len(output.Name) > 0 {
+			outputs[i] += fmt.Sprintf(" %v", output.Name)
+		}
+	}
+	constant := ""
+	if method.Const {
+		constant = "constant "
+	}
+	return fmt.Sprintf("function %v(%v) %sreturns(%v)", method.RawName, strings.Join(inputs, ", "), constant, strings.Join(outputs, ", "))
+}
+
+// formatEvent transforms raw event representation into a user friendly one.
+func formatEvent(event abi.Event, structs map[string]*tmplStruct) string {
+	inputs := make([]string, len(event.Inputs))
+	for i, input := range event.Inputs {
+		if input.Indexed {
+			inputs[i] = fmt.Sprintf("%v indexed %v", resolveArgName(input, structs), input.Name)
+		} else {
+			inputs[i] = fmt.Sprintf("%v %v", resolveArgName(input, structs), input.Name)
+		}
+	}
+	return fmt.Sprintf("event %v(%v)", event.RawName, strings.Join(inputs, ", "))
+}

accounts/abi/bind/template.go

@@ -22,6 +22,7 @@ import "github.com/ethereum/go-ethereum/accounts/abi"
 type tmplData struct {
 	Package   string                   // Name of the package to place the generated file in
 	Contracts map[string]*tmplContract // List of contracts to generate into this file
+	Libraries map[string]string        // Map the bytecode's link pattern to the library name
 }
 
 // tmplContract contains the data needed to generate an individual contract binding.
@@ -29,10 +30,14 @@ type tmplContract struct {
 	Type        string                 // Type name of the main contract binding
 	InputABI    string                 // JSON ABI used as the input to generate the binding from
 	InputBin    string                 // Optional EVM bytecode used to denetare deploy code from
+	FuncSigs    map[string]string      // Optional map: string signature -> 4-byte signature
 	Constructor abi.Method             // Contract constructor for deploy parametrization
 	Calls       map[string]*tmplMethod // Contract calls that only read state data
 	Transacts   map[string]*tmplMethod // Contract calls that write state data
 	Events      map[string]*tmplEvent  // Contract events accessors
+	Libraries   map[string]string      // Same as tmplData, but filtered to only keep what the contract needs
+	Structs     map[string]*tmplStruct // Contract struct type definitions
+	Library     bool
 }
 
 // tmplMethod is a wrapper around an abi.Method that contains a few preprocessed
@@ -49,6 +54,21 @@ type tmplEvent struct {
 	Normalized abi.Event // Normalized version of the parsed fields
 }
 
+// tmplField is a wrapper around a struct field with binding language
+// struct type definition and relative filed name.
+type tmplField struct {
+	Type    string   // Field type representation depends on target binding language
+	Name    string   // Field name converted from the raw user-defined field name
+	SolKind abi.Type // Raw abi type information
+}
+
+// tmplStruct is a wrapper around an abi.tuple contains a auto-generated
+// struct name.
+type tmplStruct struct {
+	Name   string       // Auto-generated struct name(We can't obtain the raw struct name through abi)
+	Fields []*tmplField // Struct fields definition depends on the binding language.
+}
+
 // tmplSource is language to template mapping containing all the supported
 // programming languages the package can generate to.
 var tmplSource = map[Lang]string{
@@ -89,19 +109,32 @@ var (
 )
 
 {{range $contract := .Contracts}}
+	{{$structs := $contract.Structs}}
 	// {{.Type}}ABI is the input ABI used to generate the binding from.
 	const {{.Type}}ABI = "{{.InputABI}}"
 
+	{{if $contract.FuncSigs}}
+		// {{.Type}}FuncSigs maps the 4-byte function signature to its string representation.
+		var {{.Type}}FuncSigs = map[string]string{
+			{{range $strsig, $binsig := .FuncSigs}}"{{$binsig}}": "{{$strsig}}",
+			{{end}}
+		}
+	{{end}}
+
 	{{if .InputBin}}
 		// {{.Type}}Bin is the compiled bytecode used for deploying new contracts.
-		const {{.Type}}Bin = ` + "`" + `{{.InputBin}}` + "`" + `
+		var {{.Type}}Bin = "0x{{.InputBin}}"
 
 		// Deploy{{.Type}} deploys a new Ethereum contract, binding an instance of {{.Type}} to it.
-		func Deploy{{.Type}}(auth *bind.TransactOpts, backend bind.ContractBackend {{range .Constructor.Inputs}}, {{.Name}} {{bindtype .Type}}{{end}}) (common.Address, *types.Transaction, *{{.Type}}, error) {
+		func Deploy{{.Type}}(auth *bind.TransactOpts, backend bind.ContractBackend {{range .Constructor.Inputs}}, {{.Name}} {{bindtype .Type $structs}}{{end}}) (common.Address, *types.Transaction, *{{.Type}}, error) {
 		  parsed, err := abi.JSON(strings.NewReader({{.Type}}ABI))
 		  if err != nil {
 		    return common.Address{}, nil, nil, err
 		  }
+		  {{range $pattern, $name := .Libraries}}
+			{{decapitalise $name}}Addr, _, _, _ := Deploy{{capitalise $name}}(auth, backend)
+			{{$contract.Type}}Bin = strings.Replace({{$contract.Type}}Bin, "__${{$pattern}}$__", {{decapitalise $name}}Addr.String()[2:], -1)
+		  {{end}}
 		  address, tx, contract, err := bind.DeployContract(auth, parsed, common.FromHex({{.Type}}Bin), backend {{range .Constructor.Inputs}}, {{.Name}}{{end}})
 		  if err != nil {
 		    return common.Address{}, nil, nil, err
@@ -114,7 +147,7 @@ var (
 	type {{.Type}} struct {
 	  {{.Type}}Caller     // Read-only binding to the contract
 	  {{.Type}}Transactor // Write-only binding to the contract
-		{{.Type}}Filterer   // Log filterer for contract events
+	  {{.Type}}Filterer   // Log filterer for contract events
 	}
 
 	// {{.Type}}Caller is an auto generated read-only Go binding around an Ethereum contract.
@@ -252,16 +285,24 @@ var (
 		return _{{$contract.Type}}.Contract.contract.Transact(opts, method, params...)
 	}
 
+	{{range .Structs}}
+		// {{.Name}} is an auto generated low-level Go binding around an user-defined struct.
+		type {{.Name}} struct {
+		{{range $field := .Fields}}
+		{{$field.Name}} {{$field.Type}}{{end}}
+		}
+	{{end}}
+
 	{{range .Calls}}
-		// {{.Normalized.Name}} is a free data retrieval call binding the contract method 0x{{printf "%x" .Original.Id}}.
+		// {{.Normalized.Name}} is a free data retrieval call binding the contract method 0x{{printf "%x" .Original.ID}}.
 		//
-		// Solidity: {{.Original.String}}
+		// Solidity: {{formatmethod .Original $structs}}
-		func (_{{$contract.Type}} *{{$contract.Type}}Caller) {{.Normalized.Name}}(opts *bind.CallOpts {{range .Normalized.Inputs}}, {{.Name}} {{bindtype .Type}} {{end}}) ({{if .Structured}}struct{ {{range .Normalized.Outputs}}{{.Name}} {{bindtype .Type}};{{end}} },{{else}}{{range .Normalized.Outputs}}{{bindtype .Type}},{{end}}{{end}} error) {
+		func (_{{$contract.Type}} *{{$contract.Type}}Caller) {{.Normalized.Name}}(opts *bind.CallOpts {{range .Normalized.Inputs}}, {{.Name}} {{bindtype .Type $structs}} {{end}}) ({{if .Structured}}struct{ {{range .Normalized.Outputs}}{{.Name}} {{bindtype .Type $structs}};{{end}} },{{else}}{{range .Normalized.Outputs}}{{bindtype .Type $structs}},{{end}}{{end}} error) {
 			{{if .Structured}}ret := new(struct{
-				{{range .Normalized.Outputs}}{{.Name}} {{bindtype .Type}}
+				{{range .Normalized.Outputs}}{{.Name}} {{bindtype .Type $structs}}
 				{{end}}
 			}){{else}}var (
-				{{range $i, $_ := .Normalized.Outputs}}ret{{$i}} = new({{bindtype .Type}})
+				{{range $i, $_ := .Normalized.Outputs}}ret{{$i}} = new({{bindtype .Type $structs}})
 				{{end}}
 			){{end}}
 			out := {{if .Structured}}ret{{else}}{{if eq (len .Normalized.Outputs) 1}}ret0{{else}}&[]interface{}{
@@ -272,40 +313,40 @@ var (
 			return {{if .Structured}}*ret,{{else}}{{range $i, $_ := .Normalized.Outputs}}*ret{{$i}},{{end}}{{end}} err
 		}
 
-		// {{.Normalized.Name}} is a free data retrieval call binding the contract method 0x{{printf "%x" .Original.Id}}.
+		// {{.Normalized.Name}} is a free data retrieval call binding the contract method 0x{{printf "%x" .Original.ID}}.
 		//
-		// Solidity: {{.Original.String}}
+		// Solidity: {{formatmethod .Original $structs}}
-		func (_{{$contract.Type}} *{{$contract.Type}}Session) {{.Normalized.Name}}({{range $i, $_ := .Normalized.Inputs}}{{if ne $i 0}},{{end}} {{.Name}} {{bindtype .Type}} {{end}}) ({{if .Structured}}struct{ {{range .Normalized.Outputs}}{{.Name}} {{bindtype .Type}};{{end}} }, {{else}} {{range .Normalized.Outputs}}{{bindtype .Type}},{{end}} {{end}} error) {
+		func (_{{$contract.Type}} *{{$contract.Type}}Session) {{.Normalized.Name}}({{range $i, $_ := .Normalized.Inputs}}{{if ne $i 0}},{{end}} {{.Name}} {{bindtype .Type $structs}} {{end}}) ({{if .Structured}}struct{ {{range .Normalized.Outputs}}{{.Name}} {{bindtype .Type $structs}};{{end}} }, {{else}} {{range .Normalized.Outputs}}{{bindtype .Type $structs}},{{end}} {{end}} error) {
 		  return _{{$contract.Type}}.Contract.{{.Normalized.Name}}(&_{{$contract.Type}}.CallOpts {{range .Normalized.Inputs}}, {{.Name}}{{end}})
 		}
 
-		// {{.Normalized.Name}} is a free data retrieval call binding the contract method 0x{{printf "%x" .Original.Id}}.
+		// {{.Normalized.Name}} is a free data retrieval call binding the contract method 0x{{printf "%x" .Original.ID}}.
 		//
-		// Solidity: {{.Original.String}}
+		// Solidity: {{formatmethod .Original $structs}}
-		func (_{{$contract.Type}} *{{$contract.Type}}CallerSession) {{.Normalized.Name}}({{range $i, $_ := .Normalized.Inputs}}{{if ne $i 0}},{{end}} {{.Name}} {{bindtype .Type}} {{end}}) ({{if .Structured}}struct{ {{range .Normalized.Outputs}}{{.Name}} {{bindtype .Type}};{{end}} }, {{else}} {{range .Normalized.Outputs}}{{bindtype .Type}},{{end}} {{end}} error) {
+		func (_{{$contract.Type}} *{{$contract.Type}}CallerSession) {{.Normalized.Name}}({{range $i, $_ := .Normalized.Inputs}}{{if ne $i 0}},{{end}} {{.Name}} {{bindtype .Type $structs}} {{end}}) ({{if .Structured}}struct{ {{range .Normalized.Outputs}}{{.Name}} {{bindtype .Type $structs}};{{end}} }, {{else}} {{range .Normalized.Outputs}}{{bindtype .Type $structs}},{{end}} {{end}} error) {
 		  return _{{$contract.Type}}.Contract.{{.Normalized.Name}}(&_{{$contract.Type}}.CallOpts {{range .Normalized.Inputs}}, {{.Name}}{{end}})
 		}
 	{{end}}
 
 	{{range .Transacts}}
-		// {{.Normalized.Name}} is a paid mutator transaction binding the contract method 0x{{printf "%x" .Original.Id}}.
+		// {{.Normalized.Name}} is a paid mutator transaction binding the contract method 0x{{printf "%x" .Original.ID}}.
 		//
-		// Solidity: {{.Original.String}}
+		// Solidity: {{formatmethod .Original $structs}}
-		func (_{{$contract.Type}} *{{$contract.Type}}Transactor) {{.Normalized.Name}}(opts *bind.TransactOpts {{range .Normalized.Inputs}}, {{.Name}} {{bindtype .Type}} {{end}}) (*types.Transaction, error) {
+		func (_{{$contract.Type}} *{{$contract.Type}}Transactor) {{.Normalized.Name}}(opts *bind.TransactOpts {{range .Normalized.Inputs}}, {{.Name}} {{bindtype .Type $structs}} {{end}}) (*types.Transaction, error) {
 			return _{{$contract.Type}}.contract.Transact(opts, "{{.Original.Name}}" {{range .Normalized.Inputs}}, {{.Name}}{{end}})
 		}
 
-		// {{.Normalized.Name}} is a paid mutator transaction binding the contract method 0x{{printf "%x" .Original.Id}}.
+		// {{.Normalized.Name}} is a paid mutator transaction binding the contract method 0x{{printf "%x" .Original.ID}}.
 		//
-		// Solidity: {{.Original.String}}
+		// Solidity: {{formatmethod .Original $structs}}
-		func (_{{$contract.Type}} *{{$contract.Type}}Session) {{.Normalized.Name}}({{range $i, $_ := .Normalized.Inputs}}{{if ne $i 0}},{{end}} {{.Name}} {{bindtype .Type}} {{end}}) (*types.Transaction, error) {
+		func (_{{$contract.Type}} *{{$contract.Type}}Session) {{.Normalized.Name}}({{range $i, $_ := .Normalized.Inputs}}{{if ne $i 0}},{{end}} {{.Name}} {{bindtype .Type $structs}} {{end}}) (*types.Transaction, error) {
 		  return _{{$contract.Type}}.Contract.{{.Normalized.Name}}(&_{{$contract.Type}}.TransactOpts {{range $i, $_ := .Normalized.Inputs}}, {{.Name}}{{end}})
 		}
 
-		// {{.Normalized.Name}} is a paid mutator transaction binding the contract method 0x{{printf "%x" .Original.Id}}.
+		// {{.Normalized.Name}} is a paid mutator transaction binding the contract method 0x{{printf "%x" .Original.ID}}.
 		//
-		// Solidity: {{.Original.String}}
+		// Solidity: {{formatmethod .Original $structs}}
-		func (_{{$contract.Type}} *{{$contract.Type}}TransactorSession) {{.Normalized.Name}}({{range $i, $_ := .Normalized.Inputs}}{{if ne $i 0}},{{end}} {{.Name}} {{bindtype .Type}} {{end}}) (*types.Transaction, error) {
+		func (_{{$contract.Type}} *{{$contract.Type}}TransactorSession) {{.Normalized.Name}}({{range $i, $_ := .Normalized.Inputs}}{{if ne $i 0}},{{end}} {{.Name}} {{bindtype .Type $structs}} {{end}}) (*types.Transaction, error) {
 		  return _{{$contract.Type}}.Contract.{{.Normalized.Name}}(&_{{$contract.Type}}.TransactOpts {{range $i, $_ := .Normalized.Inputs}}, {{.Name}}{{end}})
 		}
 	{{end}}
@@ -377,14 +418,14 @@ var (
 
 		// {{$contract.Type}}{{.Normalized.Name}} represents a {{.Normalized.Name}} event raised by the {{$contract.Type}} contract.
 		type {{$contract.Type}}{{.Normalized.Name}} struct { {{range .Normalized.Inputs}}
-			{{capitalise .Name}} {{if .Indexed}}{{bindtopictype .Type}}{{else}}{{bindtype .Type}}{{end}}; {{end}}
+			{{capitalise .Name}} {{if .Indexed}}{{bindtopictype .Type $structs}}{{else}}{{bindtype .Type $structs}}{{end}}; {{end}}
 			Raw types.Log // Blockchain specific contextual infos
 		}
 
-		// Filter{{.Normalized.Name}} is a free log retrieval operation binding the contract event 0x{{printf "%x" .Original.Id}}.
+		// Filter{{.Normalized.Name}} is a free log retrieval operation binding the contract event 0x{{printf "%x" .Original.ID}}.
 		//
-		// Solidity: {{.Original.String}}
+		// Solidity: {{formatevent .Original $structs}}
- 		func (_{{$contract.Type}} *{{$contract.Type}}Filterer) Filter{{.Normalized.Name}}(opts *bind.FilterOpts{{range .Normalized.Inputs}}{{if .Indexed}}, {{.Name}} []{{bindtype .Type}}{{end}}{{end}}) (*{{$contract.Type}}{{.Normalized.Name}}Iterator, error) {
+ 		func (_{{$contract.Type}} *{{$contract.Type}}Filterer) Filter{{.Normalized.Name}}(opts *bind.FilterOpts{{range .Normalized.Inputs}}{{if .Indexed}}, {{.Name}} []{{bindtype .Type $structs}}{{end}}{{end}}) (*{{$contract.Type}}{{.Normalized.Name}}Iterator, error) {
 			{{range .Normalized.Inputs}}
 			{{if .Indexed}}var {{.Name}}Rule []interface{}
 			for _, {{.Name}}Item := range {{.Name}} {
@@ -398,10 +439,10 @@ var (
 			return &{{$contract.Type}}{{.Normalized.Name}}Iterator{contract: _{{$contract.Type}}.contract, event: "{{.Original.Name}}", logs: logs, sub: sub}, nil
  		}
 
-		// Watch{{.Normalized.Name}} is a free log subscription operation binding the contract event 0x{{printf "%x" .Original.Id}}.
+		// Watch{{.Normalized.Name}} is a free log subscription operation binding the contract event 0x{{printf "%x" .Original.ID}}.
 		//
-		// Solidity: {{.Original.String}}
+		// Solidity: {{formatevent .Original $structs}}
-		func (_{{$contract.Type}} *{{$contract.Type}}Filterer) Watch{{.Normalized.Name}}(opts *bind.WatchOpts, sink chan<- *{{$contract.Type}}{{.Normalized.Name}}{{range .Normalized.Inputs}}{{if .Indexed}}, {{.Name}} []{{bindtype .Type}}{{end}}{{end}}) (event.Subscription, error) {
+		func (_{{$contract.Type}} *{{$contract.Type}}Filterer) Watch{{.Normalized.Name}}(opts *bind.WatchOpts, sink chan<- *{{$contract.Type}}{{.Normalized.Name}}{{range .Normalized.Inputs}}{{if .Indexed}}, {{.Name}} []{{bindtype .Type $structs}}{{end}}{{end}}) (event.Subscription, error) {
 			{{range .Normalized.Inputs}}
 			{{if .Indexed}}var {{.Name}}Rule []interface{}
 			for _, {{.Name}}Item := range {{.Name}} {
@@ -439,6 +480,18 @@ var (
 				}
 			}), nil
 		}
+
+		// Parse{{.Normalized.Name}} is a log parse operation binding the contract event 0x{{printf "%x" .Original.ID}}.
+		//
+		// Solidity: {{.Original.String}}
+		func (_{{$contract.Type}} *{{$contract.Type}}Filterer) Parse{{.Normalized.Name}}(log types.Log) (*{{$contract.Type}}{{.Normalized.Name}}, error) {
+			event := new({{$contract.Type}}{{.Normalized.Name}})
+			if err := _{{$contract.Type}}.contract.UnpackLog(event, "{{.Original.Name}}", log); err != nil {
+				return nil, err
+			}
+			return event, nil
+		}
+
  	{{end}}
 {{end}}
 `
@@ -452,95 +505,112 @@ const tmplSourceJava = `
 package {{.Package}};
 
 import org.ethereum.geth.*;
-import org.ethereum.geth.internal.*;
+import java.util.*;
 
 {{range $contract := .Contracts}}
-	public class {{.Type}} {
+{{$structs := $contract.Structs}}
-		// ABI is the input ABI used to generate the binding from.
+{{if not .Library}}public {{end}}class {{.Type}} {
-		public final static String ABI = "{{.InputABI}}";
+	// ABI is the input ABI used to generate the binding from.
-
+	public final static String ABI = "{{.InputABI}}";
-		{{if .InputBin}}
+	{{if $contract.FuncSigs}}
-			// BYTECODE is the compiled bytecode used for deploying new contracts.
+		// {{.Type}}FuncSigs maps the 4-byte function signature to its string representation.
-			public final static byte[] BYTECODE = "{{.InputBin}}".getBytes();
+		public final static Map<String, String> {{.Type}}FuncSigs;
-
+		static {
-			// deploy deploys a new Ethereum contract, binding an instance of {{.Type}} to it.
+			Hashtable<String, String> temp = new Hashtable<String, String>();
-			public static {{.Type}} deploy(TransactOpts auth, EthereumClient client{{range .Constructor.Inputs}}, {{bindtype .Type}} {{.Name}}{{end}}) throws Exception {
+			{{range $strsig, $binsig := .FuncSigs}}temp.put("{{$binsig}}", "{{$strsig}}");
-				Interfaces args = Geth.newInterfaces({{(len .Constructor.Inputs)}});
-				{{range $index, $element := .Constructor.Inputs}}
-				  args.set({{$index}}, Geth.newInterface()); args.get({{$index}}).set{{namedtype (bindtype .Type) .Type}}({{.Name}});
-				{{end}}
-				return new {{.Type}}(Geth.deployContract(auth, ABI, BYTECODE, client, args));
-			}
-
-			// Internal constructor used by contract deployment.
-			private {{.Type}}(BoundContract deployment) {
-				this.Address  = deployment.getAddress();
-				this.Deployer = deployment.getDeployer();
-				this.Contract = deployment;
-			}
-		{{end}}
-
-		// Ethereum address where this contract is located at.
-		public final Address Address;
-
-		// Ethereum transaction in which this contract was deployed (if known!).
-		public final Transaction Deployer;
-
-		// Contract instance bound to a blockchain address.
-		private final BoundContract Contract;
-
-		// Creates a new instance of {{.Type}}, bound to a specific deployed contract.
-		public {{.Type}}(Address address, EthereumClient client) throws Exception {
-			this(Geth.bindContract(address, ABI, client));
-		}
-
-		{{range .Calls}}
-			{{if gt (len .Normalized.Outputs) 1}}
-			// {{capitalise .Normalized.Name}}Results is the output of a call to {{.Normalized.Name}}.
-			public class {{capitalise .Normalized.Name}}Results {
-				{{range $index, $item := .Normalized.Outputs}}public {{bindtype .Type}} {{if ne .Name ""}}{{.Name}}{{else}}Return{{$index}}{{end}};
-				{{end}}
-			}
 			{{end}}
+			{{.Type}}FuncSigs = Collections.unmodifiableMap(temp);
+		}
+	{{end}}
+	{{if .InputBin}}
+	// BYTECODE is the compiled bytecode used for deploying new contracts.
+	public final static String BYTECODE = "0x{{.InputBin}}";
 
-			// {{.Normalized.Name}} is a free data retrieval call binding the contract method 0x{{printf "%x" .Original.Id}}.
+	// deploy deploys a new Ethereum contract, binding an instance of {{.Type}} to it.
-			//
+	public static {{.Type}} deploy(TransactOpts auth, EthereumClient client{{range .Constructor.Inputs}}, {{bindtype .Type $structs}} {{.Name}}{{end}}) throws Exception {
-			// Solidity: {{.Original.String}}
+		Interfaces args = Geth.newInterfaces({{(len .Constructor.Inputs)}});
-			public {{if gt (len .Normalized.Outputs) 1}}{{capitalise .Normalized.Name}}Results{{else}}{{range .Normalized.Outputs}}{{bindtype .Type}}{{end}}{{end}} {{.Normalized.Name}}(CallOpts opts{{range .Normalized.Inputs}}, {{bindtype .Type}} {{.Name}}{{end}}) throws Exception {
+		String bytecode = BYTECODE;
-				Interfaces args = Geth.newInterfaces({{(len .Normalized.Inputs)}});
+		{{if .Libraries}}
-				{{range $index, $item := .Normalized.Inputs}}args.set({{$index}}, Geth.newInterface()); args.get({{$index}}).set{{namedtype (bindtype .Type) .Type}}({{.Name}});
-				{{end}}
 
-				Interfaces results = Geth.newInterfaces({{(len .Normalized.Outputs)}});
+		// "link" contract to dependent libraries by deploying them first.
-				{{range $index, $item := .Normalized.Outputs}}Interface result{{$index}} = Geth.newInterface(); result{{$index}}.setDefault{{namedtype (bindtype .Type) .Type}}(); results.set({{$index}}, result{{$index}});
+		{{range $pattern, $name := .Libraries}}
-				{{end}}
+		{{capitalise $name}} {{decapitalise $name}}Inst = {{capitalise $name}}.deploy(auth, client);
-
+		bytecode = bytecode.replace("__${{$pattern}}$__", {{decapitalise $name}}Inst.Address.getHex().substring(2));
-				if (opts == null) {
-					opts = Geth.newCallOpts();
-				}
-				this.Contract.call(opts, results, "{{.Original.Name}}", args);
-				{{if gt (len .Normalized.Outputs) 1}}
-					{{capitalise .Normalized.Name}}Results result = new {{capitalise .Normalized.Name}}Results();
-					{{range $index, $item := .Normalized.Outputs}}result.{{if ne .Name ""}}{{.Name}}{{else}}Return{{$index}}{{end}} = results.get({{$index}}).get{{namedtype (bindtype .Type) .Type}}();
-					{{end}}
-					return result;
-				{{else}}{{range .Normalized.Outputs}}return results.get(0).get{{namedtype (bindtype .Type) .Type}}();{{end}}
-				{{end}}
-			}
 		{{end}}
+		{{end}}
+		{{range $index, $element := .Constructor.Inputs}}Interface arg{{$index}} = Geth.newInterface();arg{{$index}}.set{{namedtype (bindtype .Type $structs) .Type}}({{.Name}});args.set({{$index}},arg{{$index}});
+		{{end}}
+		return new {{.Type}}(Geth.deployContract(auth, ABI, Geth.decodeFromHex(bytecode), client, args));
+	}
 
-		{{range .Transacts}}
+	// Internal constructor used by contract deployment.
-			// {{.Normalized.Name}} is a paid mutator transaction binding the contract method 0x{{printf "%x" .Original.Id}}.
+	private {{.Type}}(BoundContract deployment) {
-			//
+		this.Address  = deployment.getAddress();
-			// Solidity: {{.Original.String}}
+		this.Deployer = deployment.getDeployer();
-			public Transaction {{.Normalized.Name}}(TransactOpts opts{{range .Normalized.Inputs}}, {{bindtype .Type}} {{.Name}}{{end}}) throws Exception {
+		this.Contract = deployment;
-				Interfaces args = Geth.newInterfaces({{(len .Normalized.Inputs)}});
+	}
-				{{range $index, $item := .Normalized.Inputs}}args.set({{$index}}, Geth.newInterface()); args.get({{$index}}).set{{namedtype (bindtype .Type) .Type}}({{.Name}});
+	{{end}}
-				{{end}}
 
-				return this.Contract.transact(opts, "{{.Original.Name}}"	, args);
+	// Ethereum address where this contract is located at.
-			}
+	public final Address Address;
+
+	// Ethereum transaction in which this contract was deployed (if known!).
+	public final Transaction Deployer;
+
+	// Contract instance bound to a blockchain address.
+	private final BoundContract Contract;
+
+	// Creates a new instance of {{.Type}}, bound to a specific deployed contract.
+	public {{.Type}}(Address address, EthereumClient client) throws Exception {
+		this(Geth.bindContract(address, ABI, client));
+	}
+
+	{{range .Calls}}
+	{{if gt (len .Normalized.Outputs) 1}}
+	// {{capitalise .Normalized.Name}}Results is the output of a call to {{.Normalized.Name}}.
+	public class {{capitalise .Normalized.Name}}Results {
+		{{range $index, $item := .Normalized.Outputs}}public {{bindtype .Type $structs}} {{if ne .Name ""}}{{.Name}}{{else}}Return{{$index}}{{end}};
 		{{end}}
 	}
+	{{end}}
+
+	// {{.Normalized.Name}} is a free data retrieval call binding the contract method 0x{{printf "%x" .Original.ID}}.
+	//
+	// Solidity: {{.Original.String}}
+	public {{if gt (len .Normalized.Outputs) 1}}{{capitalise .Normalized.Name}}Results{{else}}{{range .Normalized.Outputs}}{{bindtype .Type $structs}}{{end}}{{end}} {{.Normalized.Name}}(CallOpts opts{{range .Normalized.Inputs}}, {{bindtype .Type $structs}} {{.Name}}{{end}}) throws Exception {
+		Interfaces args = Geth.newInterfaces({{(len .Normalized.Inputs)}});
+		{{range $index, $item := .Normalized.Inputs}}Interface arg{{$index}} = Geth.newInterface();arg{{$index}}.set{{namedtype (bindtype .Type $structs) .Type}}({{.Name}});args.set({{$index}},arg{{$index}});
+		{{end}}
+
+		Interfaces results = Geth.newInterfaces({{(len .Normalized.Outputs)}});
+		{{range $index, $item := .Normalized.Outputs}}Interface result{{$index}} = Geth.newInterface(); result{{$index}}.setDefault{{namedtype (bindtype .Type $structs) .Type}}(); results.set({{$index}}, result{{$index}});
+		{{end}}
+
+		if (opts == null) {
+			opts = Geth.newCallOpts();
+		}
+		this.Contract.call(opts, results, "{{.Original.Name}}", args);
+		{{if gt (len .Normalized.Outputs) 1}}
+			{{capitalise .Normalized.Name}}Results result = new {{capitalise .Normalized.Name}}Results();
+			{{range $index, $item := .Normalized.Outputs}}result.{{if ne .Name ""}}{{.Name}}{{else}}Return{{$index}}{{end}} = results.get({{$index}}).get{{namedtype (bindtype .Type $structs) .Type}}();
+			{{end}}
+			return result;
+		{{else}}{{range .Normalized.Outputs}}return results.get(0).get{{namedtype (bindtype .Type $structs) .Type}}();{{end}}
+		{{end}}
+	}
+	{{end}}
+
+	{{range .Transacts}}
+	// {{.Normalized.Name}} is a paid mutator transaction binding the contract method 0x{{printf "%x" .Original.ID}}.
+	//
+	// Solidity: {{.Original.String}}
+	public Transaction {{.Normalized.Name}}(TransactOpts opts{{range .Normalized.Inputs}}, {{bindtype .Type $structs}} {{.Name}}{{end}}) throws Exception {
+		Interfaces args = Geth.newInterfaces({{(len .Normalized.Inputs)}});
+		{{range $index, $item := .Normalized.Inputs}}Interface arg{{$index}} = Geth.newInterface();arg{{$index}}.set{{namedtype (bindtype .Type $structs) .Type}}({{.Name}});args.set({{$index}},arg{{$index}});
+		{{end}}
+		return this.Contract.transact(opts, "{{.Original.Name}}"	, args);
+	}
+	{{end}}
+}
 {{end}}
 `

accounts/abi/bind/topics_test.go

@@ -1,4 +1,4 @@
-// Copyright 2018 The go-ethereum Authors
+// Copyright 2019 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify

accounts/abi/bind/util_test.go

@@ -56,8 +56,10 @@ func TestWaitDeployed(t *testing.T) {
 		backend := backends.NewSimulatedBackend(
 			core.GenesisAlloc{
 				crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000)},
-			}, 10000000,
+			},
+			10000000,
 		)
+		defer backend.Close()
 
 		// Create the transaction.
 		tx := types.NewContractCreation(0, big.NewInt(0), test.gas, big.NewInt(1), common.FromHex(test.code))

accounts/abi/event.go

@@ -28,7 +28,18 @@ import (
 // holds type information (inputs) about the yielded output. Anonymous events
 // don't get the signature canonical representation as the first LOG topic.
 type Event struct {
-	Name      string
+	// Name is the event name used for internal representation. It's derived from
+	// the raw name and a suffix will be added in the case of a event overload.
+	//
+	// e.g.
+	// There are two events have same name:
+	// * foo(int,int)
+	// * foo(uint,uint)
+	// The event name of the first one wll be resolved as foo while the second one
+	// will be resolved as foo0.
+	Name string
+	// RawName is the raw event name parsed from ABI.
+	RawName   string
 	Anonymous bool
 	Inputs    Arguments
 }
@@ -41,17 +52,26 @@ func (e Event) String() string {
 			inputs[i] = fmt.Sprintf("%v indexed %v", input.Type, input.Name)
 		}
 	}
-	return fmt.Sprintf("event %v(%v)", e.Name, strings.Join(inputs, ", "))
+	return fmt.Sprintf("event %v(%v)", e.RawName, strings.Join(inputs, ", "))
 }
 
-// Id returns the canonical representation of the event's signature used by the
+// Sig returns the event string signature according to the ABI spec.
-// abi definition to identify event names and types.
+//
-func (e Event) Id() common.Hash {
+// Example
+//
+//     event foo(uint32 a, int b) = "foo(uint32,int256)"
+//
+// Please note that "int" is substitute for its canonical representation "int256"
+func (e Event) Sig() string {
 	types := make([]string, len(e.Inputs))
-	i := 0
+	for i, input := range e.Inputs {
-	for _, input := range e.Inputs {
 		types[i] = input.Type.String()
-		i++
 	}
-	return common.BytesToHash(crypto.Keccak256([]byte(fmt.Sprintf("%v(%v)", e.Name, strings.Join(types, ",")))))
+	return fmt.Sprintf("%v(%v)", e.RawName, strings.Join(types, ","))
+}
+
+// ID returns the canonical representation of the event's signature used by the
+// abi definition to identify event names and types.
+func (e Event) ID() common.Hash {
+	return common.BytesToHash(crypto.Keccak256([]byte(e.Sig())))
 }

accounts/abi/event_test.go

@@ -104,8 +104,8 @@ func TestEventId(t *testing.T) {
 		}
 
 		for name, event := range abi.Events {
-			if event.Id() != test.expectations[name] {
+			if event.ID() != test.expectations[name] {
-				t.Errorf("expected id to be %x, got %x", test.expectations[name], event.Id())
+				t.Errorf("expected id to be %x, got %x", test.expectations[name], event.ID())
 			}
 		}
 	}

accounts/abi/method.go

@@ -32,7 +32,18 @@ import (
 // be flagged `false`.
 // Input specifies the required input parameters for this gives method.
 type Method struct {
-	Name    string
+	// Name is the method name used for internal representation. It's derived from
+	// the raw name and a suffix will be added in the case of a function overload.
+	//
+	// e.g.
+	// There are two functions have same name:
+	// * foo(int,int)
+	// * foo(uint,uint)
+	// The method name of the first one will be resolved as foo while the second one
+	// will be resolved as foo0.
+	Name string
+	// RawName is the raw method name parsed from ABI.
+	RawName string
 	Const   bool
 	Inputs  Arguments
 	Outputs Arguments
@@ -42,7 +53,7 @@ type Method struct {
 //
 // Example
 //
-//     function foo(uint32 a, int b)    =    "foo(uint32,int256)"
+//     function foo(uint32 a, int b) = "foo(uint32,int256)"
 //
 // Please note that "int" is substitute for its canonical representation "int256"
 func (method Method) Sig() string {
@@ -50,7 +61,7 @@ func (method Method) Sig() string {
 	for i, input := range method.Inputs {
 		types[i] = input.Type.String()
 	}
-	return fmt.Sprintf("%v(%v)", method.Name, strings.Join(types, ","))
+	return fmt.Sprintf("%v(%v)", method.RawName, strings.Join(types, ","))
 }
 
 func (method Method) String() string {
@@ -69,9 +80,11 @@ func (method Method) String() string {
 	if method.Const {
 		constant = "constant "
 	}
-	return fmt.Sprintf("function %v(%v) %sreturns(%v)", method.Name, strings.Join(inputs, ", "), constant, strings.Join(outputs, ", "))
+	return fmt.Sprintf("function %v(%v) %sreturns(%v)", method.RawName, strings.Join(inputs, ", "), constant, strings.Join(outputs, ", "))
 }
 
-func (method Method) Id() []byte {
+// ID returns the canonical representation of the method's signature used by the
+// abi definition to identify method names and types.
+func (method Method) ID() []byte {
 	return crypto.Keccak256([]byte(method.Sig()))[:4]
 }

accounts/abi/method_test.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2018 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
@@ -23,9 +23,13 @@ import (
 
 const methoddata = `
 [
-	{ "type" : "function", "name" : "balance", "constant" : true },
+	{"type": "function", "name": "balance", "constant": true },
-	{ "type" : "function", "name" : "send", "constant" : false, "inputs" : [ { "name" : "amount", "type" : "uint256" } ] },
+	{"type": "function", "name": "send", "constant": false, "inputs": [{ "name": "amount", "type": "uint256" }]},
-	{ "type" : "function", "name" : "transfer", "constant" : false, "inputs" : [ { "name" : "from", "type" : "address" }, { "name" : "to", "type" : "address" }, { "name" : "value", "type" : "uint256" } ], "outputs" : [ { "name" : "success", "type" : "bool" } ]  }
+	{"type": "function", "name": "transfer", "constant": false, "inputs": [{"name": "from", "type": "address"}, {"name": "to", "type": "address"}, {"name": "value", "type": "uint256"}], "outputs": [{"name": "success", "type": "bool"}]},
+	{"constant":false,"inputs":[{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"a","type":"tuple"}],"name":"tuple","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},
+	{"constant":false,"inputs":[{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"a","type":"tuple[]"}],"name":"tupleSlice","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},
+	{"constant":false,"inputs":[{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"a","type":"tuple[5]"}],"name":"tupleArray","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},
+	{"constant":false,"inputs":[{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"a","type":"tuple[5][]"}],"name":"complexTuple","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"}
 ]`
 
 func TestMethodString(t *testing.T) {
@@ -45,6 +49,22 @@ func TestMethodString(t *testing.T) {
 			method:      "transfer",
 			expectation: "function transfer(address from, address to, uint256 value) returns(bool success)",
 		},
+		{
+			method:      "tuple",
+			expectation: "function tuple((uint256,uint256) a) returns()",
+		},
+		{
+			method:      "tupleArray",
+			expectation: "function tupleArray((uint256,uint256)[5] a) returns()",
+		},
+		{
+			method:      "tupleSlice",
+			expectation: "function tupleSlice((uint256,uint256)[] a) returns()",
+		},
+		{
+			method:      "complexTuple",
+			expectation: "function complexTuple((uint256,uint256)[5][] a) returns()",
+		},
 	}
 
 	abi, err := JSON(strings.NewReader(methoddata))
@@ -59,3 +79,50 @@ func TestMethodString(t *testing.T) {
 		}
 	}
 }
+
+func TestMethodSig(t *testing.T) {
+	var cases = []struct {
+		method string
+		expect string
+	}{
+		{
+			method: "balance",
+			expect: "balance()",
+		},
+		{
+			method: "send",
+			expect: "send(uint256)",
+		},
+		{
+			method: "transfer",
+			expect: "transfer(address,address,uint256)",
+		},
+		{
+			method: "tuple",
+			expect: "tuple((uint256,uint256))",
+		},
+		{
+			method: "tupleArray",
+			expect: "tupleArray((uint256,uint256)[5])",
+		},
+		{
+			method: "tupleSlice",
+			expect: "tupleSlice((uint256,uint256)[])",
+		},
+		{
+			method: "complexTuple",
+			expect: "complexTuple((uint256,uint256)[5][])",
+		},
+	}
+	abi, err := JSON(strings.NewReader(methoddata))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, test := range cases {
+		got := abi.Methods[test.method].Sig()
+		if got != test.expect {
+			t.Errorf("expected string to be %s, got %s", test.expect, got)
+		}
+	}
+}

accounts/abi/pack_test.go

@@ -634,7 +634,7 @@ func TestMethodPack(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	sig := abi.Methods["slice"].Id()
+	sig := abi.Methods["slice"].ID()
 	sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...)
 	sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...)
 
@@ -648,7 +648,7 @@ func TestMethodPack(t *testing.T) {
 	}
 
 	var addrA, addrB = common.Address{1}, common.Address{2}
-	sig = abi.Methods["sliceAddress"].Id()
+	sig = abi.Methods["sliceAddress"].ID()
 	sig = append(sig, common.LeftPadBytes([]byte{32}, 32)...)
 	sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...)
 	sig = append(sig, common.LeftPadBytes(addrA[:], 32)...)
@@ -663,7 +663,7 @@ func TestMethodPack(t *testing.T) {
 	}
 
 	var addrC, addrD = common.Address{3}, common.Address{4}
-	sig = abi.Methods["sliceMultiAddress"].Id()
+	sig = abi.Methods["sliceMultiAddress"].ID()
 	sig = append(sig, common.LeftPadBytes([]byte{64}, 32)...)
 	sig = append(sig, common.LeftPadBytes([]byte{160}, 32)...)
 	sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...)
@@ -681,7 +681,7 @@ func TestMethodPack(t *testing.T) {
 		t.Errorf("expected %x got %x", sig, packed)
 	}
 
-	sig = abi.Methods["slice256"].Id()
+	sig = abi.Methods["slice256"].ID()
 	sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...)
 	sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...)
 
@@ -695,7 +695,7 @@ func TestMethodPack(t *testing.T) {
 	}
 
 	a := [2][2]*big.Int{{big.NewInt(1), big.NewInt(1)}, {big.NewInt(2), big.NewInt(0)}}
-	sig = abi.Methods["nestedArray"].Id()
+	sig = abi.Methods["nestedArray"].ID()
 	sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...)
 	sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...)
 	sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...)
@@ -712,7 +712,7 @@ func TestMethodPack(t *testing.T) {
 		t.Errorf("expected %x got %x", sig, packed)
 	}
 
-	sig = abi.Methods["nestedArray2"].Id()
+	sig = abi.Methods["nestedArray2"].ID()
 	sig = append(sig, common.LeftPadBytes([]byte{0x20}, 32)...)
 	sig = append(sig, common.LeftPadBytes([]byte{0x40}, 32)...)
 	sig = append(sig, common.LeftPadBytes([]byte{0x80}, 32)...)
@@ -728,7 +728,7 @@ func TestMethodPack(t *testing.T) {
 		t.Errorf("expected %x got %x", sig, packed)
 	}
 
-	sig = abi.Methods["nestedSlice"].Id()
+	sig = abi.Methods["nestedSlice"].ID()
 	sig = append(sig, common.LeftPadBytes([]byte{0x20}, 32)...)
 	sig = append(sig, common.LeftPadBytes([]byte{0x02}, 32)...)
 	sig = append(sig, common.LeftPadBytes([]byte{0x40}, 32)...)

accounts/abi/reflect.go

@@ -31,6 +31,14 @@ func indirect(v reflect.Value) reflect.Value {
 	return v
 }
 
+// indirectInterfaceOrPtr recursively dereferences the value until value is not interface.
+func indirectInterfaceOrPtr(v reflect.Value) reflect.Value {
+	if (v.Kind() == reflect.Interface || v.Kind() == reflect.Ptr) && v.Elem().IsValid() {
+		return indirect(v.Elem())
+	}
+	return v
+}
+
 // reflectIntKind returns the reflect using the given size and
 // unsignedness.
 func reflectIntKindAndType(unsigned bool, size int) (reflect.Kind, reflect.Type) {

accounts/abi/type.go

@@ -68,7 +68,6 @@ func NewType(t string, components []ArgumentMarshaling) (typ Type, err error) {
 	if strings.Count(t, "[") != strings.Count(t, "]") {
 		return Type{}, fmt.Errorf("invalid arg type in abi")
 	}
-
 	typ.stringKind = t
 
 	// if there are brackets, get ready to go into slice/array mode and
@@ -92,9 +91,7 @@ func NewType(t string, components []ArgumentMarshaling) (typ Type, err error) {
 			typ.Kind = reflect.Slice
 			typ.Elem = &embeddedType
 			typ.Type = reflect.SliceOf(embeddedType.Type)
-			if embeddedType.T == TupleTy {
+			typ.stringKind = embeddedType.stringKind + sliced
-				typ.stringKind = embeddedType.stringKind + sliced
-			}
 		} else if len(intz) == 1 {
 			// is a array
 			typ.T = ArrayTy
@@ -105,9 +102,7 @@ func NewType(t string, components []ArgumentMarshaling) (typ Type, err error) {
 				return Type{}, fmt.Errorf("abi: error parsing variable size: %v", err)
 			}
 			typ.Type = reflect.ArrayOf(typ.Size, embeddedType.Type)
-			if embeddedType.T == TupleTy {
+			typ.stringKind = embeddedType.stringKind + sliced
-				typ.stringKind = embeddedType.stringKind + sliced
-			}
 		} else {
 			return Type{}, fmt.Errorf("invalid formatting of array type")
 		}

accounts/abi/unpack_test.go

@@ -965,25 +965,21 @@ func TestUnpackTuple(t *testing.T) {
 	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) // ret[a] = 1
 	buff.Write(common.Hex2Bytes("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")) // ret[b] = -1
 
+	// If the result is single tuple, use struct as return value container directly.
 	v := struct {
-		Ret struct {
-			A *big.Int
-			B *big.Int
-		}
-	}{Ret: struct {
 		A *big.Int
 		B *big.Int
-	}{new(big.Int), new(big.Int)}}
+	}{new(big.Int), new(big.Int)}
 
 	err = abi.Unpack(&v, "tuple", buff.Bytes())
 	if err != nil {
 		t.Error(err)
 	} else {
-		if v.Ret.A.Cmp(big.NewInt(1)) != 0 {
+		if v.A.Cmp(big.NewInt(1)) != 0 {
-			t.Errorf("unexpected value unpacked: want %x, got %x", 1, v.Ret.A)
+			t.Errorf("unexpected value unpacked: want %x, got %x", 1, v.A)
 		}
-		if v.Ret.B.Cmp(big.NewInt(-1)) != 0 {
+		if v.B.Cmp(big.NewInt(-1)) != 0 {
-			t.Errorf("unexpected value unpacked: want %x, got %x", v.Ret.B, -1)
+			t.Errorf("unexpected value unpacked: want %x, got %x", v.B, -1)
 		}
 	}
 

accounts/accounts_test.go

@@ -1,4 +1,4 @@
-// Copyright 2019 The go-ethereum Authors
+// Copyright 2015 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify

accounts/errors.go

@@ -35,7 +35,7 @@ var ErrNotSupported = errors.New("not supported")
 
 // ErrInvalidPassphrase is returned when a decryption operation receives a bad
 // passphrase.
-var ErrInvalidPassphrase = errors.New("invalid passphrase")
+var ErrInvalidPassphrase = errors.New("invalid password")
 
 // ErrWalletAlreadyOpen is returned if a wallet is attempted to be opened the
 // second time.

accounts/external/backend.go

@@ -1,18 +1,18 @@
-// Copyright 2018 The go-ethereum Authors
+// Copyright 2019 The go-ethereum Authors
-// This file is part of go-ethereum.
+// This file is part of the go-ethereum library.
 //
-// go-ethereum is free software: you can redistribute it and/or modify
+// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
+// it under the terms of the GNU Lesser General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
-// go-ethereum is distributed in the hope that it will be useful,
+// The go-ethereum library is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
+// GNU Lesser General Public License for more details.
 //
-// You should have received a copy of the GNU General Public License
+// You should have received a copy of the GNU Lesser General Public License
-// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
 package external
 
@@ -182,18 +182,21 @@ func (api *ExternalSigner) SignText(account accounts.Account, text []byte) ([]by
 
 func (api *ExternalSigner) SignTx(account accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
 	res := ethapi.SignTransactionResult{}
-	to := common.NewMixedcaseAddress(*tx.To())
 	data := hexutil.Bytes(tx.Data())
+	var to *common.MixedcaseAddress
+	if tx.To() != nil {
+		t := common.NewMixedcaseAddress(*tx.To())
+		to = &t
+	}
 	args := &core.SendTxArgs{
 		Data:     &data,
 		Nonce:    hexutil.Uint64(tx.Nonce()),
 		Value:    hexutil.Big(*tx.Value()),
 		Gas:      hexutil.Uint64(tx.Gas()),
 		GasPrice: hexutil.Big(*tx.GasPrice()),
-		To:       &to,
+		To:       to,
 		From:     common.NewMixedcaseAddress(account.Address),
 	}
-
 	if err := api.client.Call(&res, "account_signTransaction", args); err != nil {
 		return nil, err
 	}
@@ -201,14 +204,14 @@ func (api *ExternalSigner) SignTx(account accounts.Account, tx *types.Transactio
 }
 
 func (api *ExternalSigner) SignTextWithPassphrase(account accounts.Account, passphrase string, text []byte) ([]byte, error) {
-	return []byte{}, fmt.Errorf("passphrase-operations not supported on external signers")
+	return []byte{}, fmt.Errorf("password-operations not supported on external signers")
 }
 
 func (api *ExternalSigner) SignTxWithPassphrase(account accounts.Account, passphrase string, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
-	return nil, fmt.Errorf("passphrase-operations not supported on external signers")
+	return nil, fmt.Errorf("password-operations not supported on external signers")
 }
 func (api *ExternalSigner) SignDataWithPassphrase(account accounts.Account, passphrase, mimeType string, data []byte) ([]byte, error) {
-	return nil, fmt.Errorf("passphrase-operations not supported on external signers")
+	return nil, fmt.Errorf("password-operations not supported on external signers")
 }
 
 func (api *ExternalSigner) listAccounts() ([]common.Address, error) {

accounts/keystore/keystore.go

@@ -43,7 +43,7 @@ import (
 var (
 	ErrLocked  = accounts.NewAuthNeededError("password or unlock")
 	ErrNoMatch = errors.New("no key for given address or file")
-	ErrDecrypt = errors.New("could not decrypt key with given passphrase")
+	ErrDecrypt = errors.New("could not decrypt key with given password")
 )
 
 // KeyStoreType is the reflect type of a keystore backend.

accounts/keystore/keystore_test.go

@@ -379,9 +379,9 @@ func tmpKeyStore(t *testing.T, encrypted bool) (string, *KeyStore) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	new := NewPlaintextKeyStore
+	newKs := NewPlaintextKeyStore
 	if encrypted {
-		new = func(kd string) *KeyStore { return NewKeyStore(kd, veryLightScryptN, veryLightScryptP) }
+		newKs = func(kd string) *KeyStore { return NewKeyStore(kd, veryLightScryptN, veryLightScryptP) }
 	}
-	return d, new(d)
+	return d, newKs(d)
 }

accounts/keystore/plain_test.go

@@ -96,7 +96,7 @@ func TestKeyStorePassphraseDecryptionFail(t *testing.T) {
 		t.Fatal(err)
 	}
 	if _, err = ks.GetKey(k1.Address, account.URL.Path, "bar"); err != ErrDecrypt {
-		t.Fatalf("wrong error for invalid passphrase\ngot %q\nwant %q", err, ErrDecrypt)
+		t.Fatalf("wrong error for invalid password\ngot %q\nwant %q", err, ErrDecrypt)
 	}
 }
 

accounts/keystore/testdata/keystore/README

@@ -1,5 +1,5 @@
 This directory contains accounts for testing.
-The passphrase that unlocks them is "foobar".
+The password that unlocks them is "foobar".
 
 The "good" key files which are supposed to be loadable are:
 

accounts/manager.go

@@ -21,6 +21,7 @@ import (
 	"sort"
 	"sync"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/event"
 )
 
@@ -162,6 +163,20 @@ func (am *Manager) Wallet(url string) (Wallet, error) {
 	return nil, ErrUnknownWallet
 }
 
+// Accounts returns all account addresses of all wallets within the account manager
+func (am *Manager) Accounts() []common.Address {
+	am.lock.RLock()
+	defer am.lock.RUnlock()
+
+	addresses := make([]common.Address, 0) // return [] instead of nil if empty
+	for _, wallet := range am.wallets {
+		for _, account := range wallet.Accounts() {
+			addresses = append(addresses, account.Address)
+		}
+	}
+	return addresses
+}
+
 // Find attempts to locate the wallet corresponding to a specific account. Since
 // accounts can be dynamically added to and removed from wallets, this method has
 // a linear runtime in the number of wallets.

accounts/url_test.go

@@ -1,4 +1,4 @@
-// Copyright 2017 The go-ethereum Authors
+// Copyright 2018 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify

accounts/usbwallet/ledger.go

@@ -32,6 +32,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
 )
@@ -341,7 +342,7 @@ func (w *ledgerDriver) ledgerSign(derivationPath []uint32, tx *types.Transaction
 		op = ledgerP1ContTransactionData
 	}
 	// Extract the Ethereum signature and do a sanity validation
-	if len(reply) != 65 {
+	if len(reply) != crypto.SignatureLength {
 		return common.Address{}, nil, errors.New("reply lacks signature")
 	}
 	signature := append(reply[1:], reply[0])

accounts/usbwallet/trezor/trezor.go

@@ -1,4 +1,4 @@
-// Copyright 2019 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify

accounts/usbwallet/wallet.go

@@ -479,7 +479,8 @@ func (w *wallet) Derive(path accounts.DerivationPath, pin bool) (accounts.Accoun
 
 	if _, ok := w.paths[address]; !ok {
 		w.accounts = append(w.accounts, account)
-		w.paths[address] = path
+		w.paths[address] = make(accounts.DerivationPath, len(path))
+		copy(w.paths[address], path)
 	}
 	return account, nil
 }

appveyor.yml

@@ -23,8 +23,8 @@ environment:
 install:
   - git submodule update --init
   - rmdir C:\go /s /q
-  - appveyor DownloadFile https://dl.google.com/go/go1.12.6.windows-%GETH_ARCH%.zip
+  - appveyor DownloadFile https://dl.google.com/go/go1.12.9.windows-%GETH_ARCH%.zip
-  - 7z x go1.12.6.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+  - 7z x go1.12.9.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
   - go version
   - gcc --version
 

build/ci.go

@@ -137,7 +137,8 @@ var (
 	// Note: yakkety is unsupported because it was officially deprecated on Launchpad.
 	// Note: zesty is unsupported because it was officially deprecated on Launchpad.
 	// Note: artful is unsupported because it was officially deprecated on Launchpad.
-	debDistros = []string{"trusty", "xenial", "bionic", "cosmic", "disco"}
+	// Note: cosmic is unsupported because it was officially deprecated on Launchpad.
+	debDistros = []string{"trusty", "xenial", "bionic", "disco", "eoan"}
 )
 
 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
@@ -899,7 +900,7 @@ func doXCodeFramework(cmdline []string) {
 	// Build the iOS XCode framework
 	build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind"))
 	build.MustRun(gomobileTool("init"))
-	bind := gomobileTool("bind", "-ldflags", "-s -w", "--target", "ios", "--tags", "ios", "-v", "github.com/ethereum/go-ethereum/mobile")
+	bind := gomobileTool("bind", "-ldflags", "-s -w", "--target", "ios", "-v", "github.com/ethereum/go-ethereum/mobile")
 
 	if *local {
 		// If we're building locally, use the build folder and stop afterwards

build/update-license.go

@@ -62,16 +62,22 @@ var (
 	skipPrefixes = []string{
 		// boring stuff
 		"vendor/", "tests/testdata/", "build/",
+
 		// don't relicense vendored sources
 		"cmd/internal/browser",
+		"common/bitutil/bitutil",
+		"common/prque/",
 		"consensus/ethash/xor.go",
 		"crypto/bn256/",
 		"crypto/ecies/",
-		"crypto/secp256k1/curve.go",
+		"graphql/graphiql.go",
-		"crypto/sha3/",
 		"internal/jsre/deps",
 		"log/",
-		"common/bitutil/bitutil",
+		"metrics/",
+		"signer/rules/deps",
+
+		// skip special licenses
+		"crypto/secp256k1", // Relicensed to BSD-3 via https://github.com/ethereum/go-ethereum/pull/17225
 	}
 
 	// paths with this prefix are licensed as GPL. all other files are LGPL.
@@ -144,6 +150,13 @@ func (i info) gpl() bool {
 	return false
 }
 
+// authors implements the sort.Interface for strings in case-insensitive mode.
+type authors []string
+
+func (as authors) Len() int           { return len(as) }
+func (as authors) Less(i, j int) bool { return strings.ToLower(as[i]) < strings.ToLower(as[j]) }
+func (as authors) Swap(i, j int)      { as[i], as[j] = as[j], as[i] }
+
 func main() {
 	var (
 		files = getFiles()
@@ -262,27 +275,32 @@ func mailmapLookup(authors []string) []string {
 }
 
 func writeAuthors(files []string) {
-	merge := make(map[string]bool)
+	var (
-	// Add authors that Git reports as contributorxs.
+		dedup = make(map[string]bool)
+		list  []string
+	)
+	// Add authors that Git reports as contributors.
 	// This is the primary source of author information.
 	for _, a := range gitAuthors(files) {
-		merge[a] = true
+		if la := strings.ToLower(a); !dedup[la] {
+			list = append(list, a)
+			dedup[la] = true
+		}
 	}
 	// Add existing authors from the file. This should ensure that we
 	// never lose authors, even if Git stops listing them. We can also
 	// add authors manually this way.
 	for _, a := range readAuthors() {
-		merge[a] = true
+		if la := strings.ToLower(a); !dedup[la] {
+			list = append(list, a)
+			dedup[la] = true
+		}
 	}
 	// Write sorted list of authors back to the file.
-	var result []string
+	sort.Sort(authors(list))
-	for a := range merge {
-		result = append(result, a)
-	}
-	sort.Strings(result)
 	content := new(bytes.Buffer)
 	content.WriteString(authorsFileHeader)
-	for _, a := range result {
+	for _, a := range list {
 		content.WriteString(a)
 		content.WriteString("\n")
 	}

cmd/abigen/main.go

@@ -1,4 +1,4 @@
-// Copyright 2019 The go-ethereum Authors
+// Copyright 2016 The go-ethereum Authors
 // This file is part of go-ethereum.
 //
 // go-ethereum is free software: you can redistribute it and/or modify
@@ -18,97 +18,199 @@ package main
 
 import (
 	"encoding/json"
-	"flag"
 	"fmt"
 	"io/ioutil"
 	"os"
 	"strings"
 
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common/compiler"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/log"
+	"gopkg.in/urfave/cli.v1"
+)
+
+const (
+	commandHelperTemplate = `{{.Name}}{{if .Subcommands}} command{{end}}{{if .Flags}} [command options]{{end}} [arguments...]
+{{if .Description}}{{.Description}}
+{{end}}{{if .Subcommands}}
+SUBCOMMANDS:
+	{{range .Subcommands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}}
+	{{end}}{{end}}{{if .Flags}}
+OPTIONS:
+{{range $.Flags}}{{"\t"}}{{.}}
+{{end}}
+{{end}}`
 )
 
 var (
-	abiFlag = flag.String("abi", "", "Path to the Ethereum contract ABI json to bind, - for STDIN")
+	// Git SHA1 commit hash of the release (set via linker flags)
-	binFlag = flag.String("bin", "", "Path to the Ethereum contract bytecode (generate deploy method)")
+	gitCommit = ""
-	typFlag = flag.String("type", "", "Struct name for the binding (default = package name)")
+	gitDate   = ""
 
-	solFlag  = flag.String("sol", "", "Path to the Ethereum contract Solidity source to build and bind")
+	app *cli.App
-	solcFlag = flag.String("solc", "solc", "Solidity compiler to use if source builds are requested")
-	excFlag  = flag.String("exc", "", "Comma separated types to exclude from binding")
 
-	vyFlag    = flag.String("vy", "", "Path to the Ethereum contract Vyper source to build and bind")
+	// Flags needed by abigen
-	vyperFlag = flag.String("vyper", "vyper", "Vyper compiler to use if source builds are requested")
+	abiFlag = cli.StringFlag{
-
+		Name:  "abi",
-	pkgFlag  = flag.String("pkg", "", "Package name to generate the binding into")
+		Usage: "Path to the Ethereum contract ABI json to bind, - for STDIN",
-	outFlag  = flag.String("out", "", "Output file for the generated binding (default = stdout)")
+	}
-	langFlag = flag.String("lang", "go", "Destination language for the bindings (go, java, objc)")
+	binFlag = cli.StringFlag{
+		Name:  "bin",
+		Usage: "Path to the Ethereum contract bytecode (generate deploy method)",
+	}
+	typeFlag = cli.StringFlag{
+		Name:  "type",
+		Usage: "Struct name for the binding (default = package name)",
+	}
+	jsonFlag = cli.StringFlag{
+		Name:  "combined-json",
+		Usage: "Path to the combined-json file generated by compiler",
+	}
+	solFlag = cli.StringFlag{
+		Name:  "sol",
+		Usage: "Path to the Ethereum contract Solidity source to build and bind",
+	}
+	solcFlag = cli.StringFlag{
+		Name:  "solc",
+		Usage: "Solidity compiler to use if source builds are requested",
+		Value: "solc",
+	}
+	vyFlag = cli.StringFlag{
+		Name:  "vy",
+		Usage: "Path to the Ethereum contract Vyper source to build and bind",
+	}
+	vyperFlag = cli.StringFlag{
+		Name:  "vyper",
+		Usage: "Vyper compiler to use if source builds are requested",
+		Value: "vyper",
+	}
+	excFlag = cli.StringFlag{
+		Name:  "exc",
+		Usage: "Comma separated types to exclude from binding",
+	}
+	pkgFlag = cli.StringFlag{
+		Name:  "pkg",
+		Usage: "Package name to generate the binding into",
+	}
+	outFlag = cli.StringFlag{
+		Name:  "out",
+		Usage: "Output file for the generated binding (default = stdout)",
+	}
+	langFlag = cli.StringFlag{
+		Name:  "lang",
+		Usage: "Destination language for the bindings (go, java, objc)",
+		Value: "go",
+	}
 )
 
-func main() {
+func init() {
-	// Parse and ensure all needed inputs are specified
+	app = utils.NewApp(gitCommit, gitDate, "ethereum checkpoint helper tool")
-	flag.Parse()
+	app.Flags = []cli.Flag{
-
+		abiFlag,
-	if *abiFlag == "" && *solFlag == "" && *vyFlag == "" {
+		binFlag,
-		fmt.Printf("No contract ABI (--abi), Solidity source (--sol), or Vyper source (--vy) specified\n")
+		typeFlag,
-		os.Exit(-1)
+		jsonFlag,
-	} else if (*abiFlag != "" || *binFlag != "" || *typFlag != "") && (*solFlag != "" || *vyFlag != "") {
+		solFlag,
-		fmt.Printf("Contract ABI (--abi), bytecode (--bin) and type (--type) flags are mutually exclusive with the Solidity (--sol) and Vyper (--vy) flags\n")
+		solcFlag,
-		os.Exit(-1)
+		vyFlag,
-	} else if *solFlag != "" && *vyFlag != "" {
+		vyperFlag,
-		fmt.Printf("Solidity (--sol) and Vyper (--vy) flags are mutually exclusive\n")
+		excFlag,
-		os.Exit(-1)
+		pkgFlag,
+		outFlag,
+		langFlag,
 	}
-	if *pkgFlag == "" {
+	app.Action = utils.MigrateFlags(abigen)
-		fmt.Printf("No destination package specified (--pkg)\n")
+	cli.CommandHelpTemplate = commandHelperTemplate
-		os.Exit(-1)
+}
+
+func abigen(c *cli.Context) error {
+	utils.CheckExclusive(c, abiFlag, jsonFlag, solFlag, vyFlag) // Only one source can be selected.
+	if c.GlobalString(pkgFlag.Name) == "" {
+		utils.Fatalf("No destination package specified (--pkg)")
 	}
 	var lang bind.Lang
-	switch *langFlag {
+	switch c.GlobalString(langFlag.Name) {
 	case "go":
 		lang = bind.LangGo
 	case "java":
 		lang = bind.LangJava
 	case "objc":
 		lang = bind.LangObjC
+		utils.Fatalf("Objc binding generation is uncompleted")
 	default:
-		fmt.Printf("Unsupported destination language \"%s\" (--lang)\n", *langFlag)
+		utils.Fatalf("Unsupported destination language \"%s\" (--lang)", c.GlobalString(langFlag.Name))
-		os.Exit(-1)
 	}
 	// If the entire solidity code was specified, build and bind based on that
 	var (
 		abis  []string
 		bins  []string
 		types []string
+		sigs  []map[string]string
+		libs  = make(map[string]string)
 	)
-	if *solFlag != "" || *vyFlag != "" || *abiFlag == "-" {
+	if c.GlobalString(abiFlag.Name) != "" {
+		// Load up the ABI, optional bytecode and type name from the parameters
+		var (
+			abi []byte
+			err error
+		)
+		input := c.GlobalString(abiFlag.Name)
+		if input == "-" {
+			abi, err = ioutil.ReadAll(os.Stdin)
+		} else {
+			abi, err = ioutil.ReadFile(input)
+		}
+		if err != nil {
+			utils.Fatalf("Failed to read input ABI: %v", err)
+		}
+		abis = append(abis, string(abi))
+
+		var bin []byte
+		if binFile := c.GlobalString(binFlag.Name); binFile != "" {
+			if bin, err = ioutil.ReadFile(binFile); err != nil {
+				utils.Fatalf("Failed to read input bytecode: %v", err)
+			}
+			if strings.Contains(string(bin), "//") {
+				utils.Fatalf("Contract has additional library references, please use other mode(e.g. --combined-json) to catch library infos")
+			}
+		}
+		bins = append(bins, string(bin))
+
+		kind := c.GlobalString(typeFlag.Name)
+		if kind == "" {
+			kind = c.GlobalString(pkgFlag.Name)
+		}
+		types = append(types, kind)
+	} else {
 		// Generate the list of types to exclude from binding
 		exclude := make(map[string]bool)
-		for _, kind := range strings.Split(*excFlag, ",") {
+		for _, kind := range strings.Split(c.GlobalString(excFlag.Name), ",") {
 			exclude[strings.ToLower(kind)] = true
 		}
-
-		var contracts map[string]*compiler.Contract
 		var err error
+		var contracts map[string]*compiler.Contract
 
 		switch {
-		case *solFlag != "":
+		case c.GlobalIsSet(solFlag.Name):
-			contracts, err = compiler.CompileSolidity(*solcFlag, *solFlag)
+			contracts, err = compiler.CompileSolidity(c.GlobalString(solcFlag.Name), c.GlobalString(solFlag.Name))
 			if err != nil {
-				fmt.Printf("Failed to build Solidity contract: %v\n", err)
+				utils.Fatalf("Failed to build Solidity contract: %v", err)
-				os.Exit(-1)
 			}
-		case *vyFlag != "":
+		case c.GlobalIsSet(vyFlag.Name):
-			contracts, err = compiler.CompileVyper(*vyperFlag, *vyFlag)
+			contracts, err = compiler.CompileVyper(c.GlobalString(vyperFlag.Name), c.GlobalString(vyFlag.Name))
 			if err != nil {
-				fmt.Printf("Failed to build Vyper contract: %v\n", err)
+				utils.Fatalf("Failed to build Vyper contract: %v", err)
-				os.Exit(-1)
 			}
-		default:
+		case c.GlobalIsSet(jsonFlag.Name):
-			contracts, err = contractsFromStdin()
+			jsonOutput, err := ioutil.ReadFile(c.GlobalString(jsonFlag.Name))
 			if err != nil {
-				fmt.Printf("Failed to read input ABIs from STDIN: %v\n", err)
+				utils.Fatalf("Failed to read combined-json from compiler: %v", err)
-				os.Exit(-1)
+			}
+			contracts, err = compiler.ParseCombinedJSON(jsonOutput, "", "", "", "")
+			if err != nil {
+				utils.Fatalf("Failed to read contract information from json output: %v", err)
 			}
 		}
 		// Gather all non-excluded contract for binding
@@ -118,61 +220,39 @@ func main() {
 			}
 			abi, err := json.Marshal(contract.Info.AbiDefinition) // Flatten the compiler parse
 			if err != nil {
-				fmt.Printf("Failed to parse ABIs from compiler output: %v\n", err)
+				utils.Fatalf("Failed to parse ABIs from compiler output: %v", err)
-				os.Exit(-1)
 			}
 			abis = append(abis, string(abi))
 			bins = append(bins, contract.Code)
-
+			sigs = append(sigs, contract.Hashes)
 			nameParts := strings.Split(name, ":")
 			types = append(types, nameParts[len(nameParts)-1])
-		}
-	} else {
-		// Otherwise load up the ABI, optional bytecode and type name from the parameters
-		abi, err := ioutil.ReadFile(*abiFlag)
 
-		if err != nil {
+			libPattern := crypto.Keccak256Hash([]byte(name)).String()[2:36]
-			fmt.Printf("Failed to read input ABI: %v\n", err)
+			libs[libPattern] = nameParts[len(nameParts)-1]
-			os.Exit(-1)
 		}
-		abis = append(abis, string(abi))
-
-		var bin []byte
-		if *binFlag != "" {
-			if bin, err = ioutil.ReadFile(*binFlag); err != nil {
-				fmt.Printf("Failed to read input bytecode: %v\n", err)
-				os.Exit(-1)
-			}
-		}
-		bins = append(bins, string(bin))
-
-		kind := *typFlag
-		if kind == "" {
-			kind = *pkgFlag
-		}
-		types = append(types, kind)
 	}
 	// Generate the contract binding
-	code, err := bind.Bind(types, abis, bins, *pkgFlag, lang)
+	code, err := bind.Bind(types, abis, bins, sigs, c.GlobalString(pkgFlag.Name), lang, libs)
 	if err != nil {
-		fmt.Printf("Failed to generate ABI binding: %v\n", err)
+		utils.Fatalf("Failed to generate ABI binding: %v", err)
-		os.Exit(-1)
 	}
 	// Either flush it out to a file or display on the standard output
-	if *outFlag == "" {
+	if !c.GlobalIsSet(outFlag.Name) {
 		fmt.Printf("%s\n", code)
-		return
+		return nil
 	}
-	if err := ioutil.WriteFile(*outFlag, []byte(code), 0600); err != nil {
+	if err := ioutil.WriteFile(c.GlobalString(outFlag.Name), []byte(code), 0600); err != nil {
-		fmt.Printf("Failed to write ABI binding: %v\n", err)
+		utils.Fatalf("Failed to write ABI binding: %v", err)
-		os.Exit(-1)
 	}
+	return nil
 }
 
-func contractsFromStdin() (map[string]*compiler.Contract, error) {
+func main() {
-	bytes, err := ioutil.ReadAll(os.Stdin)
+	log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
-	if err != nil {
+
-		return nil, err
+	if err := app.Run(os.Args); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
 	}
-	return compiler.ParseCombinedJSON(bytes, "", "", "", "")
 }

cmd/checkpoint-admin/common.go

@@ -0,0 +1,120 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"strconv"
+
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/accounts/external"
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/contracts/checkpointoracle"
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rpc"
+	"gopkg.in/urfave/cli.v1"
+)
+
+// newClient creates a client with specified remote URL.
+func newClient(ctx *cli.Context) *ethclient.Client {
+	client, err := ethclient.Dial(ctx.GlobalString(nodeURLFlag.Name))
+	if err != nil {
+		utils.Fatalf("Failed to connect to Ethereum node: %v", err)
+	}
+	return client
+}
+
+// newRPCClient creates a rpc client with specified node URL.
+func newRPCClient(url string) *rpc.Client {
+	client, err := rpc.Dial(url)
+	if err != nil {
+		utils.Fatalf("Failed to connect to Ethereum node: %v", err)
+	}
+	return client
+}
+
+// getContractAddr retrieves the register contract address through
+// rpc request.
+func getContractAddr(client *rpc.Client) common.Address {
+	var addr string
+	if err := client.Call(&addr, "les_getCheckpointContractAddress"); err != nil {
+		utils.Fatalf("Failed to fetch checkpoint oracle address: %v", err)
+	}
+	return common.HexToAddress(addr)
+}
+
+// getCheckpoint retrieves the specified checkpoint or the latest one
+// through rpc request.
+func getCheckpoint(ctx *cli.Context, client *rpc.Client) *params.TrustedCheckpoint {
+	var checkpoint *params.TrustedCheckpoint
+
+	if ctx.GlobalIsSet(indexFlag.Name) {
+		var result [3]string
+		index := uint64(ctx.GlobalInt64(indexFlag.Name))
+		if err := client.Call(&result, "les_getCheckpoint", index); err != nil {
+			utils.Fatalf("Failed to get local checkpoint %v, please ensure the les API is exposed", err)
+		}
+		checkpoint = &params.TrustedCheckpoint{
+			SectionIndex: index,
+			SectionHead:  common.HexToHash(result[0]),
+			CHTRoot:      common.HexToHash(result[1]),
+			BloomRoot:    common.HexToHash(result[2]),
+		}
+	} else {
+		var result [4]string
+		err := client.Call(&result, "les_latestCheckpoint")
+		if err != nil {
+			utils.Fatalf("Failed to get local checkpoint %v, please ensure the les API is exposed", err)
+		}
+		index, err := strconv.ParseUint(result[0], 0, 64)
+		if err != nil {
+			utils.Fatalf("Failed to parse checkpoint index %v", err)
+		}
+		checkpoint = &params.TrustedCheckpoint{
+			SectionIndex: index,
+			SectionHead:  common.HexToHash(result[1]),
+			CHTRoot:      common.HexToHash(result[2]),
+			BloomRoot:    common.HexToHash(result[3]),
+		}
+	}
+	return checkpoint
+}
+
+// newContract creates a registrar contract instance with specified
+// contract address or the default contracts for mainnet or testnet.
+func newContract(client *rpc.Client) (common.Address, *checkpointoracle.CheckpointOracle) {
+	addr := getContractAddr(client)
+	if addr == (common.Address{}) {
+		utils.Fatalf("No specified registrar contract address")
+	}
+	contract, err := checkpointoracle.NewCheckpointOracle(addr, ethclient.NewClient(client))
+	if err != nil {
+		utils.Fatalf("Failed to setup registrar contract %s: %v", addr, err)
+	}
+	return addr, contract
+}
+
+// newClefSigner sets up a clef backend and returns a clef transaction signer.
+func newClefSigner(ctx *cli.Context) *bind.TransactOpts {
+	clef, err := external.NewExternalSigner(ctx.String(clefURLFlag.Name))
+	if err != nil {
+		utils.Fatalf("Failed to create clef signer %v", err)
+	}
+	return bind.NewClefTransactor(clef, accounts.Account{Address: common.HexToAddress(ctx.String(signerFlag.Name))})
+}

cmd/checkpoint-admin/exec.go

@@ -0,0 +1,311 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"fmt"
+	"math/big"
+	"strings"
+	"time"
+
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/contracts/checkpointoracle"
+	"github.com/ethereum/go-ethereum/contracts/checkpointoracle/contract"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rpc"
+	"gopkg.in/urfave/cli.v1"
+)
+
+var commandDeploy = cli.Command{
+	Name:  "deploy",
+	Usage: "Deploy a new checkpoint oracle contract",
+	Flags: []cli.Flag{
+		nodeURLFlag,
+		clefURLFlag,
+		signerFlag,
+		signersFlag,
+		thresholdFlag,
+	},
+	Action: utils.MigrateFlags(deploy),
+}
+
+var commandSign = cli.Command{
+	Name:  "sign",
+	Usage: "Sign the checkpoint with the specified key",
+	Flags: []cli.Flag{
+		nodeURLFlag,
+		clefURLFlag,
+		signerFlag,
+		indexFlag,
+		hashFlag,
+		oracleFlag,
+	},
+	Action: utils.MigrateFlags(sign),
+}
+
+var commandPublish = cli.Command{
+	Name:  "publish",
+	Usage: "Publish a checkpoint into the oracle",
+	Flags: []cli.Flag{
+		nodeURLFlag,
+		clefURLFlag,
+		signerFlag,
+		indexFlag,
+		signaturesFlag,
+	},
+	Action: utils.MigrateFlags(publish),
+}
+
+// deploy deploys the checkpoint registrar contract.
+//
+// Note the network where the contract is deployed depends on
+// the network where the connected node is located.
+func deploy(ctx *cli.Context) error {
+	// Gather all the addresses that should be permitted to sign
+	var addrs []common.Address
+	for _, account := range strings.Split(ctx.String(signersFlag.Name), ",") {
+		if trimmed := strings.TrimSpace(account); !common.IsHexAddress(trimmed) {
+			utils.Fatalf("Invalid account in --signers: '%s'", trimmed)
+		}
+		addrs = append(addrs, common.HexToAddress(account))
+	}
+	// Retrieve and validate the signing threshold
+	needed := ctx.Int(thresholdFlag.Name)
+	if needed == 0 || needed > len(addrs) {
+		utils.Fatalf("Invalid signature threshold %d", needed)
+	}
+	// Print a summary to ensure the user understands what they're signing
+	fmt.Printf("Deploying new checkpoint oracle:\n\n")
+	for i, addr := range addrs {
+		fmt.Printf("Admin %d => %s\n", i+1, addr.Hex())
+	}
+	fmt.Printf("\nSignatures needed to publish: %d\n", needed)
+
+	// setup clef signer, create an abigen transactor and an RPC client
+	transactor, client := newClefSigner(ctx), newClient(ctx)
+
+	// Deploy the checkpoint oracle
+	fmt.Println("Sending deploy request to Clef...")
+	oracle, tx, _, err := contract.DeployCheckpointOracle(transactor, client, addrs, big.NewInt(int64(params.CheckpointFrequency)),
+		big.NewInt(int64(params.CheckpointProcessConfirmations)), big.NewInt(int64(needed)))
+	if err != nil {
+		utils.Fatalf("Failed to deploy checkpoint oracle %v", err)
+	}
+	log.Info("Deployed checkpoint oracle", "address", oracle, "tx", tx.Hash().Hex())
+
+	return nil
+}
+
+// sign creates the signature for specific checkpoint
+// with local key. Only contract admins have the permission to
+// sign checkpoint.
+func sign(ctx *cli.Context) error {
+	var (
+		offline bool // The indicator whether we sign checkpoint by offline.
+		chash   common.Hash
+		cindex  uint64
+		address common.Address
+
+		node   *rpc.Client
+		oracle *checkpointoracle.CheckpointOracle
+	)
+	if !ctx.GlobalIsSet(nodeURLFlag.Name) {
+		// Offline mode signing
+		offline = true
+		if !ctx.IsSet(hashFlag.Name) {
+			utils.Fatalf("Please specify the checkpoint hash (--hash) to sign in offline mode")
+		}
+		chash = common.HexToHash(ctx.String(hashFlag.Name))
+
+		if !ctx.IsSet(indexFlag.Name) {
+			utils.Fatalf("Please specify checkpoint index (--index) to sign in offline mode")
+		}
+		cindex = ctx.Uint64(indexFlag.Name)
+
+		if !ctx.IsSet(oracleFlag.Name) {
+			utils.Fatalf("Please specify oracle address (--oracle) to sign in offline mode")
+		}
+		address = common.HexToAddress(ctx.String(oracleFlag.Name))
+	} else {
+		// Interactive mode signing, retrieve the data from the remote node
+		node = newRPCClient(ctx.GlobalString(nodeURLFlag.Name))
+
+		checkpoint := getCheckpoint(ctx, node)
+		chash, cindex, address = checkpoint.Hash(), checkpoint.SectionIndex, getContractAddr(node)
+
+		// Check the validity of checkpoint
+		reqCtx, cancelFn := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancelFn()
+
+		head, err := ethclient.NewClient(node).HeaderByNumber(reqCtx, nil)
+		if err != nil {
+			return err
+		}
+		num := head.Number.Uint64()
+		if num < ((cindex+1)*params.CheckpointFrequency + params.CheckpointProcessConfirmations) {
+			utils.Fatalf("Invalid future checkpoint")
+		}
+		_, oracle = newContract(node)
+		latest, _, h, err := oracle.Contract().GetLatestCheckpoint(nil)
+		if err != nil {
+			return err
+		}
+		if cindex < latest {
+			utils.Fatalf("Checkpoint is too old")
+		}
+		if cindex == latest && (latest != 0 || h.Uint64() != 0) {
+			utils.Fatalf("Stale checkpoint, latest registered %d, given %d", latest, cindex)
+		}
+	}
+	var (
+		signature string
+		signer    string
+	)
+	// isAdmin checks whether the specified signer is admin.
+	isAdmin := func(addr common.Address) error {
+		signers, err := oracle.Contract().GetAllAdmin(nil)
+		if err != nil {
+			return err
+		}
+		for _, s := range signers {
+			if s == addr {
+				return nil
+			}
+		}
+		return fmt.Errorf("signer %v is not the admin", addr.Hex())
+	}
+	// Print to the user the data thy are about to sign
+	fmt.Printf("Oracle     => %s\n", address.Hex())
+	fmt.Printf("Index %4d => %s\n", cindex, chash.Hex())
+
+	// Sign checkpoint in clef mode.
+	signer = ctx.String(signerFlag.Name)
+
+	if !offline {
+		if err := isAdmin(common.HexToAddress(signer)); err != nil {
+			return err
+		}
+	}
+	clef := newRPCClient(ctx.String(clefURLFlag.Name))
+	p := make(map[string]string)
+	buf := make([]byte, 8)
+	binary.BigEndian.PutUint64(buf, cindex)
+	p["address"] = address.Hex()
+	p["message"] = hexutil.Encode(append(buf, chash.Bytes()...))
+
+	fmt.Println("Sending signing request to Clef...")
+	if err := clef.Call(&signature, "account_signData", accounts.MimetypeDataWithValidator, signer, p); err != nil {
+		utils.Fatalf("Failed to sign checkpoint, err %v", err)
+	}
+	fmt.Printf("Signer     => %s\n", signer)
+	fmt.Printf("Signature  => %s\n", signature)
+	return nil
+}
+
+// sighash calculates the hash of the data to sign for the checkpoint oracle.
+func sighash(index uint64, oracle common.Address, hash common.Hash) []byte {
+	buf := make([]byte, 8)
+	binary.BigEndian.PutUint64(buf, index)
+
+	data := append([]byte{0x19, 0x00}, append(oracle[:], append(buf, hash[:]...)...)...)
+	return crypto.Keccak256(data)
+}
+
+// ecrecover calculates the sender address from a sighash and signature combo.
+func ecrecover(sighash []byte, sig []byte) common.Address {
+	sig[64] -= 27
+	defer func() { sig[64] += 27 }()
+
+	signer, err := crypto.SigToPub(sighash, sig)
+	if err != nil {
+		utils.Fatalf("Failed to recover sender from signature %x: %v", sig, err)
+	}
+	return crypto.PubkeyToAddress(*signer)
+}
+
+// publish registers the specified checkpoint which generated by connected node
+// with a authorised private key.
+func publish(ctx *cli.Context) error {
+	// Print the checkpoint oracle's current status to make sure we're interacting
+	// with the correct network and contract.
+	status(ctx)
+
+	// Gather the signatures from the CLI
+	var sigs [][]byte
+	for _, sig := range strings.Split(ctx.String(signaturesFlag.Name), ",") {
+		trimmed := strings.TrimPrefix(strings.TrimSpace(sig), "0x")
+		if len(trimmed) != 130 {
+			utils.Fatalf("Invalid signature in --signature: '%s'", trimmed)
+		} else {
+			sigs = append(sigs, common.Hex2Bytes(trimmed))
+		}
+	}
+	// Retrieve the checkpoint we want to sign to sort the signatures
+	var (
+		client       = newRPCClient(ctx.GlobalString(nodeURLFlag.Name))
+		addr, oracle = newContract(client)
+		checkpoint   = getCheckpoint(ctx, client)
+		sighash      = sighash(checkpoint.SectionIndex, addr, checkpoint.Hash())
+	)
+	for i := 0; i < len(sigs); i++ {
+		for j := i + 1; j < len(sigs); j++ {
+			signerA := ecrecover(sighash, sigs[i])
+			signerB := ecrecover(sighash, sigs[j])
+			if bytes.Compare(signerA.Bytes(), signerB.Bytes()) > 0 {
+				sigs[i], sigs[j] = sigs[j], sigs[i]
+			}
+		}
+	}
+	// Retrieve recent header info to protect replay attack
+	reqCtx, cancelFn := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancelFn()
+
+	head, err := ethclient.NewClient(client).HeaderByNumber(reqCtx, nil)
+	if err != nil {
+		return err
+	}
+	num := head.Number.Uint64()
+	recent, err := ethclient.NewClient(client).HeaderByNumber(reqCtx, big.NewInt(int64(num-128)))
+	if err != nil {
+		return err
+	}
+	// Print a summary of the operation that's going to be performed
+	fmt.Printf("Publishing %d => %s:\n\n", checkpoint.SectionIndex, checkpoint.Hash().Hex())
+	for i, sig := range sigs {
+		fmt.Printf("Signer %d => %s\n", i+1, ecrecover(sighash, sig).Hex())
+	}
+	fmt.Println()
+	fmt.Printf("Sentry number => %d\nSentry hash   => %s\n", recent.Number, recent.Hash().Hex())
+
+	// Publish the checkpoint into the oracle
+	fmt.Println("Sending publish request to Clef...")
+	tx, err := oracle.RegisterCheckpoint(newClefSigner(ctx), checkpoint.SectionIndex, checkpoint.Hash().Bytes(), recent.Number, recent.Hash(), sigs)
+	if err != nil {
+		utils.Fatalf("Register contract failed %v", err)
+	}
+	log.Info("Successfully registered checkpoint", "tx", tx.Hash().Hex())
+	return nil
+}

cmd/checkpoint-admin/main.go

@@ -0,0 +1,117 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+// checkpoint-admin is a utility that can be used to query checkpoint information
+// and register stable checkpoints into an oracle contract.
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/common/fdlimit"
+	"github.com/ethereum/go-ethereum/log"
+	"gopkg.in/urfave/cli.v1"
+)
+
+const (
+	commandHelperTemplate = `{{.Name}}{{if .Subcommands}} command{{end}}{{if .Flags}} [command options]{{end}} [arguments...]
+{{if .Description}}{{.Description}}
+{{end}}{{if .Subcommands}}
+SUBCOMMANDS:
+	{{range .Subcommands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}}
+	{{end}}{{end}}{{if .Flags}}
+OPTIONS:
+{{range $.Flags}}{{"\t"}}{{.}}
+{{end}}
+{{end}}`
+)
+
+var (
+	// Git SHA1 commit hash of the release (set via linker flags)
+	gitCommit = ""
+	gitDate   = ""
+)
+
+var app *cli.App
+
+func init() {
+	app = utils.NewApp(gitCommit, gitDate, "ethereum checkpoint helper tool")
+	app.Commands = []cli.Command{
+		commandStatus,
+		commandDeploy,
+		commandSign,
+		commandPublish,
+	}
+	app.Flags = []cli.Flag{
+		oracleFlag,
+		nodeURLFlag,
+	}
+	cli.CommandHelpTemplate = commandHelperTemplate
+}
+
+// Commonly used command line flags.
+var (
+	indexFlag = cli.Int64Flag{
+		Name:  "index",
+		Usage: "Checkpoint index (query latest from remote node if not specified)",
+	}
+	hashFlag = cli.StringFlag{
+		Name:  "hash",
+		Usage: "Checkpoint hash (query latest from remote node if not specified)",
+	}
+	oracleFlag = cli.StringFlag{
+		Name:  "oracle",
+		Usage: "Checkpoint oracle address (query from remote node if not specified)",
+	}
+	thresholdFlag = cli.Int64Flag{
+		Name:  "threshold",
+		Usage: "Minimal number of signatures required to approve a checkpoint",
+	}
+	nodeURLFlag = cli.StringFlag{
+		Name:  "rpc",
+		Value: "http://localhost:8545",
+		Usage: "The rpc endpoint of a local or remote geth node",
+	}
+	clefURLFlag = cli.StringFlag{
+		Name:  "clef",
+		Value: "http://localhost:8550",
+		Usage: "The rpc endpoint of clef",
+	}
+	signerFlag = cli.StringFlag{
+		Name:  "signer",
+		Usage: "Signer address for clef signing",
+	}
+	signersFlag = cli.StringFlag{
+		Name:  "signers",
+		Usage: "Comma separated accounts of trusted checkpoint signers",
+	}
+	signaturesFlag = cli.StringFlag{
+		Name:  "signatures",
+		Usage: "Comma separated checkpoint signatures to submit",
+	}
+)
+
+func main() {
+	log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
+	fdlimit.Raise(2048)
+
+	if err := app.Run(os.Args); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}

cmd/checkpoint-admin/status.go

@@ -0,0 +1,61 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/common"
+	"gopkg.in/urfave/cli.v1"
+)
+
+var commandStatus = cli.Command{
+	Name:  "status",
+	Usage: "Fetches the signers and checkpoint status of the oracle contract",
+	Flags: []cli.Flag{
+		nodeURLFlag,
+	},
+	Action: utils.MigrateFlags(status),
+}
+
+// status fetches the admin list of specified registrar contract.
+func status(ctx *cli.Context) error {
+	// Create a wrapper around the checkpoint oracle contract
+	addr, oracle := newContract(newRPCClient(ctx.GlobalString(nodeURLFlag.Name)))
+	fmt.Printf("Oracle => %s\n", addr.Hex())
+	fmt.Println()
+
+	// Retrieve the list of authorized signers (admins)
+	admins, err := oracle.Contract().GetAllAdmin(nil)
+	if err != nil {
+		return err
+	}
+	for i, admin := range admins {
+		fmt.Printf("Admin %d => %s\n", i+1, admin.Hex())
+	}
+	fmt.Println()
+
+	// Retrieve the latest checkpoint
+	index, checkpoint, height, err := oracle.Contract().GetLatestCheckpoint(nil)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("Checkpoint (published at #%d) %d => %s\n", height, index, common.Hash(checkpoint).Hex())
+
+	return nil
+}

cmd/clef/README.md

@@ -1,41 +1,38 @@
-Clef
+# Clef
-----
-Clef can be used to sign transactions and data and is meant as a replacement for geth's account management.
-This allows DApps not to depend on geth's account management. When a DApp wants to sign data it can send the data to
-the signer, the signer will then provide the user with context and asks the user for permission to sign the data. If
-the users grants the signing request the signer will send the signature back to the DApp.
-  
-This setup allows a DApp to connect to a remote Ethereum node and send transactions that are locally signed. This can
-help in situations when a DApp is connected to a remote node because a local Ethereum node is not available, not
-synchronised with the chain or a particular Ethereum node that has no built-in (or limited) account management.
-  
-Clef can run as a daemon on the same machine, or off a usb-stick like [usb armory](https://inversepath.com/usbarmory),
-or a separate VM in a [QubesOS](https://www.qubes-os.org/) type os setup.
 
-Check out 
+Clef can be used to sign transactions and data and is meant as a(n eventual) replacement for Geth's account management. This allows DApps to not depend on Geth's account management. When a DApp wants to sign data (or a transaction), it can send the content to Clef, which will then provide the user with context and asks for permission to sign the content. If the users grants the signing request, Clef will send the signature back to the DApp.
 
-* the [tutorial](tutorial.md) for some concrete examples on how the signer works.
+This setup allows a DApp to connect to a remote Ethereum node and send transactions that are locally signed. This can help in situations when a DApp is connected to an untrusted remote Ethereum node, because a local one is not available, not synchronised with the chain, or is a node that has no built-in (or limited) account management.
-* the [setup docs](docs/setup.md) for some information on how to configure it to work on QubesOS or USBArmory. 
+
-* the [data types](datatypes.md) for detailed information on the json types used in the communication between
+Clef can run as a daemon on the same machine, off a usb-stick like [USB armory](https://inversepath.com/usbarmory), or even a separate VM in a [QubesOS](https://www.qubes-os.org/) type setup.
-  clef and an external UI 
+
+Check out the
+
+* [CLI tutorial](tutorial.md) for some concrete examples on how Clef works.
+* [Setup docs](docs/setup.md) for infos on how to configure Clef on QubesOS or USB Armory.
+* [Data types](datatypes.md) for details on the communication messages between Clef and an external UI.
 
 ## Command line flags
+
 Clef accepts the following command line options:
+
 ```
 COMMANDS:
    init    Initialize the signer, generate secret storage
    attest  Attest that a js-file is to be used
    setpw   Store a credential for a keystore file
+   delpw   Remove a credential for a keystore file
    gendoc  Generate documentation about json-rpc format
    help    Shows a list of commands or help for one command
-   
+
 GLOBAL OPTIONS:
    --loglevel value        log level to emit to the screen (default: 4)
    --keystore value        Directory for the keystore (default: "$HOME/.ethereum/keystore")
    --configdir value       Directory for Clef configuration (default: "$HOME/.clef")
-   --chainid value         Chain id to use for signing (1=mainnet, 3=ropsten, 4=rinkeby, 5=Goerli) (default: 1)
+   --chainid value         Chain id to use for signing (1=mainnet, 3=Ropsten, 4=Rinkeby, 5=Goerli) (default: 1)
    --lightkdf              Reduce key-derivation RAM & CPU usage at some expense of KDF strength
    --nousb                 Disables monitoring for and managing USB hardware wallets
+   --pcscdpath value       Path to the smartcard daemon (pcscd) socket file (default: "/run/pcscd/pcscd.comm")
    --rpcaddr value         HTTP-RPC server listening interface (default: "localhost")
    --rpcvhosts value       Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard. (default: "localhost")
    --ipcdisable            Disable the IPC-RPC server
@@ -43,60 +40,48 @@ GLOBAL OPTIONS:
    --rpc                   Enable the HTTP-RPC server
    --rpcport value         HTTP-RPC server listening port (default: 8550)
    --signersecret value    A file containing the (encrypted) master seed to encrypt Clef data, e.g. keystore credentials and ruleset hash
-   --4bytedb value         File containing 4byte-identifiers (default: "./4byte.json")
    --4bytedb-custom value  File used for writing new 4byte-identifiers submitted via API (default: "./4byte-custom.json")
    --auditlog value        File used to emit audit logs. Set to "" to disable (default: "audit.log")
-   --rules value           Enable rule-engine (default: "rules.json")
+   --rules value           Path to the rule file to auto-authorize requests with
    --stdio-ui              Use STDIN/STDOUT as a channel for an external UI. This means that an STDIN/STDOUT is used for RPC-communication with a e.g. a graphical user interface, and can be used when Clef is started by an external process.
    --stdio-ui-test         Mechanism to test interface between Clef and UI. Requires 'stdio-ui'.
    --advanced              If enabled, issues warnings instead of rejections for suspicious requests. Default off
    --help, -h              show help
    --version, -v           print the version
-   
 ```
 
-
 Example:
-```
-signer -keystore /my/keystore -chainid 4
-```
 
+```
+$ clef -keystore /my/keystore -chainid 4
+```
 
 ## Security model
 
-The security model of the signer is as follows:
+The security model of Clef is as follows:
 
-* One critical component (the signer binary / daemon) is responsible for handling cryptographic operations: signing, private keys, encryption/decryption of keystore files.
+* One critical component (the Clef binary / daemon) is responsible for handling cryptographic operations: signing, private keys, encryption/decryption of keystore files.
-* The signer binary has a well-defined 'external' API.
+* Clef has a well-defined 'external' API.
 * The 'external' API is considered UNTRUSTED.
-* The signer binary also communicates with whatever process that invoked the binary, via stdin/stdout.
+* Clef also communicates with whatever process that invoked the binary, via stdin/stdout.
   * This channel is considered 'trusted'. Over this channel, approvals and passwords are communicated.
 
-The general flow for signing a transaction using e.g. geth is as follows:
+The general flow for signing a transaction using e.g. Geth is as follows:
 ![image](sign_flow.png)
 
-In this case, `geth` would be started with `--externalsigner=http://localhost:8550` and would relay requests to `eth.sendTransaction`.
+In this case, `geth` would be started with `--signer http://localhost:8550` and would relay requests to `eth.sendTransaction`.
 
 ## TODOs
 
 Some snags and todos
 
-* [ ] The signer should take a startup param "--no-change", for UIs that do not contain the capability
+* [ ] Clef should take a startup param "--no-change", for UIs that do not contain the capability to perform changes to things, only approve/deny. Such a UI should be able to start the signer in a more secure mode by telling it that it only wants approve/deny capabilities.
-   to perform changes to things, only approve/deny. Such a UI should be able to start the signer in
+* [x] It would be nice if Clef could collect new 4byte-id:s/method selectors, and have a secondary database for those (`4byte_custom.json`). Users could then (optionally) submit their collections for inclusion upstream.
-   a more secure mode by telling it that it only wants approve/deny capabilities.
+* [ ] It should be possible to configure Clef to check if an account is indeed known to it, before passing on to the UI. The reason it currently does not, is that it would make it possible to enumerate accounts if it immediately returned "unknown account" (side channel attack).
-
+* [x] It should be possible to configure Clef to auto-allow listing (certain) accounts, instead of asking every time.
-* [x] It would be nice if the signer could collect new 4byte-id:s/method selectors, and have a
+* [x] Done Upon startup, Clef should spit out some info to the caller (particularly important when executed in `stdio-ui`-mode), invoking methods with the following info:
-secondary database for those (`4byte_custom.json`). Users could then (optionally) submit their collections for
-inclusion upstream.
-
-* It should be possible to configure the signer to check if an account is indeed known to it, before
-passing on to the UI. The reason it currently does not, is that it would make it possible to enumerate
-accounts if it immediately returned "unknown account".
-* [x] It should be possible to configure the signer to auto-allow listing (certain) accounts, instead of asking every time.
-* [x] Done Upon startup, the signer should spit out some info to the caller (particularly important when executed in `stdio-ui`-mode),
-invoking methods with the following info:
   * [x] Version info about the signer
-  * [x] Address of API (http/ipc)
+  * [x] Address of API (HTTP/IPC)
   * [ ] List of known accounts
 * [ ] Have a default timeout on signing operations, so that if the user has not answered within e.g. 60 seconds, the request is rejected.
 * [ ] `account_signRawTransaction`
@@ -109,21 +94,16 @@ invoking methods with the following info:
       * the number of unique recipients
 
 * Geth todos
-    - The signer should pass the `Origin` header as call-info to the UI. As of right now, the way that info about the request is
+    - The signer should pass the `Origin` header as call-info to the UI. As of right now, the way that info about the request is put together is a bit of a hack into the HTTP server. This could probably be greatly improved.
-put together is a bit of a hack into the http server. This could probably be greatly improved
+    - Relay: Geth should be started in `geth --signer localhost:8550`.
-    - Relay: Geth should be started in `geth --external_signer localhost:8550`.
+    - Currently, the Geth APIs use `common.Address` in the arguments to transaction submission (e.g `to` field). This type is 20 `bytes`, and is incapable of carrying checksum information. The signer uses `common.MixedcaseAddress`, which retains the original input.
-    - Currently, the Geth APIs use `common.Address` in the arguments to transaction submission (e.g `to` field). This
+    - The Geth API should switch to use the same type, and relay `to`-account verbatim to the external API.
-  type is 20 `bytes`, and is incapable of carrying checksum information. The signer uses `common.MixedcaseAddress`, which
-  retains the original input.
-    - The Geth api should switch to use the same type, and relay `to`-account verbatim to the external api.
-
 * [x] Storage
-    * [x] An encrypted key-value storage should be implemented
+    * [x] An encrypted key-value storage should be implemented.
     * See [rules.md](rules.md) for more info about this.
-
 * Another potential thing to introduce is pairing.
   * To prevent spurious requests which users just accept, implement a way to "pair" the caller with the signer (external API).
-  * Thus geth/mist/cpp would cryptographically handshake and afterwards the caller would be allowed to make signing requests.
+  * Thus Geth/cpp would cryptographically handshake and afterwards the caller would be allowed to make signing requests.
   * This feature would make the addition of rules less dangerous.
 
 * Wallets / accounts. Add API methods for wallets.
@@ -132,37 +112,31 @@ put together is a bit of a hack into the http server. This could probably be gre
 
 ### External API
 
-The signer listens to HTTP requests on `rpcaddr`:`rpcport`, with the same JSONRPC standard as Geth. The messages are
+Clef listens to HTTP requests on `rpcaddr`:`rpcport` (or to IPC on `ipcpath`), with the same JSON-RPC standard as Geth. The messages are expected to be [JSON-RPC 2.0 standard](https://www.jsonrpc.org/specification).
-expected to be JSON [jsonrpc 2.0 standard](http://www.jsonrpc.org/specification).
 
-Some of these call can require user interaction. Clients must be aware that responses
+Some of these call can require user interaction. Clients must be aware that responses may be delayed significantly or may never be received if a users decides to ignore the confirmation request.
-may be delayed significantly or may never be received if a users decides to ignore the confirmation request.
 
-The External API is **untrusted** : it does not accept credentials over this api, nor does it expect
+The External API is **untrusted**: it does not accept credentials over this API, nor does it expect that requests have any authority.
-that requests have any authority.
 
-### UI API
+### Internal UI API
 
-The signer has one native console-based UI, for operation without any standalone tools.
+Clef has one native console-based UI, for operation without any standalone tools. However, there is also an API to communicate with an external UI. To enable that UI, the signer needs to be executed with the `--stdio-ui` option, which allocates `stdin` / `stdout` for the UI API.
-However, there is also an API to communicate with an external UI. To enable that UI,
-the signer needs to be executed with the `--stdio-ui` option, which allocates the
-`stdin`/`stdout` for the UI-api.
 
 An example (insecure) proof-of-concept of has been implemented in `pythonsigner.py`.
 
 The model is as follows:
 
 * The user starts the UI app (`pythonsigner.py`).
-* The UI app starts the `signer` with `--stdio-ui`, and listens to the
+* The UI app starts `clef` with `--stdio-ui`, and listens to the
 process output for confirmation-requests.
-* The `signer` opens the external http api.
+* `clef` opens the external HTTP API.
-* When the `signer` receives requests, it sends a `jsonrpc` request via `stdout`.
+* When the `signer` receives requests, it sends a JSON-RPC request via `stdout`.
-* The UI app prompts the user accordingly, and responds to the `signer`
+* The UI app prompts the user accordingly, and responds to `clef`.
-* The `signer` signs (or not), and responds to the original request.
+* `clef` signs (or not), and responds to the original request.
 
 ## External API
 
-See the [external api changelog](extapi_changelog.md) for information about changes to this API.
+See the [external API changelog](extapi_changelog.md) for information about changes to this API.
 
 ### Encoding
 - number: positive integers that are hex encoded
@@ -187,7 +161,7 @@ None
 #### Result
   - address [string]: account address that is derived from the generated key
   - url [string]: location of the keyfile
-  
+
 #### Sample call
 ```json
 {
@@ -221,9 +195,9 @@ None
 #### Result
   - array with account records:
      - account.address [string]: account address that is derived from the generated key
-     - account.type [string]: type of the 
+     - account.type [string]: type of the
      - account.url [string]: location of the account
-  
+
 #### Sample call
 ```json
 {
@@ -272,7 +246,7 @@ Response
 
 #### Result
   - signed transaction in RLP encoded form [data]
-  
+
 #### Sample call
 ```json
 {
@@ -372,7 +346,7 @@ Bash example:
 
 #### Result
   - calculated signature [data]
-  
+
 #### Sample call
 ```json
 {
@@ -407,7 +381,7 @@ Response
 
 #### Result
   - calculated signature [data]
-  
+
 #### Sample call
 ```json
 {
@@ -505,7 +479,7 @@ Derive the address from the account that was used to sign data with content type
 
 #### Result
   - derived account [address]
-  
+
 #### Sample call
 ```json
 {
@@ -534,16 +508,16 @@ Response
 #### Import account
    Import a private key into the keystore. The imported key is expected to be encrypted according to the web3 keystore
    format.
-   
+
 #### Arguments
-  - account [object]: key in [web3 keystore format](https://github.com/ethereum/wiki/wiki/Web3-Secret-Storage-Definition) (retrieved with account_export) 
+  - account [object]: key in [web3 keystore format](https://github.com/ethereum/wiki/wiki/Web3-Secret-Storage-Definition) (retrieved with account_export)
 
 #### Result
   - imported key [object]:
      - key.address [address]: address of the imported key
      - key.type [string]: type of the account
      - key.url [string]: key URL
-  
+
 #### Sample call
 ```json
 {
@@ -592,16 +566,16 @@ Response
 ### account_export
 
 #### Export account from keystore
-   Export a private key from the keystore. The exported private key is encrypted with the original passphrase. When the
+   Export a private key from the keystore. The exported private key is encrypted with the original password. When the
-   key is imported later this passphrase is required.
+   key is imported later this password is required.
-   
+
 #### Arguments
   - account [address]: export private key that is associated with this account
 
 #### Result
   - exported key, see [web3 keystore format](https://github.com/ethereum/wiki/wiki/Web3-Secret-Storage-Definition) for
   more information
-  
+
 #### Sample call
 ```json
 {
@@ -643,8 +617,6 @@ Response
 }
 ```
 
-
-
 ## UI API
 
 These methods needs to be implemented by a UI listener.
@@ -655,7 +627,7 @@ See `pythonsigner`, which can be invoked via `python3 pythonsigner.py test` to p
 
 All methods in this API uses object-based parameters, so that there can be no mixups of parameters: each piece of data is accessed by key.
 
-See the [ui api changelog](intapi_changelog.md) for information about changes to this API.
+See the [ui API changelog](intapi_changelog.md) for information about changes to this API.
 
 OBS! A slight deviation from `json` standard is in place: every request and response should be confined to a single line.
 Whereas the `json` specification allows for linebreaks, linebreaks __should not__ be used in this communication channel, to make
@@ -909,7 +881,7 @@ TLDR; Use this method to keep track of signed transactions, instead of using the
 
 ### OnSignerStartup / `ui_onSignerStartup`
 
-This method provide the UI with information about what API version the signer uses (both internal and external) aswell as build-info and external api,
+This method provide the UI with information about what API version the signer uses (both internal and external) aswell as build-info and external API,
 in k/v-form.
 
 Example call:
@@ -942,7 +914,7 @@ A UI should conform to the following rules.
   * For example, not load icons, stylesheets from the internet
   * Not load files from the filesystem, unless they reside in the same local directory (e.g. config files)
 * A Graphical UI MUST show the blocky-identicon for ethereum addresses.
-* A UI MUST warn display approproate warning if the destination-account is formatted with invalid checksum.
+* A UI MUST warn display appropriate warning if the destination-account is formatted with invalid checksum.
 * A UI MUST NOT open any ports or services
   * The signer opens the public port
 * A UI SHOULD verify the permissions on the signer binary, and refuse to execute or warn if permissions allow non-user write.
@@ -953,9 +925,9 @@ A UI should conform to the following rules.
 along with the UI.
 
 
-### UI Implementations 
+### UI Implementations
 
-There are a couple of implementation for a UI. We'll try to keep this list up to date. 
+There are a couple of implementation for a UI. We'll try to keep this list up to date.
 
 | Name | Repo | UI type| No external resources| Blocky support| Verifies permissions | Hash information | No secondary storage | Statically linked| Can modify parameters|
 | ---- | ---- | -------| ---- | ---- | ---- |---- | ---- | ---- | ---- |

cmd/clef/datatypes.md

@@ -11,7 +11,7 @@ Example:
   "content_type": "text/plain",
   "address": "0xDEADbEeF000000000000000000000000DeaDbeEf",
   "raw_data": "GUV0aGVyZXVtIFNpZ25lZCBNZXNzYWdlOgoxMWhlbGxvIHdvcmxk",
-  "message": [
+  "messages": [
     {
       "name": "message",
       "value": "\u0019Ethereum Signed Message:\n11hello world",
@@ -133,7 +133,7 @@ This occurs _after_ successful completion of the entire signing procedure, but r
 
 A ruleset that implements a rate limitation needs to know what transactions are sent out to the external interface. By hooking into this methods, the ruleset can maintain track of that count.
 
-**OBS:** Note that if an attacker can restore your `clef` data to a previous point in time (e.g through a backup), the attacker can reset such windows, even if he/she is unable to decrypt the content. 
+**OBS:** Note that if an attacker can restore your `clef` data to a previous point in time (e.g through a backup), the attacker can reset such windows, even if he/she is unable to decrypt the content.
 
 The `OnApproved` method cannot be responded to, it's purely informative
 
@@ -179,7 +179,7 @@ Example:
 ```
 ### ListRequest
 
-Sent when a request has been made to list addresses. The UI is provided with the full `account`s, including local directory names. Note: this information is not passed back to the external caller, who only sees the `address`es. 
+Sent when a request has been made to list addresses. The UI is provided with the full `account`s, including local directory names. Note: this information is not passed back to the external caller, who only sees the `address`es.
 
 Example:
 ```json

cmd/clef/extapi_changelog.md

@@ -1,4 +1,15 @@
-### Changelog for external API
+## Changelog for external API
+
+The API uses [semantic versioning](https://semver.org/).
+
+TL;DR: Given a version number MAJOR.MINOR.PATCH, increment the:
+
+* MAJOR version when you make incompatible API changes,
+* MINOR version when you add functionality in a backwards-compatible manner, and
+* PATCH version when you make backwards-compatible bug fixes.
+
+Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format.
+
 
 ### 6.0.0
 
@@ -14,15 +25,15 @@ The addition of `contentType` makes it possible to use the method for different
   * signing clique headers,
   * signing plain personal messages,
 * The external method `account_signTypedData` implements [EIP-712](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-712.md) and makes it possible to sign typed data.
-  
+
 #### 4.0.0
 
-* The external `account_Ecrecover`-method was removed. 
+* The external `account_Ecrecover`-method was removed.
 * The external `account_Import`-method was removed.
 
 #### 3.0.0
 
-* The external `account_List`-method was changed to not expose `url`, which contained info about the local filesystem. It now returns only a list of addresses. 
+* The external `account_List`-method was changed to not expose `url`, which contained info about the local filesystem. It now returns only a list of addresses.
 
 #### 2.0.0
 
@@ -33,15 +44,3 @@ makes the `accounts_signTransaction` identical to the old `eth_signTransaction`.
 #### 1.0.0
 
 Initial release.
-
-### Versioning
-
-The API uses [semantic versioning](https://semver.org/).
-
-TLDR; Given a version number MAJOR.MINOR.PATCH, increment the:
-
-* MAJOR version when you make incompatible API changes,
-* MINOR version when you add functionality in a backwards-compatible manner, and
-* PATCH version when you make backwards-compatible bug fixes.
-
-Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format.

cmd/clef/intapi_changelog.md

@@ -1,24 +1,39 @@
-### Changelog for internal API (ui-api)
+## Changelog for internal API (ui-api)
 
-### 6.0.0 
+The API uses [semantic versioning](https://semver.org/).
 
-Removed `password` from responses to operations which require them. This is for two reasons, 
+TL;DR: Given a version number MAJOR.MINOR.PATCH, increment the:
 
-- Consistency between how rulesets operate and how manual processing works. A rule can `Approve` but require the actual password to be stored in the clef storage. 
+* MAJOR version when you make incompatible API changes,
-With this change, the same stored password can be used even if rulesets are not enabled, but storage is. 
+* MINOR version when you add functionality in a backwards-compatible manner, and
-- It also removes the usability-shortcut that a UI might otherwise want to implement; remembering passwords. Since we now will not require the 
+* PATCH version when you make backwards-compatible bug fixes.
-password on every `Approve`, there's no need for the UI to cache it locally. 
+
-  - In a future update, we'll likely add `clef_storePassword` to the internal API, so the user can store it via his UI (currently only CLI works). 
+Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format.
+
+### 7.0.0
+
+- The `message` field was renamed to `messages` in all data signing request methods to better reflect that it's a list, not a value.
+- The `storage.Put` and `storage.Get` methods in the rule execution engine were lower-cased to `storage.put` and `storage.get` to be consistent with JavaScript call conventions.
+
+### 6.0.0
+
+Removed `password` from responses to operations which require them. This is for two reasons,
+
+- Consistency between how rulesets operate and how manual processing works. A rule can `Approve` but require the actual password to be stored in the clef storage.
+With this change, the same stored password can be used even if rulesets are not enabled, but storage is.
+- It also removes the usability-shortcut that a UI might otherwise want to implement; remembering passwords. Since we now will not require the
+password on every `Approve`, there's no need for the UI to cache it locally.
+  - In a future update, we'll likely add `clef_storePassword` to the internal API, so the user can store it via his UI (currently only CLI works).
 
 Affected datatypes:
 - `SignTxResponse`
 - `SignDataResponse`
 - `NewAccountResponse`
 
-If `clef` requires a password, the `OnInputRequired` will be used to collect it. 
+If `clef` requires a password, the `OnInputRequired` will be used to collect it.
 
 
-### 5.0.0 
+### 5.0.0
 
 Changed the namespace format to adhere to the legacy ethereum format: `name_methodName`. Changes:
 
@@ -38,7 +53,7 @@ Changed the namespace format to adhere to the legacy ethereum format: `name_meth
 ### 4.0.0
 
 * Bidirectional communication implemented, so the UI can query `clef` via the stdin/stdout RPC channel. Methods implemented are:
-  - `clef_listWallets` 
+  - `clef_listWallets`
   - `clef_listAccounts`
   - `clef_listWallets`
   - `clef_deriveAccount`
@@ -48,10 +63,10 @@ Changed the namespace format to adhere to the legacy ethereum format: `name_meth
   - `clef_setChainId`
   - `clef_export`
   - `clef_import`
- 
-* The type `Account` was modified (the json-field `type` was removed), to consist of 
 
-```golang
+* The type `Account` was modified (the json-field `type` was removed), to consist of
+
+```go
 type Account struct {
 	Address common.Address `json:"address"` // Ethereum account address derived from the key
 	URL     URL            `json:"url"`     // Optional resource locator within a backend
@@ -64,7 +79,7 @@ type Account struct {
 * Make `ShowError`, `OnApprovedTx`, `OnSignerStartup` be json-rpc [notifications](https://www.jsonrpc.org/specification#notification):
 
 > A Notification is a Request object without an "id" member. A Request object that is a Notification signifies the Client's lack of interest in the corresponding Response object, and as such no Response object needs to be returned to the client. The Server MUST NOT reply to a Notification, including those that are within a batch request.
-> 
+>
 >  Notifications are not confirmable by definition, since they do not have a Response object to be returned. As such, the Client would not be aware of any errors (like e.g. "Invalid params","Internal error"
 ### 3.1.0
 
@@ -79,15 +94,17 @@ type Account struct {
 * Add `OnInputRequired(info UserInputRequest)` to internal API. This method is used when Clef needs user input, e.g. passwords.
 
 The following structures are used:
-```golang
+
-       UserInputRequest struct {
+```go
-               Prompt     string `json:"prompt"`
+UserInputRequest struct {
-               Title      string `json:"title"`
+	Prompt     string `json:"prompt"`
-               IsPassword bool   `json:"isPassword"`
+	Title      string `json:"title"`
-       }
+	IsPassword bool   `json:"isPassword"`
-       UserInputResponse struct {
+}
-               Text string `json:"text"`
+UserInputResponse struct {
-       }
+	Text string `json:"text"`
+}
+```
 
 ### 2.0.0
 
@@ -161,15 +178,3 @@ Example call:
 #### 1.0.0
 
 Initial release.
-
-### Versioning
-
-The API uses [semantic versioning](https://semver.org/).
-
-TLDR; Given a version number MAJOR.MINOR.PATCH, increment the:
-
-* MAJOR version when you make incompatible API changes,
-* MINOR version when you add functionality in a backwards-compatible manner, and
-* PATCH version when you make backwards-compatible bug fixes.
-
-Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format.

cmd/clef/main.go

@@ -53,6 +53,8 @@ import (
 	"github.com/ethereum/go-ethereum/signer/fourbyte"
 	"github.com/ethereum/go-ethereum/signer/rules"
 	"github.com/ethereum/go-ethereum/signer/storage"
+	colorable "github.com/mattn/go-colorable"
+	"github.com/mattn/go-isatty"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -93,7 +95,7 @@ var (
 	chainIdFlag = cli.Int64Flag{
 		Name:  "chainid",
 		Value: params.MainnetChainConfig.ChainID.Int64(),
-		Usage: "Chain id to use for signing (1=mainnet, 3=ropsten, 4=rinkeby, 5=Goerli)",
+		Usage: "Chain id to use for signing (1=mainnet, 3=Ropsten, 4=Rinkeby, 5=Goerli)",
 	}
 	rpcPortFlag = cli.IntFlag{
 		Name:  "rpcport",
@@ -116,8 +118,7 @@ var (
 	}
 	ruleFlag = cli.StringFlag{
 		Name:  "rules",
-		Usage: "Enable rule-engine",
+		Usage: "Path to the rule file to auto-authorize requests with",
-		Value: "",
 	}
 	stdiouiFlag = cli.BoolFlag{
 		Name: "stdio-ui",
@@ -160,7 +161,6 @@ incoming requests.
 Whenever you make an edit to the rule file, you need to use attestation to tell
 Clef that the file is 'safe' to execute.`,
 	}
-
 	setCredentialCommand = cli.Command{
 		Action:    utils.MigrateFlags(setCredential),
 		Name:      "setpw",
@@ -172,8 +172,20 @@ Clef that the file is 'safe' to execute.`,
 			signerSecretFlag,
 		},
 		Description: `
-The setpw command stores a password for a given address (keyfile). If you enter a blank passphrase, it will
+The setpw command stores a password for a given address (keyfile).
-remove any stored credential for that address (keyfile)
+`}
+	delCredentialCommand = cli.Command{
+		Action:    utils.MigrateFlags(removeCredential),
+		Name:      "delpw",
+		Usage:     "Remove a credential for a keystore file",
+		ArgsUsage: "<address>",
+		Flags: []cli.Flag{
+			logLevelFlag,
+			configdirFlag,
+			signerSecretFlag,
+		},
+		Description: `
+The delpw command removes a password for a given address (keyfile).
 `}
 	gendocCommand = cli.Command{
 		Action: GenDoc,
@@ -210,9 +222,9 @@ func init() {
 		advancedMode,
 	}
 	app.Action = signer
-	app.Commands = []cli.Command{initCommand, attestCommand, setCredentialCommand, gendocCommand}
+	app.Commands = []cli.Command{initCommand, attestCommand, setCredentialCommand, delCredentialCommand, gendocCommand}
-
 }
+
 func main() {
 	if err := app.Run(os.Args); err != nil {
 		fmt.Fprintln(os.Stderr, err)
@@ -221,11 +233,20 @@ func main() {
 }
 
 func initializeSecrets(c *cli.Context) error {
+	// Get past the legal message
 	if err := initialize(c); err != nil {
 		return err
 	}
+	// Ensure the master key does not yet exist, we're not willing to overwrite
 	configDir := c.GlobalString(configdirFlag.Name)
-
+	if err := os.Mkdir(configDir, 0700); err != nil && !os.IsExist(err) {
+		return err
+	}
+	location := filepath.Join(configDir, "masterseed.json")
+	if _, err := os.Stat(location); err == nil {
+		return fmt.Errorf("master key %v already exists, will not overwrite", location)
+	}
+	// Key file does not exist yet, generate a new one and encrypt it
 	masterSeed := make([]byte, 256)
 	num, err := io.ReadFull(rand.Reader, masterSeed)
 	if err != nil {
@@ -234,18 +255,18 @@ func initializeSecrets(c *cli.Context) error {
 	if num != len(masterSeed) {
 		return fmt.Errorf("failed to read enough random")
 	}
-
 	n, p := keystore.StandardScryptN, keystore.StandardScryptP
 	if c.GlobalBool(utils.LightKDFFlag.Name) {
 		n, p = keystore.LightScryptN, keystore.LightScryptP
 	}
-	text := "The master seed of clef is locked with a password. Please give a password. Do not forget this password."
+	text := "The master seed of clef will be locked with a password.\nPlease specify a password. Do not forget this password!"
 	var password string
 	for {
 		password = getPassPhrase(text, true)
 		if err := core.ValidatePasswordFormat(password); err != nil {
 			fmt.Printf("invalid password: %v\n", err)
 		} else {
+			fmt.Println()
 			break
 		}
 	}
@@ -253,28 +274,27 @@ func initializeSecrets(c *cli.Context) error {
 	if err != nil {
 		return fmt.Errorf("failed to encrypt master seed: %v", err)
 	}
-
+	// Double check the master key path to ensure nothing wrote there in between
-	err = os.Mkdir(configDir, 0700)
+	if err = os.Mkdir(configDir, 0700); err != nil && !os.IsExist(err) {
-	if err != nil && !os.IsExist(err) {
 		return err
 	}
-	location := filepath.Join(configDir, "masterseed.json")
 	if _, err := os.Stat(location); err == nil {
-		return fmt.Errorf("file %v already exists, will not overwrite", location)
+		return fmt.Errorf("master key %v already exists, will not overwrite", location)
 	}
-	err = ioutil.WriteFile(location, cipherSeed, 0400)
+	// Write the file and print the usual warning message
-	if err != nil {
+	if err = ioutil.WriteFile(location, cipherSeed, 0400); err != nil {
 		return err
 	}
 	fmt.Printf("A master seed has been generated into %s\n", location)
 	fmt.Printf(`
-This is required to be able to store credentials, such as :
+This is required to be able to store credentials, such as:
 * Passwords for keystores (used by rule engine)
-* Storage for javascript rules
+* Storage for JavaScript auto-signing rules
-* Hash of rule-file
+* Hash of JavaScript rule-file
 
-You should treat that file with utmost secrecy, and make a backup of it.
+You should treat 'masterseed.json' with utmost secrecy and make a backup of it!
-NOTE: This file does not contain your accounts. Those need to be backed up separately!
+* The password is necessary but not enough, you need to back up the master seed too!
+* The master seed does not contain your accounts, those need to be backed up separately!
 
 `)
 	return nil
@@ -305,14 +325,18 @@ func attestFile(ctx *cli.Context) error {
 
 func setCredential(ctx *cli.Context) error {
 	if len(ctx.Args()) < 1 {
-		utils.Fatalf("This command requires an address to be passed as an argument.")
+		utils.Fatalf("This command requires an address to be passed as an argument")
 	}
 	if err := initialize(ctx); err != nil {
 		return err
 	}
-
+	addr := ctx.Args().First()
-	address := ctx.Args().First()
+	if !common.IsHexAddress(addr) {
-	password := getPassPhrase("Enter a passphrase to store with this address.", true)
+		utils.Fatalf("Invalid address specified: %s", addr)
+	}
+	address := common.HexToAddress(addr)
+	password := getPassPhrase("Please enter a password to store for this address:", true)
+	fmt.Println()
 
 	stretchedKey, err := readMasterKey(ctx, nil)
 	if err != nil {
@@ -322,10 +346,38 @@ func setCredential(ctx *cli.Context) error {
 	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
 	pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey)
 
-	// Initialize the encrypted storages
 	pwStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey)
-	pwStorage.Put(address, password)
+	pwStorage.Put(address.Hex(), password)
-	log.Info("Credential store updated", "key", address)
+
+	log.Info("Credential store updated", "set", address)
+	return nil
+}
+
+func removeCredential(ctx *cli.Context) error {
+	if len(ctx.Args()) < 1 {
+		utils.Fatalf("This command requires an address to be passed as an argument")
+	}
+	if err := initialize(ctx); err != nil {
+		return err
+	}
+	addr := ctx.Args().First()
+	if !common.IsHexAddress(addr) {
+		utils.Fatalf("Invalid address specified: %s", addr)
+	}
+	address := common.HexToAddress(addr)
+
+	stretchedKey, err := readMasterKey(ctx, nil)
+	if err != nil {
+		utils.Fatalf(err.Error())
+	}
+	configDir := ctx.GlobalString(configdirFlag.Name)
+	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
+	pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey)
+
+	pwStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey)
+	pwStorage.Del(address.Hex())
+
+	log.Info("Credential store updated", "unset", address)
 	return nil
 }
 
@@ -340,13 +392,23 @@ func initialize(c *cli.Context) error {
 		if !confirm(legalWarning) {
 			return fmt.Errorf("aborted by user")
 		}
+		fmt.Println()
 	}
+	usecolor := (isatty.IsTerminal(os.Stderr.Fd()) || isatty.IsCygwinTerminal(os.Stderr.Fd())) && os.Getenv("TERM") != "dumb"
+	output := io.Writer(logOutput)
+	if usecolor {
+		output = colorable.NewColorable(logOutput)
+	}
+	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(c.Int(logLevelFlag.Name)), log.StreamHandler(output, log.TerminalFormat(usecolor))))
 
-	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(c.Int(logLevelFlag.Name)), log.StreamHandler(logOutput, log.TerminalFormat(true))))
 	return nil
 }
 
 func signer(c *cli.Context) error {
+	// If we have some unrecognized command, bail out
+	if args := c.Args(); len(args) > 0 {
+		return fmt.Errorf("invalid command: %q", args[0])
+	}
 	if err := initialize(c); err != nil {
 		return err
 	}
@@ -376,7 +438,7 @@ func signer(c *cli.Context) error {
 
 	configDir := c.GlobalString(configdirFlag.Name)
 	if stretchedKey, err := readMasterKey(c, ui); err != nil {
-		log.Info("No master seed provided, rules disabled", "error", err)
+		log.Warn("Failed to open master, rules disabled", "err", err)
 	} else {
 		vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
 
@@ -390,17 +452,17 @@ func signer(c *cli.Context) error {
 		jsStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "jsstorage.json"), jskey)
 		configStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confkey)
 
-		//Do we have a rule-file?
+		// Do we have a rule-file?
 		if ruleFile := c.GlobalString(ruleFlag.Name); ruleFile != "" {
-			ruleJS, err := ioutil.ReadFile(c.GlobalString(ruleFile))
+			ruleJS, err := ioutil.ReadFile(ruleFile)
 			if err != nil {
-				log.Info("Could not load rulefile, rules not enabled", "file", "rulefile")
+				log.Warn("Could not load rules, disabling", "file", ruleFile, "err", err)
 			} else {
 				shasum := sha256.Sum256(ruleJS)
 				foundShaSum := hex.EncodeToString(shasum[:])
-				storedShasum := configStorage.Get("ruleset_sha256")
+				storedShasum, _ := configStorage.Get("ruleset_sha256")
 				if storedShasum != foundShaSum {
-					log.Info("Could not validate ruleset hash, rules not enabled", "got", foundShaSum, "expected", storedShasum)
+					log.Warn("Rule hash not attested, disabling", "hash", foundShaSum, "attested", storedShasum)
 				} else {
 					// Initialize rules
 					ruleEngine, err := rules.NewRuleEvaluator(ui, jsStorage)
@@ -452,7 +514,6 @@ func signer(c *cli.Context) error {
 			Version:   "1.0"},
 	}
 	if c.GlobalBool(utils.RPCEnabledFlag.Name) {
-
 		vhosts := splitAndTrim(c.GlobalString(utils.RPCVirtualHostsFlag.Name))
 		cors := splitAndTrim(c.GlobalString(utils.RPCCORSDomainFlag.Name))
 
@@ -469,7 +530,6 @@ func signer(c *cli.Context) error {
 			listener.Close()
 			log.Info("HTTP endpoint closed", "url", httpEndpoint)
 		}()
-
 	}
 	if !c.GlobalBool(utils.IPCDisabledFlag.Name) {
 		if c.IsSet(utils.IPCPathFlag.Name) {
@@ -496,8 +556,8 @@ func signer(c *cli.Context) error {
 	}
 	ui.OnSignerStartup(core.StartupInfo{
 		Info: map[string]interface{}{
-			"extapi_version": core.ExternalAPIVersion,
 			"intapi_version": core.InternalAPIVersion,
+			"extapi_version": core.ExternalAPIVersion,
 			"extapi_http":    extapiURL,
 			"extapi_ipc":     ipcapiURL,
 		},
@@ -592,7 +652,6 @@ func readMasterKey(ctx *cli.Context, ui core.UIClientAPI) ([]byte, error) {
 	if len(masterSeed) < 256 {
 		return nil, fmt.Errorf("master seed of insufficient length, expected >255 bytes, got %d", len(masterSeed))
 	}
-
 	// Create vault location
 	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), masterSeed)[:10]))
 	err = os.Mkdir(vaultLocation, 0700)
@@ -620,13 +679,12 @@ func checkFile(filename string) error {
 // confirm displays a text and asks for user confirmation
 func confirm(text string) bool {
 	fmt.Printf(text)
-	fmt.Printf("\nEnter 'ok' to proceed:\n>")
+	fmt.Printf("\nEnter 'ok' to proceed:\n> ")
 
 	text, err := bufio.NewReader(os.Stdin).ReadString('\n')
 	if err != nil {
 		log.Crit("Failed to read user input", "err", err)
 	}
-
 	if text := strings.TrimSpace(text); text == "ok" {
 		return true
 	}
@@ -642,7 +700,7 @@ func testExternalUI(api *core.SignerAPI) {
 
 	a := common.HexToAddress("0xdeadbeef000000000000000000000000deadbeef")
 	addErr := func(errStr string) {
-		log.Info("Test error", "error", errStr)
+		log.Info("Test error", "err", errStr)
 		errs = append(errs, errStr)
 	}
 
@@ -717,8 +775,8 @@ func testExternalUI(api *core.SignerAPI) {
 		data := `{"types":{"EIP712Domain":[{"name":"name","type":"string"},{"name":"version","type":"string"},{"name":"chainId","type":"uint256"},{"name":"verifyingContract","type":"address"}],"Person":[{"name":"name","type":"string"},{"name":"test","type":"uint8"},{"name":"wallet","type":"address"}],"Mail":[{"name":"from","type":"Person"},{"name":"to","type":"Person"},{"name":"contents","type":"string"}]},"primaryType":"Mail","domain":{"name":"Ether Mail","version":"1","chainId":"1","verifyingContract":"0xCCCcccccCCCCcCCCCCCcCcCccCcCCCcCcccccccC"},"message":{"from":{"name":"Cow","test":"3","wallet":"0xcD2a3d9F938E13CD947Ec05AbC7FE734Df8DD826"},"to":{"name":"Bob","wallet":"0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB","test":"2"},"contents":"Hello, Bob!"}}`
 		//_, err := api.SignData(ctx, accounts.MimetypeTypedData, *addr, hexutil.Encode([]byte(data)))
 		var typedData core.TypedData
-		err := json.Unmarshal([]byte(data), &typedData)
+		json.Unmarshal([]byte(data), &typedData)
-		_, err = api.SignTypedData(ctx, *addr, typedData)
+		_, err := api.SignTypedData(ctx, *addr, typedData)
 		expectApprove("sign 712 typed data", err)
 	}
 	{ // Sign data test - plain text
@@ -787,17 +845,17 @@ func testExternalUI(api *core.SignerAPI) {
 // TODO: there are many `getPassPhrase` functions, it will be better to abstract them into one.
 func getPassPhrase(prompt string, confirmation bool) string {
 	fmt.Println(prompt)
-	password, err := console.Stdin.PromptPassword("Passphrase: ")
+	password, err := console.Stdin.PromptPassword("Password: ")
 	if err != nil {
-		utils.Fatalf("Failed to read passphrase: %v", err)
+		utils.Fatalf("Failed to read password: %v", err)
 	}
 	if confirmation {
-		confirm, err := console.Stdin.PromptPassword("Repeat passphrase: ")
+		confirm, err := console.Stdin.PromptPassword("Repeat password: ")
 		if err != nil {
-			utils.Fatalf("Failed to read passphrase confirmation: %v", err)
+			utils.Fatalf("Failed to read password confirmation: %v", err)
 		}
 		if password != confirm {
-			utils.Fatalf("Passphrases do not match")
+			utils.Fatalf("Passwords do not match")
 		}
 	}
 	return password
@@ -864,14 +922,14 @@ func GenDoc(ctx *cli.Context) {
 			"of the work in canonicalizing and making sense of the data, and it's up to the UI to present" +
 			"the user with the contents of the `message`"
 		sighash, msg := accounts.TextAndHash([]byte("hello world"))
-		message := []*core.NameValueType{{"message", msg, accounts.MimetypeTextPlain}}
+		messages := []*core.NameValueType{{"message", msg, accounts.MimetypeTextPlain}}
 
 		add("SignDataRequest", desc, &core.SignDataRequest{
 			Address:     common.NewMixedcaseAddress(a),
 			Meta:        meta,
 			ContentType: accounts.MimetypeTextPlain,
 			Rawdata:     []byte(msg),
-			Message:     message,
+			Messages:    messages,
 			Hash:        sighash})
 	}
 	{ // Sign plain text response
@@ -982,29 +1040,3 @@ These data types are defined in the channel between clef and the UI`)
 		fmt.Println(elem)
 	}
 }
-
-/**
-//Create Account
-
-curl -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_new","params":["test"],"id":67}' localhost:8550
-
-// List accounts
-
-curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_list","params":[""],"id":67}' http://localhost:8550/
-
-// Make Transaction
-// safeSend(0x12)
-// 4401a6e40000000000000000000000000000000000000000000000000000000000000012
-
-// supplied abi
-curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x82A2A876D39022B3019932D30Cd9c97ad5616813","gas":"0x333","gasPrice":"0x123","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x10", "data":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"},"test"],"id":67}' http://localhost:8550/
-
-// Not supplied
-curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x82A2A876D39022B3019932D30Cd9c97ad5616813","gas":"0x333","gasPrice":"0x123","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x10", "data":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"}],"id":67}' http://localhost:8550/
-
-// Sign data
-
-curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_sign","params":["0x694267f14675d7e1b9494fd8d72fefe1755710fa","bazonk gaz baz"],"id":67}' http://localhost:8550/
-
-
-**/

cmd/clef/pythonsigner.py

@@ -42,7 +42,6 @@ class PipeTransport(ServerTransport):
         self.output.write("\n")
 
 class StdIOHandler():
-
     def __init__(self):
         pass
 
@@ -76,7 +75,7 @@ class StdIOHandler():
         :param transaction: transaction info
         :param call_info: info abou the call, e.g. if ABI info could not be
         :param meta: metadata about the request, e.g. where the call comes from
-        :return: 
+        :return:
         """
         transaction = req.get('transaction')
         _from       = req.get('from')
@@ -158,8 +157,7 @@ class StdIOHandler():
         return
 
 def main(args):
-
+    cmd = ["clef", "--stdio-ui"]
-    cmd = ["./clef", "--stdio-ui"]
     if len(args) > 0 and args[0] == "test":
         cmd.extend(["--stdio-ui-test"])
     print("cmd: {}".format(" ".join(cmd)))

cmd/clef/rules.md

@@ -19,32 +19,30 @@ The section below deals with both of them
 A ruleset file is implemented as a `js` file. Under the hood, the ruleset-engine is a `SignerUI`, implementing the same methods as the `json-rpc` methods
 defined in the UI protocol. Example:
 
-```javascript
+```js
-
+function asBig(str) {
-function asBig(str){
+	if (str.slice(0, 2) == "0x") {
-    if(str.slice(0,2) == "0x"){ return new BigNumber(str.slice(2),16)}
+		return new BigNumber(str.slice(2), 16)
-    return new BigNumber(str)
+	}
+	return new BigNumber(str)
 }
 
 // Approve transactions to a certain contract if value is below a certain limit
-function ApproveTx(req){
+function ApproveTx(req) {
-
+	var limit = big.Newint("0xb1a2bc2ec50000")
-    var limit = big.Newint("0xb1a2bc2ec50000")
 	var value = asBig(req.transaction.value);
 
-	if(req.transaction.to.toLowerCase()=="0xae967917c465db8578ca9024c205720b1a3651a9")
+	if (req.transaction.to.toLowerCase() == "0xae967917c465db8578ca9024c205720b1a3651a9") && value.lt(limit)) {
-	    && value.lt(limit) ){
+		return "Approve"
-	    return "Approve"
+	}
-	 }
+	// If we return "Reject", it will be rejected.
-    // If we return "Reject", it will be rejected.
+	// By not returning anything, it will be passed to the next UI, for manual processing
-    // By not returning anything, it will be passed to the next UI, for manual processing
 }
 
-//Approve listings if request made from IPC
+// Approve listings if request made from IPC
 function ApproveListing(req){
     if (req.metadata.scheme == "ipc"){ return "Approve"}
 }
-
 ```
 
 Whenever the external API is called (and the ruleset is enabled), the `signer` calls the UI, which is an instance of a ruleset-engine. The ruleset-engine
@@ -72,7 +70,7 @@ The Otto vm has a few [caveats](https://github.com/robertkrimen/otto):
 Additionally, a few more have been added
 
 * The rule execution cannot load external javascript files.
-* The only preloaded libary is [`bignumber.js`](https://github.com/MikeMcl/bignumber.js) version `2.0.3`. This one is fairly old, and is not aligned with the documentation at the github repository.
+* The only preloaded library is [`bignumber.js`](https://github.com/MikeMcl/bignumber.js) version `2.0.3`. This one is fairly old, and is not aligned with the documentation at the github repository.
 * Each invocation is made in a fresh virtual machine. This means that you cannot store data in global variables between invocations. This is a deliberate choice -- if you want to store data, use the disk-backed `storage`, since rules should not rely on ephemeral data.
 * Javascript API parameters are _always_ an object. This is also a design choice, to ensure that parameters are accessed by _key_ and not by order. This is to prevent mistakes due to missing parameters or parameter changes.
 * The JS engine has access to `storage` and `console`.
@@ -140,97 +138,97 @@ This is now implemented (with ephemeral non-encrypted storage for now, so not ye
 ## Example 1: ruleset for a rate-limited window
 
 
-```javascript
+```js
-
+function big(str) {
-	function big(str){
+	if (str.slice(0, 2) == "0x") {
-		if(str.slice(0,2) == "0x"){ return new BigNumber(str.slice(2),16)}
+		return new BigNumber(str.slice(2), 16)
-		return new BigNumber(str)
 	}
+	return new BigNumber(str)
+}
 
-	// Time window: 1 week
+// Time window: 1 week
-	var window = 1000* 3600*24*7;
+var window = 1000* 3600*24*7;
 
-	// Limit : 1 ether
+// Limit : 1 ether
-	var limit = new BigNumber("1e18");
+var limit = new BigNumber("1e18");
 
-	function isLimitOk(transaction){
+function isLimitOk(transaction) {
-		var value = big(transaction.value)
+	var value = big(transaction.value)
-		// Start of our window function
+	// Start of our window function
-		var windowstart = new Date().getTime() - window;
+	var windowstart = new Date().getTime() - window;
 
-		var txs = [];
+	var txs = [];
-		var stored = storage.Get('txs');
+	var stored = storage.get('txs');
-
-		if(stored != ""){
-			txs = JSON.parse(stored)
-		}
-		// First, remove all that have passed out of the time-window
-		var newtxs = txs.filter(function(tx){return tx.tstamp > windowstart});
-		console.log(txs, newtxs.length);
-
-		// Secondly, aggregate the current sum
-		sum = new BigNumber(0)
-
-		sum = newtxs.reduce(function(agg, tx){ return big(tx.value).plus(agg)}, sum);
-		console.log("ApproveTx > Sum so far", sum);
-		console.log("ApproveTx > Requested", value.toNumber());
-
-		// Would we exceed weekly limit ?
-		return sum.plus(value).lt(limit)
 
+	if (stored != "") {
+		txs = JSON.parse(stored)
 	}
-	function ApproveTx(r){
+	// First, remove all that have passed out of the time-window
-		if (isLimitOk(r.transaction)){
+	var newtxs = txs.filter(function(tx){return tx.tstamp > windowstart});
-			return "Approve"
+	console.log(txs, newtxs.length);
-		}
-		return "Nope"
-	}
 
-	/**
+	// Secondly, aggregate the current sum
-	* OnApprovedTx(str) is called when a transaction has been approved and signed. The parameter
+	sum = new BigNumber(0)
- 	* 'response_str' contains the return value that will be sent to the external caller.
-	* The return value from this method is ignore - the reason for having this callback is to allow the
-	* ruleset to keep track of approved transactions.
-	*
-	* When implementing rate-limited rules, this callback should be used.
-	* If a rule responds with neither 'Approve' nor 'Reject' - the tx goes to manual processing. If the user
-	* then accepts the transaction, this method will be called.
-	*
-	* TLDR; Use this method to keep track of signed transactions, instead of using the data in ApproveTx.
-	*/
- 	function OnApprovedTx(resp){
-		var value = big(resp.tx.value)
-		var txs = []
-		// Load stored transactions
-		var stored = storage.Get('txs');
-		if(stored != ""){
-			txs = JSON.parse(stored)
-		}
-		// Add this to the storage
-		txs.push({tstamp: new Date().getTime(), value: value});
-		storage.Put("txs", JSON.stringify(txs));
-	}
 
+	sum = newtxs.reduce(function(agg, tx){ return big(tx.value).plus(agg)}, sum);
+	console.log("ApproveTx > Sum so far", sum);
+	console.log("ApproveTx > Requested", value.toNumber());
+
+	// Would we exceed weekly limit ?
+	return sum.plus(value).lt(limit)
+
+}
+function ApproveTx(r) {
+	if (isLimitOk(r.transaction)) {
+		return "Approve"
+	}
+	return "Nope"
+}
+
+/**
+* OnApprovedTx(str) is called when a transaction has been approved and signed. The parameter
+	* 'response_str' contains the return value that will be sent to the external caller.
+* The return value from this method is ignore - the reason for having this callback is to allow the
+* ruleset to keep track of approved transactions.
+*
+* When implementing rate-limited rules, this callback should be used.
+* If a rule responds with neither 'Approve' nor 'Reject' - the tx goes to manual processing. If the user
+* then accepts the transaction, this method will be called.
+*
+* TLDR; Use this method to keep track of signed transactions, instead of using the data in ApproveTx.
+*/
+function OnApprovedTx(resp) {
+	var value = big(resp.tx.value)
+	var txs = []
+	// Load stored transactions
+	var stored = storage.get('txs');
+	if (stored != "") {
+		txs = JSON.parse(stored)
+	}
+	// Add this to the storage
+	txs.push({tstamp: new Date().getTime(), value: value});
+	storage.put("txs", JSON.stringify(txs));
+}
 ```
 
 ## Example 2: allow destination
 
-```javascript
+```js
-
+function ApproveTx(r) {
-	function ApproveTx(r){
+	if (r.transaction.from.toLowerCase() == "0x0000000000000000000000000000000000001337") {
-		if(r.transaction.from.toLowerCase()=="0x0000000000000000000000000000000000001337"){ return "Approve"}
+		return "Approve"
-		if(r.transaction.from.toLowerCase()=="0x000000000000000000000000000000000000dead"){ return "Reject"}
-		// Otherwise goes to manual processing
 	}
-
+	if (r.transaction.from.toLowerCase() == "0x000000000000000000000000000000000000dead") {
+		return "Reject"
+	}
+	// Otherwise goes to manual processing
+}
 ```
 
 ## Example 3: Allow listing
 
-```javascript
+```js
-
+function ApproveListing() {
-    function ApproveListing(){
+	return "Approve"
-        return "Approve"
+}
-    }
+```
-
-```

cmd/clef/tests/testsigner.js

@@ -1,3 +1,19 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
 // This file is a test-utility for testing clef-functionality
 //
 // Start clef with

cmd/clef/tutorial.md

@@ -1,200 +1,278 @@
-## Initializing the signer
+## Initializing Clef
 
-First, initialize the master seed.
+First thing's first, Clef needs to store some data itself. Since that data might be sensitive (passwords, signing rules, accounts), Clef's entire storage is encrypted. To support encrypting data, the first step is to initialize Clef with a random master seed, itself too encrypted with your chosen password:
 
 ```text
-#./signer init
+$ clef init
 
 WARNING!
 
-The signer is alpha software, and not yet publically released. This software has _not_ been audited, and there
+Clef is an account management tool. It may, like any software, contain bugs.
-are no guarantees about the workings of this software. It may contain severe flaws. You should not use this software
-unless you agree to take full responsibility for doing so, and know what you are doing.
 
-TLDR; THIS IS NOT PRODUCTION-READY SOFTWARE!
+Please take care to
+- backup your keystore files,
+- verify that the keystore(s) can be opened with your password.
 
+Clef is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+PURPOSE. See the GNU General Public License for more details.
 
 Enter 'ok' to proceed:
->ok
+> ok
-A master seed has been generated into /home/martin/.signer/secrets.dat
 
-This is required to be able to store credentials, such as :
+The master seed of clef will be locked with a password.
+Please specify a password. Do not forget this password!
+Password:
+Repeat password:
+
+A master seed has been generated into /home/martin/.clef/masterseed.json
+
+This is required to be able to store credentials, such as:
 * Passwords for keystores (used by rule engine)
-* Storage for javascript rules
+* Storage for JavaScript auto-signing rules
-* Hash of rule-file
+* Hash of JavaScript rule-file
 
-You should treat that file with utmost secrecy, and make a backup of it.
+You should treat 'masterseed.json' with utmost secrecy and make a backup of it!
-NOTE: This file does not contain your accounts. Those need to be backed up separately!
+* The password is necessary but not enough, you need to back up the master seed too!
+* The master seed does not contain your accounts, those need to be backed up separately!
 ```
 
-(for readability purposes, we'll remove the WARNING printout in the rest of this document)
+*For readability purposes, we'll remove the WARNING printout, user confirmation and the unlocking of the master seed in the rest of this document.*
 
-## Creating rules
+## Remote interactions
 
-Now, you can create a rule-file. Note that it is not mandatory to use predefined rules, but it's really handy.
+Clef is capable of managing both key-file based accounts as well as hardware wallets. To evaluate clef, we're going to point it to our Rinkeby testnet keystore and specify the Rinkeby chain ID for signing (Clef doesn't have a backing chain, so it doesn't know what network it runs on).
 
-```javascript
+```text
-function ApproveListing(){
+$ clef --keystore ~/.ethereum/rinkeby/keystore --chainid 4
+
+INFO [07-01|11:00:46.385] Starting signer                          chainid=4 keystore=$HOME/.ethereum/rinkeby/keystore light-kdf=false advanced=false
+DEBUG[07-01|11:00:46.389] FS scan times                            list=3.521941ms set=9.017µs diff=4.112µs
+DEBUG[07-01|11:00:46.391] Ledger support enabled
+DEBUG[07-01|11:00:46.391] Trezor support enabled via HID
+DEBUG[07-01|11:00:46.391] Trezor support enabled via WebUSB
+INFO [07-01|11:00:46.391] Audit logs configured                    file=audit.log
+DEBUG[07-01|11:00:46.392] IPC registered                           namespace=account
+INFO [07-01|11:00:46.392] IPC endpoint opened                      url=$HOME/.clef/clef.ipc
+------- Signer info -------
+* intapi_version : 7.0.0
+* extapi_version : 6.0.0
+* extapi_http : n/a
+* extapi_ipc : $HOME/.clef/clef.ipc
+```
+
+By default, Clef starts up in CLI (Command Line Interface) mode. Arbitrary remote processes may *request* account interactions (e.g. sign a transaction), which the user will need to individually *confirm*.
+
+To test this out, we can *request* Clef to list all account via its *External API endpoint*:
+
+```text
+echo '{"id": 1, "jsonrpc": "2.0", "method": "account_list"}' | nc -U ~/.clef/clef.ipc
+```
+
+This will prompt the user within the Clef CLI to confirm or deny the request:
+
+```text
+-------- List Account request--------------
+A request has been made to list all accounts.
+You can select which accounts the caller can see
+  [x] 0xD9C9Cd5f6779558b6e0eD4e6Acf6b1947E7fA1F3
+    URL: keystore://$HOME/.ethereum/rinkeby/keystore/UTC--2017-04-14T15-15-00.327614556Z--d9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3
+  [x] 0x086278A6C067775F71d6B2BB1856Db6E28c30418
+    URL: keystore://$HOME/.ethereum/rinkeby/keystore/UTC--2018-02-06T22-53-11.211657239Z--086278a6c067775f71d6b2bb1856db6e28c30418
+-------------------------------------------
+Request context:
+	NA -> NA -> NA
+
+Additional HTTP header data, provided by the external caller:
+	User-Agent:
+	Origin:
+Approve? [y/N]:
+>
+```
+
+Depending on whether we approve or deny the request, the original NetCat process will get:
+
+```text
+{"jsonrpc":"2.0","id":1,"result":["0xd9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3","0x086278a6c067775f71d6b2bb1856db6e28c30418"]}
+
+or
+
+{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"Request denied"}}
+```
+
+Apart from listing accounts, you can also *request* creating a new account; signing transactions and data; and recovering signatures. You can find the available methods in the Clef [External API Spec](https://github.com/ethereum/go-ethereum/tree/master/cmd/clef#external-api-1) and the [External API Changelog](https://github.com/ethereum/go-ethereum/blob/master/cmd/clef/extapi_changelog.md).
+
+*Note, the number of things you can do from the External API is deliberately small, since we want to limit the power of remote calls by as much as possible! Clef has an [Internal API](https://github.com/ethereum/go-ethereum/tree/master/cmd/clef#ui-api-1) too for the UI (User Interface) which is much richer and can support custom interfaces on top. But that's out of scope here.*
+
+## Automatic rules
+
+For most users, manually confirming every transaction is the way to go. However, there are cases when it makes sense to set up some rules which permit Clef to sign a transaction without prompting the user. One such example would be running a signer on Rinkeby or other PoA networks.
+
+For starters, we can create a rule file that automatically permits anyone to list our available accounts without user confirmation. The rule file is a tiny JavaScript snippet that you can program however you want:
+
+```js
+function ApproveListing() {
     return "Approve"
 }
 ```
 
-Get the `sha256` hash. If you have openssl, you can do `openssl sha256 rules.js`...
+Of course, Clef isn't going to just accept and run arbitrary scripts you give it, that would be dangerous if someone changes your rule file! Instead, you need to explicitly *attest* the rule file, which entails injecting its hash into Clef's secure store.
-```text
-#sha256sum rules.js
-6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72  rules.js
-```
-...now `attest` the file...
-```text
-#./signer attest 6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72
 
-INFO [02-21|12:14:38] Ruleset attestation updated              sha256=6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72
+```text
+$ sha256sum rules.js
+645b58e4f945e24d0221714ff29f6aa8e860382ced43490529db1695f5fcc71c  rules.js
+
+$ clef attest 645b58e4f945e24d0221714ff29f6aa8e860382ced43490529db1695f5fcc71c
+Decrypt master seed of clef
+Password:
+INFO [07-01|13:25:03.290] Ruleset attestation updated              sha256=645b58e4f945e24d0221714ff29f6aa8e860382ced43490529db1695f5fcc71c
 ```
 
-...and (this is required only for non-production versions) load a mock-up `4byte.json` by copying the file from the source to your current working directory:
+At this point, we can start Clef with the rule file:
-```text
-#cp $GOPATH/src/github.com/ethereum/go-ethereum/cmd/clef/4byte.json $PWD
-```
 
-At this point, we can start the signer with the rule-file:
 ```text
-#./signer --rules rules.js --rpc
+$ clef --keystore ~/.ethereum/rinkeby/keystore --chainid 4 --rules rules.js
 
-INFO [09-25|20:28:11.866] Using CLI as UI-channel 
+INFO [07-01|13:39:49.726] Rule engine configured                   file=rules.js
-INFO [09-25|20:28:11.876] Loaded 4byte db                          signatures=5509 file=./4byte.json
+INFO [07-01|13:39:49.726] Starting signer                          chainid=4 keystore=$HOME/.ethereum/rinkeby/keystore light-kdf=false advanced=false
-INFO [09-25|20:28:11.877] Rule engine configured                   file=./rules.js
+DEBUG[07-01|13:39:49.726] FS scan times                            list=35.15µs set=4.251µs diff=2.766µs
-DEBUG[09-25|20:28:11.877] FS scan times                            list=100.781µs set=13.253µs diff=5.761µs
+DEBUG[07-01|13:39:49.727] Ledger support enabled
-DEBUG[09-25|20:28:11.884] Ledger support enabled 
+DEBUG[07-01|13:39:49.727] Trezor support enabled via HID
-DEBUG[09-25|20:28:11.888] Trezor support enabled 
+DEBUG[07-01|13:39:49.727] Trezor support enabled via WebUSB
-INFO [09-25|20:28:11.888] Audit logs configured                    file=audit.log
+INFO [07-01|13:39:49.728] Audit logs configured                    file=audit.log
-DEBUG[09-25|20:28:11.888] HTTP registered                          namespace=account
+DEBUG[07-01|13:39:49.728] IPC registered                           namespace=account
-INFO [09-25|20:28:11.890] HTTP endpoint opened                     url=http://localhost:8550
+INFO [07-01|13:39:49.728] IPC endpoint opened                      url=$HOME/.clef/clef.ipc
-DEBUG[09-25|20:28:11.890] IPC registered                           namespace=account
-INFO [09-25|20:28:11.890] IPC endpoint opened                      url=<nil>
 ------- Signer info -------
-* extapi_version : 2.0.0
+* intapi_version : 7.0.0
-* intapi_version : 2.0.0
+* extapi_version : 6.0.0
-* extapi_http : http://localhost:8550
+* extapi_http : n/a
-* extapi_ipc : <nil>
+* extapi_ipc : $HOME/.clef/clef.ipc
 ```
 
-Any list-requests will now be auto-approved by our rule-file.
+Any account listing *request* will now be auto-approved by the rule file:
+
+```text
+$ echo '{"id": 1, "jsonrpc": "2.0", "method": "account_list"}' | nc -U ~/.clef/clef.ipc
+{"jsonrpc":"2.0","id":1,"result":["0xd9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3","0x086278a6c067775f71d6b2bb1856db6e28c30418"]}
+```
 
 ## Under the hood
 
 While doing the operations above, these files have been created:
 
 ```text
-#ls -laR ~/.signer/
+$ ls -laR ~/.clef/
-/home/martin/.signer/:
-total 16
-drwx------  3 martin martin 4096 feb 21 12:14 .
-drwxr-xr-x 71 martin martin 4096 feb 21 12:12 ..
-drwx------  2 martin martin 4096 feb 21 12:14 43f73718397aa54d1b22
--rwx------  1 martin martin  256 feb 21 12:12 secrets.dat
 
-/home/martin/.signer/43f73718397aa54d1b22:
+$HOME/.clef/:
+total 24
+drwxr-x--x   3 user user  4096 Jul  1 13:45 .
+drwxr-xr-x 102 user user 12288 Jul  1 13:39 ..
+drwx------   2 user user  4096 Jul  1 13:25 02f90c0603f4f2f60188
+-r--------   1 user user   868 Jun 28 13:55 masterseed.json
+
+$HOME/.clef/02f90c0603f4f2f60188:
 total 12
-drwx------ 2 martin martin 4096 feb 21 12:14 .
+drwx------ 2 user user 4096 Jul  1 13:25 .
-drwx------ 3 martin martin 4096 feb 21 12:14 ..
+drwxr-x--x 3 user user 4096 Jul  1 13:45 ..
--rw------- 1 martin martin  159 feb 21 12:14 config.json
+-rw------- 1 user user  159 Jul  1 13:25 config.json
-
-#cat /home/martin/.signer/43f73718397aa54d1b22/config.json
-{"ruleset_sha256":{"iv":"6v4W4tfJxj3zZFbl","c":"6dt5RTDiTq93yh1qDEjpsat/tsKG7cb+vr3sza26IPL2fvsQ6ZoqFx++CPUa8yy6fD9Bbq41L01ehkKHTG3pOAeqTW6zc/+t0wv3AB6xPmU="}}
 
+$ cat ~/.clef/02f90c0603f4f2f60188/config.json
+{"ruleset_sha256":{"iv":"SWWEtnl+R+I+wfG7","c":"I3fjmwmamxVcfGax7D0MdUOL29/rBWcs73WBILmYK0o1CrX7wSMc3y37KsmtlZUAjp0oItYq01Ow8VGUOzilG91tDHInB5YHNtm/YkufEbo="}}
 ```
 
-In `~/.signer`, the `secrets.dat` file was created, containing the `master_seed`.
+In `$HOME/.clef`, the `masterseed.json` file was created, containing the master seed. This seed was then used to derive a few other things:
-The `master_seed` was then used to derive a few other things:
 
-- `vault_location` : in this case `43f73718397aa54d1b22` .
+- **Vault location**: in this case `02f90c0603f4f2f60188`.
-   - Thus, if you use a different `master_seed`, another `vault_location` will be used that does not conflict with each other.
+   - If you use a different master seed, a different vault location will be used that does not conflict with each other (e.g. `clef --signersecret /path/to/file`). This allows you to run multiple instances of Clef, each with its own rules (e.g. mainnet + testnet).
-   - Example: `signer --signersecret /path/to/afile ...`
+- **`config.json`**: the encrypted key/value storage for configuration data, currently only containing the key `ruleset_sha256`, the attested hash of the automatic rules to use.
-- `config.json` which is the encrypted key/value storage for configuration data, containing the key `ruleset_sha256`.
 
+## Advanced rules
 
-## Adding credentials
+In order to make more useful rules - like signing transactions - the signer needs access to the passwords needed to unlock keys from the keystore. You can inject an unlock password via `clef setpw`.
-
-In order to make more useful rules like signing transactions, the signer needs access to the passwords needed to unlock keystores.
 
 ```text
-#./signer addpw "0x694267f14675d7e1b9494fd8d72fefe1755710fa" "test_password"
+$ clef setpw 0xd9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3
 
-INFO [02-21|13:43:21] Credential store updated                 key=0x694267f14675d7e1b9494fd8d72fefe1755710fa
+Please enter a password to store for this address:
+Password:
+Repeat password:
+
+Decrypt master seed of clef
+Password:
+INFO [07-01|14:05:56.031] Credential store updated                 key=0xd9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3
 ```
-## More advanced rules
 
-Now let's update the rules to make use of credentials:
+Now let's update the rules to make use of the new credentials:
 
-```javascript
+```js
-function ApproveListing(){
+function ApproveListing() {
     return "Approve"
 }
-function ApproveSignData(r){
+
-    if( r.address.toLowerCase() == "0x694267f14675d7e1b9494fd8d72fefe1755710fa")
+function ApproveSignData(req) {
-    {
+    if (req.address.toLowerCase() == "0xd9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3") {
-        if(r.message.indexOf("bazonk") >= 0){
+        if (req.messages[0].value.indexOf("bazonk") >= 0) {
             return "Approve"
         }
         return "Reject"
     }
     // Otherwise goes to manual processing
 }
-
 ```
+
 In this example:
-* Any requests to sign data with the account `0x694...` will be
-    * auto-approved if the message contains with `bazonk`
-    * auto-rejected if it does not.
-* Any other signing-requests will be passed along for manual approve/reject.
 
-_Note: make sure that `0x694...` is an account you have access to. You can create it either via the clef or the traditional account cli tool. If the latter was chosen, make sure both clef and geth use the same keystore by specifing `--keystore path/to/your/keystore` when running clef._
+- Any requests to sign data with the account `0xd9c9...` will be:
+    - Auto-approved if the message contains `bazonk`,
+    - Auto-rejected if the message does not contain `bazonk`,
+- Any other requests will be passed along for manual confirmation.
+
+*Note, to make this example work, please use you own accounts. You can create a new account either via Clef or the traditional account CLI tools. If the latter was chosen, make sure both Clef and Geth use the same keystore by specifying `--keystore path/to/your/keystore` when running Clef.*
+
+Attest the new rule file so that Clef will accept loading it:
 
-Attest the new file...
 ```text
-#sha256sum rules.js
+$ sha256sum rules.js
-2a0cb661dacfc804b6e95d935d813fd17c0997a7170e4092ffbc34ca976acd9f  rules.js
+f163a1738b649259bb9b369c593fdc4c6b6f86cc87e343c3ba58faee03c2a178  rules.js
 
-#./signer attest 2a0cb661dacfc804b6e95d935d813fd17c0997a7170e4092ffbc34ca976acd9f
+$ clef attest f163a1738b649259bb9b369c593fdc4c6b6f86cc87e343c3ba58faee03c2a178
-
+Decrypt master seed of clef
-INFO [02-21|14:36:30] Ruleset attestation updated              sha256=2a0cb661dacfc804b6e95d935d813fd17c0997a7170e4092ffbc34ca976acd9f
+Password:
+INFO [07-01|14:11:28.509] Ruleset attestation updated              sha256=f163a1738b649259bb9b369c593fdc4c6b6f86cc87e343c3ba58faee03c2a178
 ```
 
-And start the signer:
+Restart Clef with the new rules in place:
 
 ```
-#./signer --rules rules.js --rpc
+$ clef --keystore ~/.ethereum/rinkeby/keystore --chainid 4 --rules rules.js
 
-INFO [09-25|21:02:16.450] Using CLI as UI-channel 
+INFO [07-01|14:12:41.636] Rule engine configured                   file=rules.js
-INFO [09-25|21:02:16.466] Loaded 4byte db                          signatures=5509 file=./4byte.json
+INFO [07-01|14:12:41.636] Starting signer                          chainid=4 keystore=$HOME/.ethereum/rinkeby/keystore light-kdf=false advanced=false
-INFO [09-25|21:02:16.467] Rule engine configured                   file=./rules.js
+DEBUG[07-01|14:12:41.636] FS scan times                            list=46.722µs set=4.47µs diff=2.157µs
-DEBUG[09-25|21:02:16.468] FS scan times                            list=1.45262ms set=21.926µs diff=6.944µs
+DEBUG[07-01|14:12:41.637] Ledger support enabled
-DEBUG[09-25|21:02:16.473] Ledger support enabled 
+DEBUG[07-01|14:12:41.637] Trezor support enabled via HID
-DEBUG[09-25|21:02:16.475] Trezor support enabled 
+DEBUG[07-01|14:12:41.638] Trezor support enabled via WebUSB
-INFO [09-25|21:02:16.476] Audit logs configured                    file=audit.log
+INFO [07-01|14:12:41.638] Audit logs configured                    file=audit.log
-DEBUG[09-25|21:02:16.476] HTTP registered                          namespace=account
+DEBUG[07-01|14:12:41.638] IPC registered                           namespace=account
-INFO [09-25|21:02:16.478] HTTP endpoint opened                     url=http://localhost:8550
+INFO [07-01|14:12:41.638] IPC endpoint opened                      url=$HOME/.clef/clef.ipc
-DEBUG[09-25|21:02:16.478] IPC registered                           namespace=account
-INFO [09-25|21:02:16.478] IPC endpoint opened                      url=<nil>
 ------- Signer info -------
-* extapi_version : 2.0.0
+* intapi_version : 7.0.0
-* intapi_version : 2.0.0
+* extapi_version : 6.0.0
-* extapi_http : http://localhost:8550
+* extapi_http : n/a
-* extapi_ipc : <nil>
+* extapi_ipc : $HOME/.clef/clef.ipc
 ```
 
-And then test signing, once with `bazonk` and once without:
+Then test signing, once with `bazonk` and once without:
 
 ```
-#curl -H "Content-Type: application/json" -X POST --data "{\"jsonrpc\":\"2.0\",\"method\":\"account_sign\",\"params\":[\"0x694267f14675d7e1b9494fd8d72fefe1755710fa\",\"0x$(xxd -pu <<< '  bazonk baz gaz')\"],\"id\":67}" http://localhost:8550/
+$ echo '{"id": 1, "jsonrpc":"2.0", "method":"account_signData", "params":["data/plain", "0xd9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3", "0x202062617a6f6e6b2062617a2067617a0a"]}' | nc -U ~/.clef/clef.ipc
-{"jsonrpc":"2.0","id":67,"result":"0x93e6161840c3ae1efc26dc68dedab6e8fc233bb3fefa1b4645dbf6609b93dace160572ea4ab33240256bb6d3dadb60dcd9c515d6374d3cf614ee897408d41d541c"}
+{"jsonrpc":"2.0","id":1,"result":"0x4f93e3457027f6be99b06b3392d0ebc60615ba448bb7544687ef1248dea4f5317f789002df783979c417d969836b6fda3710f5bffb296b4d51c8aaae6e2ac4831c"}
-
-#curl -H "Content-Type: application/json" -X POST --data "{\"jsonrpc\":\"2.0\",\"method\":\"account_sign\",\"params\":[\"0x694267f14675d7e1b9494fd8d72fefe1755710fa\",\"0x$(xxd -pu <<< '  bonk baz gaz')\"],\"id\":67}" http://localhost:8550/
-{"jsonrpc":"2.0","id":67,"error":{"code":-32000,"message":"Request denied"}}
 
+$ echo '{"id": 1, "jsonrpc":"2.0", "method":"account_signData", "params":["data/plain", "0xd9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3", "0x2020626f6e6b2062617a2067617a0a"]}' | nc -U ~/.clef/clef.ipc
+{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"Request denied"}}
 ```
 
-Meanwhile, in the signer output:
+Meanwhile, in the Clef output log you can see:
 ```text
 INFO [02-21|14:42:41] Op approved
 INFO [02-21|14:42:56] Op rejected
@@ -203,9 +281,73 @@ INFO [02-21|14:42:56] Op rejected
 The signer also stores all traffic over the external API in a log file. The last 4 lines shows the two requests and their responses:
 
 ```text
-#tail -n 4 audit.log
+$ tail -n 4 audit.log
-t=2018-02-21T14:42:41+0100 lvl=info msg=Sign       api=signer type=request  metadata="{\"remote\":\"127.0.0.1:49706\",\"local\":\"localhost:8550\",\"scheme\":\"HTTP/1.1\"}" addr="0x694267f14675d7e1b9494fd8d72fefe1755710fa [chksum INVALID]" data=202062617a6f6e6b2062617a2067617a0a
+t=2019-07-01T15:52:14+0300 lvl=info msg=SignData   api=signer type=request  metadata="{\"remote\":\"NA\",\"local\":\"NA\",\"scheme\":\"NA\",\"User-Agent\":\"\",\"Origin\":\"\"}" addr="0xd9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3 [chksum INVALID]" data=0x202062617a6f6e6b2062617a2067617a0a content-type=data/plain
-t=2018-02-21T14:42:42+0100 lvl=info msg=Sign       api=signer type=response data=93e6161840c3ae1efc26dc68dedab6e8fc233bb3fefa1b4645dbf6609b93dace160572ea4ab33240256bb6d3dadb60dcd9c515d6374d3cf614ee897408d41d541c error=nil
+t=2019-07-01T15:52:14+0300 lvl=info msg=SignData   api=signer type=response data=4f93e3457027f6be99b06b3392d0ebc60615ba448bb7544687ef1248dea4f5317f789002df783979c417d969836b6fda3710f5bffb296b4d51c8aaae6e2ac4831c error=nil
-t=2018-02-21T14:42:56+0100 lvl=info msg=Sign       api=signer type=request  metadata="{\"remote\":\"127.0.0.1:49708\",\"local\":\"localhost:8550\",\"scheme\":\"HTTP/1.1\"}" addr="0x694267f14675d7e1b9494fd8d72fefe1755710fa [chksum INVALID]" data=2020626f6e6b2062617a2067617a0a
+t=2019-07-01T15:52:23+0300 lvl=info msg=SignData   api=signer type=request  metadata="{\"remote\":\"NA\",\"local\":\"NA\",\"scheme\":\"NA\",\"User-Agent\":\"\",\"Origin\":\"\"}" addr="0xd9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3 [chksum INVALID]" data=0x2020626f6e6b2062617a2067617a0a     content-type=data/plain
-t=2018-02-21T14:42:56+0100 lvl=info msg=Sign       api=signer type=response data=                                                                                                                                   error="Request denied"
+t=2019-07-01T15:52:23+0300 lvl=info msg=SignData   api=signer type=response data=                                     error="Request denied"
-```
+```
+
+For more details on writing automatic rules, please see the [rules spec](https://github.com/ethereum/go-ethereum/blob/master/cmd/clef/rules.md).
+
+## Geth integration
+
+Of course, as awesome as Clef is, it's not feasible to interact with it via JSON RPC by hand. Long term, we're hoping to convince the general Ethereum community to support Clef as a general signer (it's only 3-5 methods), thus allowing your favorite DApp, Metamask, MyCrypto, etc to request signatures directly.
+
+Until then however, we're trying to pave the way via Geth. Geth v1.9.0 has built in support via `--signer <API endpoint>` for using a local or remote Clef instance as an account backend!
+
+We can try this by running Clef with our previous rules on Rinkeby (for now it's a good idea to allow auto-listing accounts, since Geth likes to retrieve them once in a while).
+
+```text
+$ clef --keystore ~/.ethereum/rinkeby/keystore --chainid 4 --rules rules.js
+```
+
+In a different window we can start Geth, list our accounts, even list our wallets to see where the accounts originate from:
+
+```text
+$ geth --rinkeby --signer=~/.clef/clef.ipc console
+
+> eth.accounts
+["0xd9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3", "0x086278a6c067775f71d6b2bb1856db6e28c30418"]
+
+> personal.listWallets
+[{
+    accounts: [{
+        address: "0xd9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3",
+        url: "extapi://$HOME/.clef/clef.ipc"
+    }, {
+        address: "0x086278a6c067775f71d6b2bb1856db6e28c30418",
+        url: "extapi://$HOME/.clef/clef.ipc"
+    }],
+    status: "ok [version=6.0.0]",
+    url: "extapi://$HOME/.clef/clef.ipc"
+}]
+
+> eth.sendTransaction({from: eth.accounts[0], to: eth.accounts[0]})
+```
+
+Lastly, when we requested a transaction to be sent, Clef prompted us in the original window to approve it:
+
+```text
+--------- Transaction request-------------
+to:       0xD9C9Cd5f6779558b6e0eD4e6Acf6b1947E7fA1F3
+from:     0xD9C9Cd5f6779558b6e0eD4e6Acf6b1947E7fA1F3 [chksum ok]
+value:    0 wei
+gas:      0x5208 (21000)
+gasprice: 1000000000 wei
+nonce:    0x2366 (9062)
+
+Request context:
+	NA -> NA -> NA
+
+Additional HTTP header data, provided by the external caller:
+	User-Agent:
+	Origin:
+-------------------------------------------
+Approve? [y/N]:
+> y
+```
+
+:boom:
+
+*Note, if you enable the external signer backend in Geth, all other account management is disabled. This is because long term we want to remove account management from Geth.*

cmd/ethkey/README.md

@@ -35,18 +35,18 @@ It is possible to refer to a file containing the message.
 To sign a message contained in a file, use the --msgfile flag.
 
 
-### `ethkey changepassphrase <keyfile>`
+### `ethkey changepassword <keyfile>`
 
-Change the passphrase of a keyfile.
+Change the password of a keyfile.
 use the `--newpasswordfile` to point to the new password file.
 
 
-## Passphrases
+## Passwords
 
 For every command that uses a keyfile, you will be prompted to provide the 
-passphrase for decrypting the keyfile.  To avoid this message, it is possible
+password for decrypting the keyfile.  To avoid this message, it is possible
-to pass the passphrase by using the `--passwordfile` flag pointing to a file that
+to pass the password by using the `--passwordfile` flag pointing to a file that
-contains the passphrase.
+contains the password.
 
 ## JSON
 

cmd/ethkey/changepassword.go

@@ -1,3 +1,19 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
 package main
 
 import (
@@ -12,15 +28,15 @@ import (
 
 var newPassphraseFlag = cli.StringFlag{
 	Name:  "newpasswordfile",
-	Usage: "the file that contains the new passphrase for the keyfile",
+	Usage: "the file that contains the new password for the keyfile",
 }
 
 var commandChangePassphrase = cli.Command{
-	Name:      "changepassphrase",
+	Name:      "changepassword",
-	Usage:     "change the passphrase on a keyfile",
+	Usage:     "change the password on a keyfile",
 	ArgsUsage: "<keyfile>",
 	Description: `
-Change the passphrase of a keyfile.`,
+Change the password of a keyfile.`,
 	Flags: []cli.Flag{
 		passphraseFlag,
 		newPassphraseFlag,
@@ -42,12 +58,12 @@ Change the passphrase of a keyfile.`,
 		}
 
 		// Get a new passphrase.
-		fmt.Println("Please provide a new passphrase")
+		fmt.Println("Please provide a new password")
 		var newPhrase string
 		if passFile := ctx.String(newPassphraseFlag.Name); passFile != "" {
 			content, err := ioutil.ReadFile(passFile)
 			if err != nil {
-				utils.Fatalf("Failed to read new passphrase file '%s': %v", passFile, err)
+				utils.Fatalf("Failed to read new password file '%s': %v", passFile, err)
 			}
 			newPhrase = strings.TrimRight(string(content), "\r\n")
 		} else {
@@ -57,7 +73,7 @@ Change the passphrase of a keyfile.`,
 		// Encrypt the key with the new passphrase.
 		newJson, err := keystore.EncryptKey(key, newPhrase, keystore.StandardScryptN, keystore.StandardScryptP)
 		if err != nil {
-			utils.Fatalf("Error encrypting with new passphrase: %v", err)
+			utils.Fatalf("Error encrypting with new password: %v", err)
 		}
 
 		// Then write the new keyfile in place of the old one.

cmd/ethkey/main.go

@@ -49,7 +49,7 @@ func init() {
 var (
 	passphraseFlag = cli.StringFlag{
 		Name:  "passwordfile",
-		Usage: "the file that contains the passphrase for the keyfile",
+		Usage: "the file that contains the password for the keyfile",
 	}
 	jsonFlag = cli.BoolFlag{
 		Name:  "json",

cmd/ethkey/message_test.go

@@ -37,8 +37,8 @@ func TestMessageSignVerify(t *testing.T) {
 	generate := runEthkey(t, "generate", keyfile)
 	generate.Expect(`
 !! Unsupported terminal, password will be echoed.
-Passphrase: {{.InputLine "foobar"}}
+Password: {{.InputLine "foobar"}}
-Repeat passphrase: {{.InputLine "foobar"}}
+Repeat password: {{.InputLine "foobar"}}
 `)
 	_, matches := generate.ExpectRegexp(`Address: (0x[0-9a-fA-F]{40})\n`)
 	address := matches[1]
@@ -48,7 +48,7 @@ Repeat passphrase: {{.InputLine "foobar"}}
 	sign := runEthkey(t, "signmessage", keyfile, message)
 	sign.Expect(`
 !! Unsupported terminal, password will be echoed.
-Passphrase: {{.InputLine "foobar"}}
+Password: {{.InputLine "foobar"}}
 `)
 	_, matches = sign.ExpectRegexp(`Signature: ([0-9a-f]+)\n`)
 	signature := matches[1]

cmd/ethkey/utils.go

@@ -31,18 +31,18 @@ import (
 // promptPassphrase prompts the user for a passphrase.  Set confirmation to true
 // to require the user to confirm the passphrase.
 func promptPassphrase(confirmation bool) string {
-	passphrase, err := console.Stdin.PromptPassword("Passphrase: ")
+	passphrase, err := console.Stdin.PromptPassword("Password: ")
 	if err != nil {
-		utils.Fatalf("Failed to read passphrase: %v", err)
+		utils.Fatalf("Failed to read password: %v", err)
 	}
 
 	if confirmation {
-		confirm, err := console.Stdin.PromptPassword("Repeat passphrase: ")
+		confirm, err := console.Stdin.PromptPassword("Repeat password: ")
 		if err != nil {
-			utils.Fatalf("Failed to read passphrase confirmation: %v", err)
+			utils.Fatalf("Failed to read password confirmation: %v", err)
 		}
 		if passphrase != confirm {
-			utils.Fatalf("Passphrases do not match")
+			utils.Fatalf("Passwords do not match")
 		}
 	}
 
@@ -58,7 +58,7 @@ func getPassphrase(ctx *cli.Context) string {
 	if passphraseFile != "" {
 		content, err := ioutil.ReadFile(passphraseFile)
 		if err != nil {
-			utils.Fatalf("Failed to read passphrase file '%s': %v",
+			utils.Fatalf("Failed to read password file '%s': %v",
 				passphraseFile, err)
 		}
 		return strings.TrimRight(string(content), "\r\n")

cmd/evm/runner.go

@@ -17,7 +17,6 @@
 package main
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -121,28 +120,36 @@ func runCmd(ctx *cli.Context) error {
 		ret  []byte
 		err  error
 	)
+	codeFileFlag := ctx.GlobalString(CodeFileFlag.Name)
+	codeFlag := ctx.GlobalString(CodeFlag.Name)
+
 	// The '--code' or '--codefile' flag overrides code in state
-	if ctx.GlobalString(CodeFileFlag.Name) != "" {
+	if codeFileFlag != "" || codeFlag != "" {
 		var hexcode []byte
-		var err error
+		if codeFileFlag != "" {
-		// If - is specified, it means that code comes from stdin
+			var err error
-		if ctx.GlobalString(CodeFileFlag.Name) == "-" {
+			// If - is specified, it means that code comes from stdin
-			//Try reading from stdin
+			if codeFileFlag == "-" {
-			if hexcode, err = ioutil.ReadAll(os.Stdin); err != nil {
+				//Try reading from stdin
-				fmt.Printf("Could not load code from stdin: %v\n", err)
+				if hexcode, err = ioutil.ReadAll(os.Stdin); err != nil {
-				os.Exit(1)
+					fmt.Printf("Could not load code from stdin: %v\n", err)
+					os.Exit(1)
+				}
+			} else {
+				// Codefile with hex assembly
+				if hexcode, err = ioutil.ReadFile(codeFileFlag); err != nil {
+					fmt.Printf("Could not load code from file: %v\n", err)
+					os.Exit(1)
+				}
 			}
 		} else {
-			// Codefile with hex assembly
+			hexcode = []byte(codeFlag)
-			if hexcode, err = ioutil.ReadFile(ctx.GlobalString(CodeFileFlag.Name)); err != nil {
-				fmt.Printf("Could not load code from file: %v\n", err)
-				os.Exit(1)
-			}
 		}
-		code = common.Hex2Bytes(string(bytes.TrimRight(hexcode, "\n")))
+		if len(hexcode)%2 != 0 {
-
+			fmt.Printf("Invalid input length for hex data (%d)\n", len(hexcode))
-	} else if ctx.GlobalString(CodeFlag.Name) != "" {
+			os.Exit(1)
-		code = common.Hex2Bytes(ctx.GlobalString(CodeFlag.Name))
+		}
+		code = common.FromHex(string(hexcode))
 	} else if fn := ctx.Args().First(); len(fn) > 0 {
 		// EASM-file to compile
 		src, err := ioutil.ReadFile(fn)
@@ -155,7 +162,6 @@ func runCmd(ctx *cli.Context) error {
 		}
 		code = common.Hex2Bytes(bin)
 	}
-
 	initialGas := ctx.GlobalUint64(GasFlag.Name)
 	if genesisConfig.GasLimit != 0 {
 		initialGas = genesisConfig.GasLimit
@@ -209,7 +215,7 @@ func runCmd(ctx *cli.Context) error {
 	if ctx.GlobalBool(DumpFlag.Name) {
 		statedb.Commit(true)
 		statedb.IntermediateRoot(true)
-		fmt.Println(string(statedb.Dump()))
+		fmt.Println(string(statedb.Dump(false, false, true)))
 	}
 
 	if memProfilePath := ctx.GlobalString(MemProfileFlag.Name); memProfilePath != "" {

cmd/evm/staterunner.go

@@ -105,7 +105,7 @@ func stateTestCmd(ctx *cli.Context) error {
 				// Test failed, mark as so and dump any state to aid debugging
 				result.Pass, result.Error = false, err.Error()
 				if ctx.GlobalBool(DumpFlag.Name) && state != nil {
-					dump := state.RawDump()
+					dump := state.RawDump(false, false, true)
 					result.State = &dump
 				}
 			}

cmd/faucet/faucet.go

@@ -506,7 +506,10 @@ func (f *faucet) apiHandler(conn *websocket.Conn) {
 				Time:    time.Now(),
 				Tx:      signed,
 			})
-			f.timeouts[username] = time.Now().Add(time.Duration(*minutesFlag*int(math.Pow(3, float64(msg.Tier)))) * time.Minute)
+			timeout := time.Duration(*minutesFlag*int(math.Pow(3, float64(msg.Tier)))) * time.Minute
+			grace := timeout / 288 // 24h timeout => 5m grace
+
+			f.timeouts[username] = time.Now().Add(timeout - grace)
 			fund = true
 		}
 		f.lock.Unlock()

cmd/geth/accountcmd.go

@@ -116,11 +116,11 @@ Print a short summary of all accounts`,
 
 Creates a new account and prints the address.
 
-The account is saved in encrypted format, you are prompted for a passphrase.
+The account is saved in encrypted format, you are prompted for a password.
 
-You must remember this passphrase to unlock your account in the future.
+You must remember this password to unlock your account in the future.
 
-For non-interactive use the passphrase can be specified with the --password flag:
+For non-interactive use the password can be specified with the --password flag:
 
 Note, this is meant to be used for testing only, it is a bad idea to save your
 password to file or expose in any other way.
@@ -142,12 +142,12 @@ password to file or expose in any other way.
 Update an existing account.
 
 The account is saved in the newest version in encrypted format, you are prompted
-for a passphrase to unlock the account and another to save the updated file.
+for a password to unlock the account and another to save the updated file.
 
 This same command can therefore be used to migrate an account of a deprecated
 format to the newest format or change the password for an account.
 
-For non-interactive use the passphrase can be specified with the --password flag:
+For non-interactive use the password can be specified with the --password flag:
 
     geth account update [options] <address>
 
@@ -174,11 +174,11 @@ Prints the address.
 
 The keyfile is assumed to contain an unencrypted private key in hexadecimal format.
 
-The account is saved in encrypted format, you are prompted for a passphrase.
+The account is saved in encrypted format, you are prompted for a password.
 
-You must remember this passphrase to unlock your account in the future.
+You must remember this password to unlock your account in the future.
 
-For non-interactive use the passphrase can be specified with the -password flag:
+For non-interactive use the password can be specified with the -password flag:
 
     geth account import [options] <keyfile>
 
@@ -247,17 +247,17 @@ func getPassPhrase(prompt string, confirmation bool, i int, passwords []string)
 	if prompt != "" {
 		fmt.Println(prompt)
 	}
-	password, err := console.Stdin.PromptPassword("Passphrase: ")
+	password, err := console.Stdin.PromptPassword("Password: ")
 	if err != nil {
-		utils.Fatalf("Failed to read passphrase: %v", err)
+		utils.Fatalf("Failed to read password: %v", err)
 	}
 	if confirmation {
-		confirm, err := console.Stdin.PromptPassword("Repeat passphrase: ")
+		confirm, err := console.Stdin.PromptPassword("Repeat password: ")
 		if err != nil {
-			utils.Fatalf("Failed to read passphrase confirmation: %v", err)
+			utils.Fatalf("Failed to read password confirmation: %v", err)
 		}
 		if password != confirm {
-			utils.Fatalf("Passphrases do not match")
+			utils.Fatalf("Passwords do not match")
 		}
 	}
 	return password
@@ -268,7 +268,7 @@ func ambiguousAddrRecovery(ks *keystore.KeyStore, err *keystore.AmbiguousAddrErr
 	for _, a := range err.Matches {
 		fmt.Println("  ", a.URL)
 	}
-	fmt.Println("Testing your passphrase against all of them...")
+	fmt.Println("Testing your password against all of them...")
 	var match *accounts.Account
 	for _, a := range err.Matches {
 		if err := ks.Unlock(a, auth); err == nil {
@@ -279,7 +279,7 @@ func ambiguousAddrRecovery(ks *keystore.KeyStore, err *keystore.AmbiguousAddrErr
 	if match == nil {
 		utils.Fatalf("None of the listed files could be unlocked.")
 	}
-	fmt.Printf("Your passphrase unlocked %s\n", match.URL)
+	fmt.Printf("Your password unlocked %s\n", match.URL)
 	fmt.Println("In order to avoid this warning, you need to remove the following duplicate key files:")
 	for _, a := range err.Matches {
 		if a != *match {

cmd/geth/accountcmd_test.go

@@ -72,8 +72,8 @@ func TestAccountNew(t *testing.T) {
 	geth.Expect(`
 Your new account is locked with a password. Please give a password. Do not forget this password.
 !! Unsupported terminal, password will be echoed.
-Passphrase: {{.InputLine "foobar"}}
+Password: {{.InputLine "foobar"}}
-Repeat passphrase: {{.InputLine "foobar"}}
+Repeat password: {{.InputLine "foobar"}}
 
 Your new key was generated
 `)
@@ -94,9 +94,9 @@ func TestAccountNewBadRepeat(t *testing.T) {
 	geth.Expect(`
 Your new account is locked with a password. Please give a password. Do not forget this password.
 !! Unsupported terminal, password will be echoed.
-Passphrase: {{.InputLine "something"}}
+Password: {{.InputLine "something"}}
-Repeat passphrase: {{.InputLine "something else"}}
+Repeat password: {{.InputLine "something else"}}
-Fatal: Passphrases do not match
+Fatal: Passwords do not match
 `)
 }
 
@@ -109,10 +109,10 @@ func TestAccountUpdate(t *testing.T) {
 	geth.Expect(`
 Unlocking account f466859ead1932d743d622cb74fc058882e8648a | Attempt 1/3
 !! Unsupported terminal, password will be echoed.
-Passphrase: {{.InputLine "foobar"}}
+Password: {{.InputLine "foobar"}}
 Please give a new password. Do not forget this password.
-Passphrase: {{.InputLine "foobar2"}}
+Password: {{.InputLine "foobar2"}}
-Repeat passphrase: {{.InputLine "foobar2"}}
+Repeat password: {{.InputLine "foobar2"}}
 `)
 }
 
@@ -121,7 +121,7 @@ func TestWalletImport(t *testing.T) {
 	defer geth.ExpectExit()
 	geth.Expect(`
 !! Unsupported terminal, password will be echoed.
-Passphrase: {{.InputLine "foo"}}
+Password: {{.InputLine "foo"}}
 Address: {d4584b5f6229b7be90727b0fc8c6b91bb427821f}
 `)
 
@@ -136,8 +136,8 @@ func TestWalletImportBadPassword(t *testing.T) {
 	defer geth.ExpectExit()
 	geth.Expect(`
 !! Unsupported terminal, password will be echoed.
-Passphrase: {{.InputLine "wrong"}}
+Password: {{.InputLine "wrong"}}
-Fatal: could not decrypt key with given passphrase
+Fatal: could not decrypt key with given password
 `)
 }
 
@@ -150,7 +150,7 @@ func TestUnlockFlag(t *testing.T) {
 	geth.Expect(`
 Unlocking account f466859ead1932d743d622cb74fc058882e8648a | Attempt 1/3
 !! Unsupported terminal, password will be echoed.
-Passphrase: {{.InputLine "foobar"}}
+Password: {{.InputLine "foobar"}}
 `)
 	geth.ExpectExit()
 
@@ -174,12 +174,12 @@ func TestUnlockFlagWrongPassword(t *testing.T) {
 	geth.Expect(`
 Unlocking account f466859ead1932d743d622cb74fc058882e8648a | Attempt 1/3
 !! Unsupported terminal, password will be echoed.
-Passphrase: {{.InputLine "wrong1"}}
+Password: {{.InputLine "wrong1"}}
 Unlocking account f466859ead1932d743d622cb74fc058882e8648a | Attempt 2/3
-Passphrase: {{.InputLine "wrong2"}}
+Password: {{.InputLine "wrong2"}}
 Unlocking account f466859ead1932d743d622cb74fc058882e8648a | Attempt 3/3
-Passphrase: {{.InputLine "wrong3"}}
+Password: {{.InputLine "wrong3"}}
-Fatal: Failed to unlock account f466859ead1932d743d622cb74fc058882e8648a (could not decrypt key with given passphrase)
+Fatal: Failed to unlock account f466859ead1932d743d622cb74fc058882e8648a (could not decrypt key with given password)
 `)
 }
 
@@ -193,9 +193,9 @@ func TestUnlockFlagMultiIndex(t *testing.T) {
 	geth.Expect(`
 Unlocking account 0 | Attempt 1/3
 !! Unsupported terminal, password will be echoed.
-Passphrase: {{.InputLine "foobar"}}
+Password: {{.InputLine "foobar"}}
 Unlocking account 2 | Attempt 1/3
-Passphrase: {{.InputLine "foobar"}}
+Password: {{.InputLine "foobar"}}
 `)
 	geth.ExpectExit()
 
@@ -238,7 +238,7 @@ func TestUnlockFlagPasswordFileWrongPassword(t *testing.T) {
 		"--password", "testdata/wrong-passwords.txt", "--unlock", "0,2")
 	defer geth.ExpectExit()
 	geth.Expect(`
-Fatal: Failed to unlock account 0 (could not decrypt key with given passphrase)
+Fatal: Failed to unlock account 0 (could not decrypt key with given password)
 `)
 }
 
@@ -258,12 +258,12 @@ func TestUnlockFlagAmbiguous(t *testing.T) {
 	geth.Expect(`
 Unlocking account f466859ead1932d743d622cb74fc058882e8648a | Attempt 1/3
 !! Unsupported terminal, password will be echoed.
-Passphrase: {{.InputLine "foobar"}}
+Password: {{.InputLine "foobar"}}
 Multiple key files exist for address f466859ead1932d743d622cb74fc058882e8648a:
    keystore://{{keypath "1"}}
    keystore://{{keypath "2"}}
-Testing your passphrase against all of them...
+Testing your password against all of them...
-Your passphrase unlocked keystore://{{keypath "1"}}
+Your password unlocked keystore://{{keypath "1"}}
 In order to avoid this warning, you need to remove the following duplicate key files:
    keystore://{{keypath "2"}}
 `)
@@ -295,11 +295,11 @@ func TestUnlockFlagAmbiguousWrongPassword(t *testing.T) {
 	geth.Expect(`
 Unlocking account f466859ead1932d743d622cb74fc058882e8648a | Attempt 1/3
 !! Unsupported terminal, password will be echoed.
-Passphrase: {{.InputLine "wrong"}}
+Password: {{.InputLine "wrong"}}
 Multiple key files exist for address f466859ead1932d743d622cb74fc058882e8648a:
    keystore://{{keypath "1"}}
    keystore://{{keypath "2"}}
-Testing your passphrase against all of them...
+Testing your password against all of them...
 Fatal: None of the listed files could be unlocked.
 `)
 	geth.ExpectExit()

cmd/geth/chaincmd.go

@@ -162,6 +162,10 @@ Remove blockchain and state databases`,
 			utils.DataDirFlag,
 			utils.CacheFlag,
 			utils.SyncModeFlag,
+			utils.IterativeOutputFlag,
+			utils.ExcludeCodeFlag,
+			utils.ExcludeStorageFlag,
+			utils.IncludeIncompletesFlag,
 		},
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
@@ -287,7 +291,7 @@ func importChain(ctx *cli.Context) error {
 	fmt.Printf("Allocations:   %.3f million\n", float64(mem.Mallocs)/1000000)
 	fmt.Printf("GC pause:      %v\n\n", time.Duration(mem.PauseTotalNs))
 
-	if ctx.GlobalIsSet(utils.NoCompactionFlag.Name) {
+	if ctx.GlobalBool(utils.NoCompactionFlag.Name) {
 		return nil
 	}
 
@@ -504,6 +508,7 @@ func dump(ctx *cli.Context) error {
 	defer stack.Close()
 
 	chain, chainDb := utils.MakeChain(ctx, stack)
+	defer chainDb.Close()
 	for _, arg := range ctx.Args() {
 		var block *types.Block
 		if hashish(arg) {
@@ -520,10 +525,20 @@ func dump(ctx *cli.Context) error {
 			if err != nil {
 				utils.Fatalf("could not create new state: %v", err)
 			}
-			fmt.Printf("%s\n", state.Dump())
+			excludeCode := ctx.Bool(utils.ExcludeCodeFlag.Name)
+			excludeStorage := ctx.Bool(utils.ExcludeStorageFlag.Name)
+			includeMissing := ctx.Bool(utils.IncludeIncompletesFlag.Name)
+			if ctx.Bool(utils.IterativeOutputFlag.Name) {
+				state.IterativeDump(excludeCode, excludeStorage, !includeMissing, json.NewEncoder(os.Stdout))
+			} else {
+				if includeMissing {
+					fmt.Printf("If you want to include accounts with missing preimages, you need iterative output, since" +
+						" otherwise the accounts will overwrite each other in the resulting mapping.")
+				}
+				fmt.Printf("%v %s\n", includeMissing, state.Dump(excludeCode, excludeStorage, false))
+			}
 		}
 	}
-	chainDb.Close()
 	return nil
 }
 

cmd/geth/config.go

@@ -30,7 +30,6 @@ import (
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/dashboard"
 	"github.com/ethereum/go-ethereum/eth"
-	"github.com/ethereum/go-ethereum/graphql"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/params"
 	whisper "github.com/ethereum/go-ethereum/whisper/whisperv6"
@@ -125,7 +124,6 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) {
 	}
 
 	// Apply flags.
-	utils.SetULC(ctx, &cfg.Eth)
 	utils.SetNodeConfig(ctx, &cfg.Node)
 	stack, err := node.New(&cfg.Node)
 	if err != nil {
@@ -135,7 +133,6 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) {
 	if ctx.GlobalIsSet(utils.EthStatsURLFlag.Name) {
 		cfg.Ethstats.URL = ctx.GlobalString(utils.EthStatsURLFlag.Name)
 	}
-
 	utils.SetShhConfig(ctx, stack, &cfg.Shh)
 	utils.SetDashboardConfig(ctx, &cfg.Dashboard)
 
@@ -154,8 +151,8 @@ func enableWhisper(ctx *cli.Context) bool {
 
 func makeFullNode(ctx *cli.Context) *node.Node {
 	stack, cfg := makeConfigNode(ctx)
-	if ctx.GlobalIsSet(utils.ConstantinopleOverrideFlag.Name) {
+	if ctx.GlobalIsSet(utils.OverrideIstanbulFlag.Name) {
-		cfg.Eth.ConstantinopleOverride = new(big.Int).SetUint64(ctx.GlobalUint64(utils.ConstantinopleOverrideFlag.Name))
+		cfg.Eth.OverrideIstanbul = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideIstanbulFlag.Name))
 	}
 	utils.RegisterEthService(stack, &cfg.Eth)
 
@@ -177,14 +174,10 @@ func makeFullNode(ctx *cli.Context) *node.Node {
 		}
 		utils.RegisterShhService(stack, &cfg.Shh)
 	}
-
+	// Configure GraphQL if requested
-	// Configure GraphQL if required
 	if ctx.GlobalIsSet(utils.GraphQLEnabledFlag.Name) {
-		if err := graphql.RegisterGraphQLService(stack, cfg.Node.GraphQLEndpoint(), cfg.Node.GraphQLCors, cfg.Node.GraphQLVirtualHosts, cfg.Node.HTTPTimeouts); err != nil {
+		utils.RegisterGraphQLService(stack, cfg.Node.GraphQLEndpoint(), cfg.Node.GraphQLCors, cfg.Node.GraphQLVirtualHosts, cfg.Node.HTTPTimeouts)
-			utils.Fatalf("Failed to register the Ethereum service: %v", err)
-		}
 	}
-
 	// Add the Ethereum Stats daemon if requested.
 	if cfg.Ethstats.URL != "" {
 		utils.RegisterEthStatsService(stack, cfg.Ethstats.URL)

cmd/geth/consolecmd.go

@@ -77,6 +77,7 @@ JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/JavaScript-Cons
 // same time.
 func localConsole(ctx *cli.Context) error {
 	// Create and start the node based on the CLI flags
+	prepare(ctx)
 	node := makeFullNode(ctx)
 	startNode(ctx, node)
 	defer node.Close()

cmd/geth/main.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"math"
 	"os"
+	"runtime"
 	godebug "runtime/debug"
 	"sort"
 	"strconv"
@@ -37,6 +38,7 @@ import (
 	"github.com/ethereum/go-ethereum/eth/downloader"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/internal/debug"
+	"github.com/ethereum/go-ethereum/les"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/node"
@@ -67,6 +69,7 @@ var (
 		utils.ExternalSignerFlag,
 		utils.NoUSBFlag,
 		utils.SmartCardDaemonPathFlag,
+		utils.OverrideIstanbulFlag,
 		utils.DashboardEnabledFlag,
 		utils.DashboardAddrFlag,
 		utils.DashboardPortFlag,
@@ -88,18 +91,19 @@ var (
 		utils.TxPoolAccountQueueFlag,
 		utils.TxPoolGlobalQueueFlag,
 		utils.TxPoolLifetimeFlag,
-		utils.ULCModeConfigFlag,
-		utils.OnlyAnnounceModeFlag,
-		utils.ULCTrustedNodesFlag,
-		utils.ULCMinTrustedFractionFlag,
 		utils.SyncModeFlag,
 		utils.ExitWhenSyncedFlag,
 		utils.GCModeFlag,
-		utils.LightServFlag,
+		utils.LightServeFlag,
-		utils.LightBandwidthInFlag,
+		utils.LightLegacyServFlag,
-		utils.LightBandwidthOutFlag,
+		utils.LightIngressFlag,
-		utils.LightPeersFlag,
+		utils.LightEgressFlag,
+		utils.LightMaxPeersFlag,
+		utils.LightLegacyPeersFlag,
 		utils.LightKDFFlag,
+		utils.UltraLightServersFlag,
+		utils.UltraLightFractionFlag,
+		utils.UltraLightOnlyAnnounceFlag,
 		utils.WhitelistFlag,
 		utils.CacheFlag,
 		utils.CacheDatabaseFlag,
@@ -137,7 +141,6 @@ var (
 		utils.GoerliFlag,
 		utils.VMEnableDebugFlag,
 		utils.NetworkIdFlag,
-		utils.ConstantinopleOverrideFlag,
 		utils.EthStatsURLFlag,
 		utils.FakePoWFlag,
 		utils.NoCompactionFlag,
@@ -240,42 +243,6 @@ func init() {
 		if err := debug.Setup(ctx, logdir); err != nil {
 			return err
 		}
-		// If we're a full node on mainnet without --cache specified, bump default cache allowance
-		if ctx.GlobalString(utils.SyncModeFlag.Name) != "light" && !ctx.GlobalIsSet(utils.CacheFlag.Name) && !ctx.GlobalIsSet(utils.NetworkIdFlag.Name) {
-			// Make sure we're not on any supported preconfigured testnet either
-			if !ctx.GlobalIsSet(utils.TestnetFlag.Name) && !ctx.GlobalIsSet(utils.RinkebyFlag.Name) && !ctx.GlobalIsSet(utils.GoerliFlag.Name) {
-				// Nope, we're really on mainnet. Bump that cache up!
-				log.Info("Bumping default cache on mainnet", "provided", ctx.GlobalInt(utils.CacheFlag.Name), "updated", 4096)
-				ctx.GlobalSet(utils.CacheFlag.Name, strconv.Itoa(4096))
-			}
-		}
-		// If we're running a light client on any network, drop the cache to some meaningfully low amount
-		if ctx.GlobalString(utils.SyncModeFlag.Name) == "light" && !ctx.GlobalIsSet(utils.CacheFlag.Name) {
-			log.Info("Dropping default light client cache", "provided", ctx.GlobalInt(utils.CacheFlag.Name), "updated", 128)
-			ctx.GlobalSet(utils.CacheFlag.Name, strconv.Itoa(128))
-		}
-		// Cap the cache allowance and tune the garbage collector
-		var mem gosigar.Mem
-		if err := mem.Get(); err == nil {
-			allowance := int(mem.Total / 1024 / 1024 / 3)
-			if cache := ctx.GlobalInt(utils.CacheFlag.Name); cache > allowance {
-				log.Warn("Sanitizing cache to Go's GC limits", "provided", cache, "updated", allowance)
-				ctx.GlobalSet(utils.CacheFlag.Name, strconv.Itoa(allowance))
-			}
-		}
-		// Ensure Go's GC ignores the database cache for trigger percentage
-		cache := ctx.GlobalInt(utils.CacheFlag.Name)
-		gogc := math.Max(20, math.Min(100, 100/(float64(cache)/1024)))
-
-		log.Debug("Sanitizing Go's GC trigger", "percent", int(gogc))
-		godebug.SetGCPercent(int(gogc))
-
-		// Start metrics export if enabled
-		utils.SetupMetrics(ctx)
-
-		// Start system runtime metrics collection
-		go metrics.CollectProcessMetrics(3 * time.Second)
-
 		return nil
 	}
 
@@ -293,6 +260,50 @@ func main() {
 	}
 }
 
+// prepare manipulates memory cache allowance and setups metric system.
+// This function should be called before launching devp2p stack.
+func prepare(ctx *cli.Context) {
+	// If we're a full node on mainnet without --cache specified, bump default cache allowance
+	if ctx.GlobalString(utils.SyncModeFlag.Name) != "light" && !ctx.GlobalIsSet(utils.CacheFlag.Name) && !ctx.GlobalIsSet(utils.NetworkIdFlag.Name) {
+		// Make sure we're not on any supported preconfigured testnet either
+		if !ctx.GlobalIsSet(utils.TestnetFlag.Name) && !ctx.GlobalIsSet(utils.RinkebyFlag.Name) && !ctx.GlobalIsSet(utils.GoerliFlag.Name) && !ctx.GlobalIsSet(utils.DeveloperFlag.Name) {
+			// Nope, we're really on mainnet. Bump that cache up!
+			log.Info("Bumping default cache on mainnet", "provided", ctx.GlobalInt(utils.CacheFlag.Name), "updated", 4096)
+			ctx.GlobalSet(utils.CacheFlag.Name, strconv.Itoa(4096))
+		}
+	}
+	// If we're running a light client on any network, drop the cache to some meaningfully low amount
+	if ctx.GlobalString(utils.SyncModeFlag.Name) == "light" && !ctx.GlobalIsSet(utils.CacheFlag.Name) {
+		log.Info("Dropping default light client cache", "provided", ctx.GlobalInt(utils.CacheFlag.Name), "updated", 128)
+		ctx.GlobalSet(utils.CacheFlag.Name, strconv.Itoa(128))
+	}
+	// Cap the cache allowance and tune the garbage collector
+	var mem gosigar.Mem
+	// Workaround until OpenBSD support lands into gosigar
+	// Check https://github.com/elastic/gosigar#supported-platforms
+	if runtime.GOOS != "openbsd" {
+		if err := mem.Get(); err == nil {
+			allowance := int(mem.Total / 1024 / 1024 / 3)
+			if cache := ctx.GlobalInt(utils.CacheFlag.Name); cache > allowance {
+				log.Warn("Sanitizing cache to Go's GC limits", "provided", cache, "updated", allowance)
+				ctx.GlobalSet(utils.CacheFlag.Name, strconv.Itoa(allowance))
+			}
+		}
+	}
+	// Ensure Go's GC ignores the database cache for trigger percentage
+	cache := ctx.GlobalInt(utils.CacheFlag.Name)
+	gogc := math.Max(20, math.Min(100, 100/(float64(cache)/1024)))
+
+	log.Debug("Sanitizing Go's GC trigger", "percent", int(gogc))
+	godebug.SetGCPercent(int(gogc))
+
+	// Start metrics export if enabled
+	utils.SetupMetrics(ctx)
+
+	// Start system runtime metrics collection
+	go metrics.CollectProcessMetrics(3 * time.Second)
+}
+
 // geth is the main entry point into the system if no special subcommand is ran.
 // It creates a default node based on the command line arguments and runs it in
 // blocking mode, waiting for it to be shut down.
@@ -300,6 +311,7 @@ func geth(ctx *cli.Context) error {
 	if args := ctx.Args(); len(args) > 0 {
 		return fmt.Errorf("invalid command: %q", args[0])
 	}
+	prepare(ctx)
 	node := makeFullNode(ctx)
 	defer node.Close()
 	startNode(ctx, node)
@@ -323,14 +335,33 @@ func startNode(ctx *cli.Context, stack *node.Node) {
 	events := make(chan accounts.WalletEvent, 16)
 	stack.AccountManager().Subscribe(events)
 
-	go func() {
+	// Create a client to interact with local geth node.
-		// Create a chain state reader for self-derivation
+	rpcClient, err := stack.Attach()
-		rpcClient, err := stack.Attach()
+	if err != nil {
-		if err != nil {
+		utils.Fatalf("Failed to attach to self: %v", err)
-			utils.Fatalf("Failed to attach to self: %v", err)
+	}
-		}
+	ethClient := ethclient.NewClient(rpcClient)
-		stateReader := ethclient.NewClient(rpcClient)
 
+	// Set contract backend for ethereum service if local node
+	// is serving LES requests.
+	if ctx.GlobalInt(utils.LightLegacyServFlag.Name) > 0 || ctx.GlobalInt(utils.LightServeFlag.Name) > 0 {
+		var ethService *eth.Ethereum
+		if err := stack.Service(&ethService); err != nil {
+			utils.Fatalf("Failed to retrieve ethereum service: %v", err)
+		}
+		ethService.SetContractBackend(ethClient)
+	}
+	// Set contract backend for les service if local node is
+	// running as a light client.
+	if ctx.GlobalString(utils.SyncModeFlag.Name) == "light" {
+		var lesService *les.LightEthereum
+		if err := stack.Service(&lesService); err != nil {
+			utils.Fatalf("Failed to retrieve light ethereum service: %v", err)
+		}
+		lesService.SetContractBackend(ethClient)
+	}
+
+	go func() {
 		// Open any wallets already attached
 		for _, wallet := range stack.AccountManager().Wallets() {
 			if err := wallet.Open(""); err != nil {
@@ -354,7 +385,7 @@ func startNode(ctx *cli.Context, stack *node.Node) {
 				}
 				derivationPaths = append(derivationPaths, accounts.DefaultBaseDerivationPath)
 
-				event.Wallet.SelfDerive(derivationPaths, stateReader)
+				event.Wallet.SelfDerive(derivationPaths, ethClient)
 
 			case accounts.WalletDropped:
 				log.Info("Old wallet dropped", "url", event.Wallet.URL())
@@ -383,7 +414,6 @@ func startNode(ctx *cli.Context, stack *node.Node) {
 						"age", common.PrettyAge(timestamp))
 					stack.Stop()
 				}
-
 			}
 		}()
 	}

cmd/geth/retesteth.go

@@ -86,7 +86,7 @@ type RetestethEthAPI interface {
 }
 
 type RetestethDebugAPI interface {
-	AccountRangeAt(ctx context.Context,
+	AccountRange(ctx context.Context,
 		blockHashOrNumber *math.HexOrDecimal256, txIndex uint64,
 		addressHash *math.HexOrDecimal256, maxResults uint64,
 	) (AccountRangeResult, error)
@@ -132,6 +132,7 @@ type CParamsParams struct {
 	ByzantiumForkBlock         *math.HexOrDecimal64  `json:"byzantiumForkBlock"`
 	ConstantinopleForkBlock    *math.HexOrDecimal64  `json:"constantinopleForkBlock"`
 	ConstantinopleFixForkBlock *math.HexOrDecimal64  `json:"constantinopleFixForkBlock"`
+	IstanbulBlock              *math.HexOrDecimal64  `json:"istanbulForkBlock"`
 	ChainID                    *math.HexOrDecimal256 `json:"chainID"`
 	MaximumExtraDataSize       math.HexOrDecimal64   `json:"maximumExtraDataSize"`
 	TieBreakingGas             bool                  `json:"tieBreakingGas"`
@@ -319,6 +320,7 @@ func (api *RetestethAPI) SetChainParams(ctx context.Context, chainParams ChainPa
 		byzantiumBlock      *big.Int
 		constantinopleBlock *big.Int
 		petersburgBlock     *big.Int
+		istanbulBlock       *big.Int
 	)
 	if chainParams.Params.HomesteadForkBlock != nil {
 		homesteadBlock = big.NewInt(int64(*chainParams.Params.HomesteadForkBlock))
@@ -345,6 +347,10 @@ func (api *RetestethAPI) SetChainParams(ctx context.Context, chainParams ChainPa
 	if constantinopleBlock != nil && petersburgBlock == nil {
 		petersburgBlock = big.NewInt(100000000000)
 	}
+	if chainParams.Params.IstanbulBlock != nil {
+		istanbulBlock = big.NewInt(int64(*chainParams.Params.IstanbulBlock))
+	}
+
 	genesis := &core.Genesis{
 		Config: &params.ChainConfig{
 			ChainID:             chainId,
@@ -357,6 +363,7 @@ func (api *RetestethAPI) SetChainParams(ctx context.Context, chainParams ChainPa
 			ByzantiumBlock:      byzantiumBlock,
 			ConstantinopleBlock: constantinopleBlock,
 			PetersburgBlock:     petersburgBlock,
+			IstanbulBlock:       istanbulBlock,
 		},
 		Nonce:      uint64(chainParams.Genesis.Nonce),
 		Timestamp:  uint64(chainParams.Genesis.Timestamp),
@@ -533,6 +540,9 @@ func (api *RetestethAPI) mineBlock() error {
 		}
 	}
 	block, err := api.engine.FinalizeAndAssemble(api.blockchain, header, statedb, txs, []*types.Header{}, receipts)
+	if err != nil {
+		return err
+	}
 	return api.importBlock(block)
 }
 
@@ -604,7 +614,7 @@ func (api *RetestethAPI) GetBlockByNumber(ctx context.Context, blockNr math.HexO
 	return nil, fmt.Errorf("block %d not found", blockNr)
 }
 
-func (api *RetestethAPI) AccountRangeAt(ctx context.Context,
+func (api *RetestethAPI) AccountRange(ctx context.Context,
 	blockHashOrNumber *math.HexOrDecimal256, txIndex uint64,
 	addressHash *math.HexOrDecimal256, maxResults uint64,
 ) (AccountRangeResult, error) {
@@ -765,10 +775,10 @@ func (api *RetestethAPI) StorageRangeAt(ctx context.Context,
 			}
 			// Ensure any modifications are committed to the state
 			// Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect
-			root = statedb.IntermediateRoot(vmenv.ChainConfig().IsEIP158(block.Number()))
+			_ = statedb.IntermediateRoot(vmenv.ChainConfig().IsEIP158(block.Number()))
 			if idx == int(txIndex) {
 				// This is to make sure root can be opened by OpenTrie
-				root, err = statedb.Commit(vmenv.ChainConfig().IsEIP158(block.Number()))
+				_, err = statedb.Commit(vmenv.ChainConfig().IsEIP158(block.Number()))
 				if err != nil {
 					return StorageRangeResult{}, err
 				}
@@ -829,7 +839,7 @@ func retesteth(ctx *cli.Context) error {
 	log.Info("Welcome to retesteth!")
 	// register signer API with server
 	var (
-		extapiURL = "n/a"
+		extapiURL string
 	)
 	apiImpl := &RetestethAPI{}
 	var testApi RetestethTestAPI = apiImpl

cmd/geth/usage.go

@@ -82,14 +82,22 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.GCModeFlag,
 			utils.EthStatsURLFlag,
 			utils.IdentityFlag,
-			utils.LightServFlag,
-			utils.LightBandwidthInFlag,
-			utils.LightBandwidthOutFlag,
-			utils.LightPeersFlag,
 			utils.LightKDFFlag,
 			utils.WhitelistFlag,
 		},
 	},
+	{
+		Name: "LIGHT CLIENT",
+		Flags: []cli.Flag{
+			utils.LightServeFlag,
+			utils.LightIngressFlag,
+			utils.LightEgressFlag,
+			utils.LightMaxPeersFlag,
+			utils.UltraLightServersFlag,
+			utils.UltraLightFractionFlag,
+			utils.UltraLightOnlyAnnounceFlag,
+		},
+	},
 	{
 		Name: "DEVELOPER CHAIN",
 		Flags: []cli.Flag{
@@ -156,20 +164,25 @@ var AppHelpFlagGroups = []flagGroup{
 	{
 		Name: "API AND CONSOLE",
 		Flags: []cli.Flag{
+			utils.IPCDisabledFlag,
+			utils.IPCPathFlag,
 			utils.RPCEnabledFlag,
 			utils.RPCListenAddrFlag,
 			utils.RPCPortFlag,
 			utils.RPCApiFlag,
 			utils.RPCGlobalGasCap,
+			utils.RPCCORSDomainFlag,
+			utils.RPCVirtualHostsFlag,
 			utils.WSEnabledFlag,
 			utils.WSListenAddrFlag,
 			utils.WSPortFlag,
 			utils.WSApiFlag,
 			utils.WSAllowedOriginsFlag,
-			utils.IPCDisabledFlag,
+			utils.GraphQLEnabledFlag,
-			utils.IPCPathFlag,
+			utils.GraphQLListenAddrFlag,
-			utils.RPCCORSDomainFlag,
+			utils.GraphQLPortFlag,
-			utils.RPCVirtualHostsFlag,
+			utils.GraphQLCORSDomainFlag,
+			utils.GraphQLVirtualHostsFlag,
 			utils.JSpathFlag,
 			utils.ExecFlag,
 			utils.PreloadJSFlag,
@@ -240,6 +253,8 @@ var AppHelpFlagGroups = []flagGroup{
 	{
 		Name: "DEPRECATED",
 		Flags: []cli.Flag{
+			utils.LightLegacyServFlag,
+			utils.LightLegacyPeersFlag,
 			utils.MinerLegacyThreadsFlag,
 			utils.MinerLegacyGasTargetFlag,
 			utils.MinerLegacyGasPriceFlag,

cmd/puppeth/genesis_test.go

@@ -18,7 +18,6 @@ package main
 
 import (
 	"encoding/json"
-	"fmt"
 	"io/ioutil"
 	"reflect"
 	"strings"
@@ -61,7 +60,7 @@ func TestAlethSturebyConverter(t *testing.T) {
 		got := strings.Split(c.Sdump(spec), "\n")
 		for i := 0; i < len(exp) && i < len(got); i++ {
 			if exp[i] != got[i] {
-				fmt.Printf("got: %v\nexp: %v\n", exp[i], got[i])
+				t.Logf("got: %v\nexp: %v\n", exp[i], got[i])
 			}
 		}
 	}
@@ -102,7 +101,7 @@ func TestParitySturebyConverter(t *testing.T) {
 		got := strings.Split(c.Sdump(spec), "\n")
 		for i := 0; i < len(exp) && i < len(got); i++ {
 			if exp[i] != got[i] {
-				fmt.Printf("got: %v\nexp: %v\n", exp[i], got[i])
+				t.Logf("got: %v\nexp: %v\n", exp[i], got[i])
 			}
 		}
 	}

cmd/puppeth/module_explorer.go

@@ -30,108 +30,86 @@ import (
 
 // explorerDockerfile is the Dockerfile required to run a block explorer.
 var explorerDockerfile = `
-FROM puppeth/explorer:latest
+FROM puppeth/blockscout:latest
-
-ADD ethstats.json /ethstats.json
-ADD chain.json /chain.json
 
+ADD genesis.json /genesis.json
 RUN \
-  echo '(cd ../eth-net-intelligence-api && pm2 start /ethstats.json)' >  explorer.sh && \
+  echo 'geth --cache 512 init /genesis.json' > explorer.sh && \
-	echo '(cd ../etherchain-light && npm start &)'                      >> explorer.sh && \
+  echo $'geth --networkid {{.NetworkID}} --syncmode "full" --gcmode "archive" --port {{.EthPort}} --bootnodes {{.Bootnodes}} --ethstats \'{{.Ethstats}}\' --cache=512 --rpc --rpcapi "net,web3,eth,shh,debug" --rpccorsdomain "*" --rpcvhosts "*" --ws --wsorigins "*" --exitwhensynced' >> explorer.sh && \
-	echo 'exec /parity/parity --chain=/chain.json --port={{.NodePort}} --tracing=on --fat-db=on --pruning=archive' >> explorer.sh
+  echo $'exec geth --networkid {{.NetworkID}} --syncmode "full" --gcmode "archive" --port {{.EthPort}} --bootnodes {{.Bootnodes}} --ethstats \'{{.Ethstats}}\' --cache=512 --rpc --rpcapi "net,web3,eth,shh,debug" --rpccorsdomain "*" --rpcvhosts "*" --ws --wsorigins "*" &' >> explorer.sh && \
+  echo '/usr/local/bin/docker-entrypoint.sh postgres &' >> explorer.sh && \
+  echo 'sleep 5' >> explorer.sh && \
+  echo 'mix do ecto.drop --force, ecto.create, ecto.migrate' >> explorer.sh && \
+  echo 'mix phx.server' >> explorer.sh
 
 ENTRYPOINT ["/bin/sh", "explorer.sh"]
 `
 
-// explorerEthstats is the configuration file for the ethstats javascript client.
-var explorerEthstats = `[
-  {
-    "name"              : "node-app",
-    "script"            : "app.js",
-    "log_date_format"   : "YYYY-MM-DD HH:mm Z",
-    "merge_logs"        : false,
-    "watch"             : false,
-    "max_restarts"      : 10,
-    "exec_interpreter"  : "node",
-    "exec_mode"         : "fork_mode",
-    "env":
-    {
-      "NODE_ENV"        : "production",
-      "RPC_HOST"        : "localhost",
-      "RPC_PORT"        : "8545",
-      "LISTENING_PORT"  : "{{.Port}}",
-      "INSTANCE_NAME"   : "{{.Name}}",
-      "CONTACT_DETAILS" : "",
-      "WS_SERVER"       : "{{.Host}}",
-      "WS_SECRET"       : "{{.Secret}}",
-      "VERBOSITY"       : 2
-    }
-  }
-]`
-
 // explorerComposefile is the docker-compose.yml file required to deploy and
 // maintain a block explorer.
 var explorerComposefile = `
 version: '2'
 services:
-  explorer:
+    explorer:
-    build: .
+        build: .
-    image: {{.Network}}/explorer
+        image: {{.Network}}/explorer
-    container_name: {{.Network}}_explorer_1
+        container_name: {{.Network}}_explorer_1
-    ports:
+        ports:
-      - "{{.NodePort}}:{{.NodePort}}"
+            - "{{.EthPort}}:{{.EthPort}}"
-      - "{{.NodePort}}:{{.NodePort}}/udp"{{if not .VHost}}
+            - "{{.EthPort}}:{{.EthPort}}/udp"{{if not .VHost}}
-      - "{{.WebPort}}:3000"{{end}}
+            - "{{.WebPort}}:4000"{{end}}
-    volumes:
+        environment:
-      - {{.Datadir}}:/root/.local/share/io.parity.ethereum
+            - ETH_PORT={{.EthPort}}
-    environment:
+            - ETH_NAME={{.EthName}}
-      - NODE_PORT={{.NodePort}}/tcp
+            - BLOCK_TRANSFORMER={{.Transformer}}{{if .VHost}}
-      - STATS={{.Ethstats}}{{if .VHost}}
+            - VIRTUAL_HOST={{.VHost}}
-      - VIRTUAL_HOST={{.VHost}}
+            - VIRTUAL_PORT=4000{{end}}
-      - VIRTUAL_PORT=3000{{end}}
+        volumes:
-    logging:
+            - {{.Datadir}}:/opt/app/.ethereum
-      driver: "json-file"
+            - {{.DBDir}}:/var/lib/postgresql/data
-      options:
+        logging:
-        max-size: "1m"
+          driver: "json-file"
-        max-file: "10"
+          options:
-    restart: always
+            max-size: "1m"
+            max-file: "10"
+        restart: always
 `
 
 // deployExplorer deploys a new block explorer container to a remote machine via
 // SSH, docker and docker-compose. If an instance with the specified network name
 // already exists there, it will be overwritten!
-func deployExplorer(client *sshClient, network string, chainspec []byte, config *explorerInfos, nocache bool) ([]byte, error) {
+func deployExplorer(client *sshClient, network string, bootnodes []string, config *explorerInfos, nocache bool, isClique bool) ([]byte, error) {
 	// Generate the content to upload to the server
 	workdir := fmt.Sprintf("%d", rand.Int63())
 	files := make(map[string][]byte)
 
 	dockerfile := new(bytes.Buffer)
 	template.Must(template.New("").Parse(explorerDockerfile)).Execute(dockerfile, map[string]interface{}{
-		"NodePort": config.nodePort,
+		"NetworkID": config.node.network,
+		"Bootnodes": strings.Join(bootnodes, ","),
+		"Ethstats":  config.node.ethstats,
+		"EthPort":   config.node.port,
 	})
 	files[filepath.Join(workdir, "Dockerfile")] = dockerfile.Bytes()
 
-	ethstats := new(bytes.Buffer)
+	transformer := "base"
-	template.Must(template.New("").Parse(explorerEthstats)).Execute(ethstats, map[string]interface{}{
+	if isClique {
-		"Port":   config.nodePort,
+		transformer = "clique"
-		"Name":   config.ethstats[:strings.Index(config.ethstats, ":")],
+	}
-		"Secret": config.ethstats[strings.Index(config.ethstats, ":")+1 : strings.Index(config.ethstats, "@")],
-		"Host":   config.ethstats[strings.Index(config.ethstats, "@")+1:],
-	})
-	files[filepath.Join(workdir, "ethstats.json")] = ethstats.Bytes()
-
 	composefile := new(bytes.Buffer)
 	template.Must(template.New("").Parse(explorerComposefile)).Execute(composefile, map[string]interface{}{
-		"Datadir":  config.datadir,
+		"Network":     network,
-		"Network":  network,
+		"VHost":       config.host,
-		"NodePort": config.nodePort,
+		"Ethstats":    config.node.ethstats,
-		"VHost":    config.webHost,
+		"Datadir":     config.node.datadir,
-		"WebPort":  config.webPort,
+		"DBDir":       config.dbdir,
-		"Ethstats": config.ethstats[:strings.Index(config.ethstats, ":")],
+		"EthPort":     config.node.port,
+		"EthName":     config.node.ethstats[:strings.Index(config.node.ethstats, ":")],
+		"WebPort":     config.port,
+		"Transformer": transformer,
 	})
 	files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes()
-
+	files[filepath.Join(workdir, "genesis.json")] = config.node.genesis
-	files[filepath.Join(workdir, "chain.json")] = chainspec
 
 	// Upload the deployment files to the remote server (and clean up afterwards)
 	if out, err := client.Upload(files); err != nil {
@@ -149,22 +127,20 @@ func deployExplorer(client *sshClient, network string, chainspec []byte, config
 // explorerInfos is returned from a block explorer status check to allow reporting
 // various configuration parameters.
 type explorerInfos struct {
-	datadir  string
+	node  *nodeInfos
-	ethstats string
+	dbdir string
-	nodePort int
+	host  string
-	webHost  string
+	port  int
-	webPort  int
 }
 
 // Report converts the typed struct into a plain string->string map, containing
 // most - but not all - fields for reporting to the user.
 func (info *explorerInfos) Report() map[string]string {
 	report := map[string]string{
-		"Data directory":         info.datadir,
+		"Website address ":        info.host,
-		"Node listener port ":    strconv.Itoa(info.nodePort),
+		"Website listener port ":  strconv.Itoa(info.port),
-		"Ethstats username":      info.ethstats,
+		"Ethereum listener port ": strconv.Itoa(info.node.port),
-		"Website address ":       info.webHost,
+		"Ethstats username":       info.node.ethstats,
-		"Website listener port ": strconv.Itoa(info.webPort),
 	}
 	return report
 }
@@ -172,7 +148,7 @@ func (info *explorerInfos) Report() map[string]string {
 // checkExplorer does a health-check against a block explorer server to verify
 // whether it's running, and if yes, whether it's responsive.
 func checkExplorer(client *sshClient, network string) (*explorerInfos, error) {
-	// Inspect a possible block explorer container on the host
+	// Inspect a possible explorer container on the host
 	infos, err := inspectContainer(client, fmt.Sprintf("%s_explorer_1", network))
 	if err != nil {
 		return nil, err
@@ -181,13 +157,13 @@ func checkExplorer(client *sshClient, network string) (*explorerInfos, error) {
 		return nil, ErrServiceOffline
 	}
 	// Resolve the port from the host, or the reverse proxy
-	webPort := infos.portmap["3000/tcp"]
+	port := infos.portmap["4000/tcp"]
-	if webPort == 0 {
+	if port == 0 {
 		if proxy, _ := checkNginx(client, network); proxy != nil {
-			webPort = proxy.port
+			port = proxy.port
 		}
 	}
-	if webPort == 0 {
+	if port == 0 {
 		return nil, ErrNotExposed
 	}
 	// Resolve the host from the reverse-proxy and the config values
@@ -196,17 +172,23 @@ func checkExplorer(client *sshClient, network string) (*explorerInfos, error) {
 		host = client.server
 	}
 	// Run a sanity check to see if the devp2p is reachable
-	nodePort := infos.portmap[infos.envvars["NODE_PORT"]]
+	p2pPort := infos.portmap[infos.envvars["ETH_PORT"]+"/tcp"]
-	if err = checkPort(client.server, nodePort); err != nil {
+	if err = checkPort(host, p2pPort); err != nil {
-		log.Warn(fmt.Sprintf("Explorer devp2p port seems unreachable"), "server", client.server, "port", nodePort, "err", err)
+		log.Warn("Explorer node seems unreachable", "server", host, "port", p2pPort, "err", err)
+	}
+	if err = checkPort(host, port); err != nil {
+		log.Warn("Explorer service seems unreachable", "server", host, "port", port, "err", err)
 	}
 	// Assemble and return the useful infos
 	stats := &explorerInfos{
-		datadir:  infos.volumes["/root/.local/share/io.parity.ethereum"],
+		node: &nodeInfos{
-		nodePort: nodePort,
+			datadir:  infos.volumes["/opt/app/.ethereum"],
-		webHost:  host,
+			port:     infos.portmap[infos.envvars["ETH_PORT"]+"/tcp"],
-		webPort:  webPort,
+			ethstats: infos.envvars["ETH_NAME"],
-		ethstats: infos.envvars["STATS"],
+		},
+		dbdir: infos.volumes["/var/lib/postgresql/data"],
+		host:  host,
+		port:  port,
 	}
 	return stats, nil
 }

cmd/puppeth/wizard_dashboard.go

@@ -77,7 +77,7 @@ func (w *wizard) deployDashboard() {
 				}
 			case "explorer":
 				if infos, err := checkExplorer(client, w.network); err == nil {
-					port = infos.webPort
+					port = infos.port
 				}
 			case "wallet":
 				if infos, err := checkWallet(client, w.network); err == nil {

cmd/puppeth/wizard_explorer.go

@@ -35,10 +35,6 @@ func (w *wizard) deployExplorer() {
 		log.Error("No ethstats server configured")
 		return
 	}
-	if w.conf.Genesis.Config.Ethash == nil {
-		log.Error("Only ethash network supported")
-		return
-	}
 	// Select the server to interact with
 	server := w.selectServer()
 	if server == "" {
@@ -50,50 +46,57 @@ func (w *wizard) deployExplorer() {
 	infos, err := checkExplorer(client, w.network)
 	if err != nil {
 		infos = &explorerInfos{
-			nodePort: 30303, webPort: 80, webHost: client.server,
+			node: &nodeInfos{port: 30303},
+			port: 80,
+			host: client.server,
 		}
 	}
 	existed := err == nil
 
-	chainspec, err := newParityChainSpec(w.network, w.conf.Genesis, w.conf.bootnodes)
+	infos.node.genesis, _ = json.MarshalIndent(w.conf.Genesis, "", "  ")
-	if err != nil {
+	infos.node.network = w.conf.Genesis.Config.ChainID.Int64()
-		log.Error("Failed to create chain spec for explorer", "err", err)
-		return
-	}
-	chain, _ := json.MarshalIndent(chainspec, "", "  ")
 
 	// Figure out which port to listen on
 	fmt.Println()
-	fmt.Printf("Which port should the explorer listen on? (default = %d)\n", infos.webPort)
+	fmt.Printf("Which port should the explorer listen on? (default = %d)\n", infos.port)
-	infos.webPort = w.readDefaultInt(infos.webPort)
+	infos.port = w.readDefaultInt(infos.port)
 
 	// Figure which virtual-host to deploy ethstats on
-	if infos.webHost, err = w.ensureVirtualHost(client, infos.webPort, infos.webHost); err != nil {
+	if infos.host, err = w.ensureVirtualHost(client, infos.port, infos.host); err != nil {
 		log.Error("Failed to decide on explorer host", "err", err)
 		return
 	}
 	// Figure out where the user wants to store the persistent data
 	fmt.Println()
-	if infos.datadir == "" {
+	if infos.node.datadir == "" {
-		fmt.Printf("Where should data be stored on the remote machine?\n")
+		fmt.Printf("Where should node data be stored on the remote machine?\n")
-		infos.datadir = w.readString()
+		infos.node.datadir = w.readString()
 	} else {
-		fmt.Printf("Where should data be stored on the remote machine? (default = %s)\n", infos.datadir)
+		fmt.Printf("Where should node data be stored on the remote machine? (default = %s)\n", infos.node.datadir)
-		infos.datadir = w.readDefaultString(infos.datadir)
+		infos.node.datadir = w.readDefaultString(infos.node.datadir)
+	}
+	// Figure out where the user wants to store the persistent data for backend database
+	fmt.Println()
+	if infos.dbdir == "" {
+		fmt.Printf("Where should postgres data be stored on the remote machine?\n")
+		infos.dbdir = w.readString()
+	} else {
+		fmt.Printf("Where should postgres data be stored on the remote machine? (default = %s)\n", infos.dbdir)
+		infos.dbdir = w.readDefaultString(infos.dbdir)
 	}
 	// Figure out which port to listen on
 	fmt.Println()
-	fmt.Printf("Which TCP/UDP port should the archive node listen on? (default = %d)\n", infos.nodePort)
+	fmt.Printf("Which TCP/UDP port should the archive node listen on? (default = %d)\n", infos.node.port)
-	infos.nodePort = w.readDefaultInt(infos.nodePort)
+	infos.node.port = w.readDefaultInt(infos.node.port)
 
 	// Set a proper name to report on the stats page
 	fmt.Println()
-	if infos.ethstats == "" {
+	if infos.node.ethstats == "" {
 		fmt.Printf("What should the explorer be called on the stats page?\n")
-		infos.ethstats = w.readString() + ":" + w.conf.ethstats
+		infos.node.ethstats = w.readString() + ":" + w.conf.ethstats
 	} else {
-		fmt.Printf("What should the explorer be called on the stats page? (default = %s)\n", infos.ethstats)
+		fmt.Printf("What should the explorer be called on the stats page? (default = %s)\n", infos.node.ethstats)
-		infos.ethstats = w.readDefaultString(infos.ethstats) + ":" + w.conf.ethstats
+		infos.node.ethstats = w.readDefaultString(infos.node.ethstats) + ":" + w.conf.ethstats
 	}
 	// Try to deploy the explorer on the host
 	nocache := false
@@ -102,7 +105,7 @@ func (w *wizard) deployExplorer() {
 		fmt.Printf("Should the explorer be built from scratch (y/n)? (default = no)\n")
 		nocache = w.readDefaultYesNo(false)
 	}
-	if out, err := deployExplorer(client, w.network, chain, infos, nocache); err != nil {
+	if out, err := deployExplorer(client, w.network, w.conf.bootnodes, infos, nocache, w.conf.Genesis.Config.Clique != nil); err != nil {
 		log.Error("Failed to deploy explorer container", "err", err)
 		if len(out) > 0 {
 			fmt.Printf("%s\n", out)

cmd/puppeth/wizard_faucet.go

@@ -147,7 +147,7 @@ func (w *wizard) deployFaucet() {
 		infos.node.keyPass = w.readPassword()
 
 		if _, err := keystore.DecryptKey([]byte(infos.node.keyJSON), infos.node.keyPass); err != nil {
-			log.Error("Failed to decrypt key with given passphrase")
+			log.Error("Failed to decrypt key with given password")
 			infos.node.keyJSON = ""
 			infos.node.keyPass = ""
 		}

cmd/puppeth/wizard_genesis.go

@@ -44,12 +44,13 @@ func (w *wizard) makeGenesis() {
 		Difficulty: big.NewInt(524288),
 		Alloc:      make(core.GenesisAlloc),
 		Config: &params.ChainConfig{
-			HomesteadBlock:      big.NewInt(1),
+			HomesteadBlock:      big.NewInt(0),
-			EIP150Block:         big.NewInt(2),
+			EIP150Block:         big.NewInt(0),
-			EIP155Block:         big.NewInt(3),
+			EIP155Block:         big.NewInt(0),
-			EIP158Block:         big.NewInt(3),
+			EIP158Block:         big.NewInt(0),
-			ByzantiumBlock:      big.NewInt(4),
+			ByzantiumBlock:      big.NewInt(0),
-			ConstantinopleBlock: big.NewInt(5),
+			ConstantinopleBlock: big.NewInt(0),
+			PetersburgBlock:     big.NewInt(0),
 		},
 	}
 	// Figure out which consensus engine to choose
@@ -191,7 +192,7 @@ func (w *wizard) importGenesis() {
 func (w *wizard) manageGenesis() {
 	// Figure out whether to modify or export the genesis
 	fmt.Println()
-	fmt.Println(" 1. Modify existing fork rules")
+	fmt.Println(" 1. Modify existing configurations")
 	fmt.Println(" 2. Export genesis configurations")
 	fmt.Println(" 3. Remove genesis configuration")
 
@@ -226,7 +227,7 @@ func (w *wizard) manageGenesis() {
 			w.conf.Genesis.Config.PetersburgBlock = w.conf.Genesis.Config.ConstantinopleBlock
 		}
 		fmt.Println()
-		fmt.Printf("Which block should Constantinople-Fix (remove EIP-1283) come into effect? (default = %v)\n", w.conf.Genesis.Config.PetersburgBlock)
+		fmt.Printf("Which block should Petersburg come into effect? (default = %v)\n", w.conf.Genesis.Config.PetersburgBlock)
 		w.conf.Genesis.Config.PetersburgBlock = w.readDefaultBigInt(w.conf.Genesis.Config.PetersburgBlock)
 
 		out, _ := json.MarshalIndent(w.conf.Genesis.Config, "", "  ")

cmd/puppeth/wizard_network.go

@@ -174,7 +174,7 @@ func (w *wizard) deployComponent() {
 	fmt.Println(" 1. Ethstats  - Network monitoring tool")
 	fmt.Println(" 2. Bootnode  - Entry point of the network")
 	fmt.Println(" 3. Sealer    - Full node minting new blocks")
-	fmt.Println(" 4. Explorer  - Chain analysis webservice (ethash only)")
+	fmt.Println(" 4. Explorer  - Chain analysis webservice")
 	fmt.Println(" 5. Wallet    - Browser wallet for quick sends")
 	fmt.Println(" 6. Faucet    - Crypto faucet to give away funds")
 	fmt.Println(" 7. Dashboard - Website listing above web-services")

cmd/puppeth/wizard_node.go

@@ -142,7 +142,7 @@ func (w *wizard) deployNode(boot bool) {
 				infos.keyPass = w.readPassword()
 
 				if _, err := keystore.DecryptKey([]byte(infos.keyJSON), infos.keyPass); err != nil {
-					log.Error("Failed to decrypt key with given passphrase")
+					log.Error("Failed to decrypt key with given password")
 					return
 				}
 			}

cmd/utils/customflags.go

@@ -20,7 +20,6 @@ import (
 	"encoding"
 	"errors"
 	"flag"
-	"fmt"
 	"math/big"
 	"os"
 	"os/user"
@@ -34,33 +33,44 @@ import (
 // Custom type which is registered in the flags library which cli uses for
 // argument parsing. This allows us to expand Value to an absolute path when
 // the argument is parsed
-type DirectoryString struct {
+type DirectoryString string
-	Value string
+
+func (s *DirectoryString) String() string {
+	return string(*s)
 }
 
-func (self *DirectoryString) String() string {
+func (s *DirectoryString) Set(value string) error {
-	return self.Value
+	*s = DirectoryString(expandPath(value))
-}
-
-func (self *DirectoryString) Set(value string) error {
-	self.Value = expandPath(value)
 	return nil
 }
 
 // Custom cli.Flag type which expand the received string to an absolute path.
 // e.g. ~/.ethereum -> /home/username/.ethereum
 type DirectoryFlag struct {
-	Name  string
+	Name   string
-	Value DirectoryString
+	Value  DirectoryString
-	Usage string
+	Usage  string
+	EnvVar string
 }
 
-func (self DirectoryFlag) String() string {
+func (f DirectoryFlag) String() string {
-	fmtString := "%s %v\t%v"
+	return cli.FlagStringer(f)
-	if len(self.Value.Value) > 0 {
+}
-		fmtString = "%s \"%v\"\t%v"
+
-	}
+// called by cli library, grabs variable from environment (if in env)
-	return fmt.Sprintf(fmtString, prefixedNames(self.Name), self.Value.Value, self.Usage)
+// and adds variable to flag set for parsing.
+func (f DirectoryFlag) Apply(set *flag.FlagSet) {
+	eachName(f.Name, func(name string) {
+		set.Var(&f.Value, f.Name, f.Usage)
+	})
+}
+
+func (f DirectoryFlag) GetName() string {
+	return f.Name
+}
+
+func (f *DirectoryFlag) Set(value string) {
+	f.Value.Set(value)
 }
 
 func eachName(longName string, fn func(string)) {
@@ -71,14 +81,6 @@ func eachName(longName string, fn func(string)) {
 	}
 }
 
-// called by cli library, grabs variable from environment (if in env)
-// and adds variable to flag set for parsing.
-func (self DirectoryFlag) Apply(set *flag.FlagSet) {
-	eachName(self.Name, func(name string) {
-		set.Var(&self.Value, self.Name, self.Usage)
-	})
-}
-
 type TextMarshaler interface {
 	encoding.TextMarshaler
 	encoding.TextUnmarshaler
@@ -103,9 +105,10 @@ func (v textMarshalerVal) Set(s string) error {
 
 // TextMarshalerFlag wraps a TextMarshaler value.
 type TextMarshalerFlag struct {
-	Name  string
+	Name   string
-	Value TextMarshaler
+	Value  TextMarshaler
-	Usage string
+	Usage  string
+	EnvVar string
 }
 
 func (f TextMarshalerFlag) GetName() string {
@@ -113,7 +116,7 @@ func (f TextMarshalerFlag) GetName() string {
 }
 
 func (f TextMarshalerFlag) String() string {
-	return fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage)
+	return cli.FlagStringer(f)
 }
 
 func (f TextMarshalerFlag) Apply(set *flag.FlagSet) {
@@ -134,9 +137,10 @@ func GlobalTextMarshaler(ctx *cli.Context, name string) TextMarshaler {
 // BigFlag is a command line flag that accepts 256 bit big integers in decimal or
 // hexadecimal syntax.
 type BigFlag struct {
-	Name  string
+	Name   string
-	Value *big.Int
+	Value  *big.Int
-	Usage string
+	Usage  string
+	EnvVar string
 }
 
 // bigValue turns *big.Int into a flag.Value
@@ -163,11 +167,7 @@ func (f BigFlag) GetName() string {
 }
 
 func (f BigFlag) String() string {
-	fmtString := "%s %v\t%v"
+	return cli.FlagStringer(f)
-	if f.Value != nil {
-		fmtString = "%s \"%v\"\t%v"
-	}
-	return fmt.Sprintf(fmtString, prefixedNames(f.Name), f.Value, f.Usage)
 }
 
 func (f BigFlag) Apply(set *flag.FlagSet) {
@@ -207,14 +207,6 @@ func prefixedNames(fullName string) (prefixed string) {
 	return
 }
 
-func (self DirectoryFlag) GetName() string {
-	return self.Name
-}
-
-func (self *DirectoryFlag) Set(value string) {
-	self.Value.Value = value
-}
-
 // Expands a file path
 // 1. replace tilde with users home dir
 // 2. expands embedded environment variables

cmd/utils/flags.go

@@ -19,14 +19,17 @@ package utils
 
 import (
 	"crypto/ecdsa"
-	"encoding/json"
+	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"math/big"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
+	"text/tabwriter"
+	"text/template"
 	"time"
 
 	"github.com/ethereum/go-ethereum/accounts"
@@ -45,6 +48,7 @@ import (
 	"github.com/ethereum/go-ethereum/eth/gasprice"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/ethstats"
+	"github.com/ethereum/go-ethereum/graphql"
 	"github.com/ethereum/go-ethereum/les"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
@@ -57,6 +61,7 @@ import (
 	"github.com/ethereum/go-ethereum/p2p/nat"
 	"github.com/ethereum/go-ethereum/p2p/netutil"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rpc"
 	whisper "github.com/ethereum/go-ethereum/whisper/whisperv6"
 	pcsclite "github.com/gballet/go-libpcsclite"
 	cli "gopkg.in/urfave/cli.v1"
@@ -88,8 +93,8 @@ GLOBAL OPTIONS:
    {{range .Flags}}{{.}}
    {{end}}{{end}}
 `
-
 	cli.CommandHelpTemplate = CommandHelpTemplate
+	cli.HelpPrinter = printHelp
 }
 
 // NewApp creates an app with sane defaults.
@@ -97,13 +102,23 @@ func NewApp(gitCommit, gitDate, usage string) *cli.App {
 	app := cli.NewApp()
 	app.Name = filepath.Base(os.Args[0])
 	app.Author = ""
-	//app.Authors = nil
 	app.Email = ""
 	app.Version = params.VersionWithCommit(gitCommit, gitDate)
 	app.Usage = usage
 	return app
 }
 
+func printHelp(out io.Writer, templ string, data interface{}) {
+	funcMap := template.FuncMap{"join": strings.Join}
+	t := template.Must(template.New("help").Funcs(funcMap).Parse(templ))
+	w := tabwriter.NewWriter(out, 38, 8, 2, ' ', 0)
+	err := t.Execute(w, data)
+	if err != nil {
+		panic(err)
+	}
+	w.Flush()
+}
+
 // These are all the command line flags we support.
 // If you add to this list, please remember to include the
 // flag in the appropriate command definition.
@@ -116,7 +131,7 @@ var (
 	DataDirFlag = DirectoryFlag{
 		Name:  "datadir",
 		Usage: "Data directory for the databases and keystore",
-		Value: DirectoryString{node.DefaultDataDir()},
+		Value: DirectoryString(node.DefaultDataDir()),
 	}
 	AncientFlag = DirectoryFlag{
 		Name:  "datadir.ancient",
@@ -152,10 +167,6 @@ var (
 		Name:  "goerli",
 		Usage: "Görli network: pre-configured proof-of-authority test network",
 	}
-	ConstantinopleOverrideFlag = cli.Uint64Flag{
-		Name:  "override.constantinople",
-		Usage: "Manually specify constantinople fork-block, overriding the bundled setting",
-	}
 	DeveloperFlag = cli.BoolFlag{
 		Name:  "dev",
 		Usage: "Ephemeral proof-of-authority network with a pre-funded developer account, mining enabled",
@@ -171,27 +182,27 @@ var (
 	DocRootFlag = DirectoryFlag{
 		Name:  "docroot",
 		Usage: "Document Root for HTTPClient file scheme",
-		Value: DirectoryString{homeDir()},
+		Value: DirectoryString(homeDir()),
 	}
 	ExitWhenSyncedFlag = cli.BoolFlag{
 		Name:  "exitwhensynced",
 		Usage: "Exits after block synchronisation completes",
 	}
-	ULCModeConfigFlag = cli.StringFlag{
+	IterativeOutputFlag = cli.BoolFlag{
-		Name:  "ulc.config",
+		Name:  "iterative",
-		Usage: "Config file to use for ultra light client mode",
+		Usage: "Print streaming JSON iteratively, delimited by newlines",
 	}
-	OnlyAnnounceModeFlag = cli.BoolFlag{
+	ExcludeStorageFlag = cli.BoolFlag{
-		Name:  "ulc.onlyannounce",
+		Name:  "nostorage",
-		Usage: "ULC server sends announcements only",
+		Usage: "Exclude storage entries (save db lookups)",
 	}
-	ULCMinTrustedFractionFlag = cli.IntFlag{
+	IncludeIncompletesFlag = cli.BoolFlag{
-		Name:  "ulc.fraction",
+		Name:  "incompletes",
-		Usage: "Minimum % of trusted ULC servers required to announce a new head",
+		Usage: "Include accounts for which we don't have the address (missing preimage)",
 	}
-	ULCTrustedNodesFlag = cli.StringFlag{
+	ExcludeCodeFlag = cli.BoolFlag{
-		Name:  "ulc.trusted",
+		Name:  "nocode",
-		Usage: "List of trusted ULC servers",
+		Usage: "Exclude contract code (save db lookups)",
 	}
 	defaultSyncMode = eth.DefaultConfig.SyncMode
 	SyncModeFlag    = TextMarshalerFlag{
@@ -204,26 +215,6 @@ var (
 		Usage: `Blockchain garbage collection mode ("full", "archive")`,
 		Value: "full",
 	}
-	LightServFlag = cli.IntFlag{
-		Name:  "lightserv",
-		Usage: "Maximum percentage of time allowed for serving LES requests (multi-threaded processing allows values over 100)",
-		Value: 0,
-	}
-	LightBandwidthInFlag = cli.IntFlag{
-		Name:  "lightbwin",
-		Usage: "Incoming bandwidth limit for light server (kilobytes/sec, 0 = unlimited)",
-		Value: 0,
-	}
-	LightBandwidthOutFlag = cli.IntFlag{
-		Name:  "lightbwout",
-		Usage: "Outgoing bandwidth limit for light server (kilobytes/sec, 0 = unlimited)",
-		Value: 0,
-	}
-	LightPeersFlag = cli.IntFlag{
-		Name:  "lightpeers",
-		Usage: "Maximum number of LES client peers",
-		Value: eth.DefaultConfig.LightPeers,
-	}
 	LightKDFFlag = cli.BoolFlag{
 		Name:  "lightkdf",
 		Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength",
@@ -232,6 +223,55 @@ var (
 		Name:  "whitelist",
 		Usage: "Comma separated block number-to-hash mappings to enforce (<number>=<hash>)",
 	}
+	OverrideIstanbulFlag = cli.Uint64Flag{
+		Name:  "override.istanbul",
+		Usage: "Manually specify Istanbul fork-block, overriding the bundled setting",
+	}
+	// Light server and client settings
+	LightLegacyServFlag = cli.IntFlag{ // Deprecated in favor of light.serve, remove in 2021
+		Name:  "lightserv",
+		Usage: "Maximum percentage of time allowed for serving LES requests (deprecated, use --light.serve)",
+		Value: eth.DefaultConfig.LightServ,
+	}
+	LightServeFlag = cli.IntFlag{
+		Name:  "light.serve",
+		Usage: "Maximum percentage of time allowed for serving LES requests (multi-threaded processing allows values over 100)",
+		Value: eth.DefaultConfig.LightServ,
+	}
+	LightIngressFlag = cli.IntFlag{
+		Name:  "light.ingress",
+		Usage: "Incoming bandwidth limit for serving light clients (kilobytes/sec, 0 = unlimited)",
+		Value: eth.DefaultConfig.LightIngress,
+	}
+	LightEgressFlag = cli.IntFlag{
+		Name:  "light.egress",
+		Usage: "Outgoing bandwidth limit for serving light clients (kilobytes/sec, 0 = unlimited)",
+		Value: eth.DefaultConfig.LightEgress,
+	}
+	LightLegacyPeersFlag = cli.IntFlag{ // Deprecated in favor of light.maxpeers, remove in 2021
+		Name:  "lightpeers",
+		Usage: "Maximum number of light clients to serve, or light servers to attach to  (deprecated, use --light.maxpeers)",
+		Value: eth.DefaultConfig.LightPeers,
+	}
+	LightMaxPeersFlag = cli.IntFlag{
+		Name:  "light.maxpeers",
+		Usage: "Maximum number of light clients to serve, or light servers to attach to",
+		Value: eth.DefaultConfig.LightPeers,
+	}
+	UltraLightServersFlag = cli.StringFlag{
+		Name:  "ulc.servers",
+		Usage: "List of trusted ultra-light servers",
+		Value: strings.Join(eth.DefaultConfig.UltraLightServers, ","),
+	}
+	UltraLightFractionFlag = cli.IntFlag{
+		Name:  "ulc.fraction",
+		Usage: "Minimum % of trusted ultra-light servers required to announce a new head",
+		Value: eth.DefaultConfig.UltraLightFraction,
+	}
+	UltraLightOnlyAnnounceFlag = cli.BoolFlag{
+		Name:  "ulc.onlyannounce",
+		Usage: "Ultra light server sends announcements only",
+	}
 	// Dashboard settings
 	DashboardEnabledFlag = cli.BoolFlag{
 		Name:  "dashboard",
@@ -269,8 +309,8 @@ var (
 	}
 	EthashDatasetDirFlag = DirectoryFlag{
 		Name:  "ethash.dagdir",
-		Usage: "Directory to store the ethash mining DAGs (default = inside home folder)",
+		Usage: "Directory to store the ethash mining DAGs",
-		Value: DirectoryString{eth.DefaultConfig.Ethash.DatasetDir},
+		Value: DirectoryString(eth.DefaultConfig.Ethash.DatasetDir),
 	}
 	EthashDatasetsInMemoryFlag = cli.IntFlag{
 		Name:  "ethash.dagsinmem",
@@ -474,6 +514,14 @@ var (
 		Usage: "Disables db compaction after import",
 	}
 	// RPC settings
+	IPCDisabledFlag = cli.BoolFlag{
+		Name:  "ipcdisable",
+		Usage: "Disable the IPC-RPC server",
+	}
+	IPCPathFlag = DirectoryFlag{
+		Name:  "ipcpath",
+		Usage: "Filename for IPC socket/pipe within the datadir (explicit paths escape it)",
+	}
 	RPCEnabledFlag = cli.BoolFlag{
 		Name:  "rpc",
 		Usage: "Enable the HTTP-RPC server",
@@ -488,30 +536,6 @@ var (
 		Usage: "HTTP-RPC server listening port",
 		Value: node.DefaultHTTPPort,
 	}
-	GraphQLEnabledFlag = cli.BoolFlag{
-		Name:  "graphql",
-		Usage: "Enable the GraphQL server",
-	}
-	GraphQLListenAddrFlag = cli.StringFlag{
-		Name:  "graphql.addr",
-		Usage: "GraphQL server listening interface",
-		Value: node.DefaultGraphQLHost,
-	}
-	GraphQLPortFlag = cli.IntFlag{
-		Name:  "graphql.port",
-		Usage: "GraphQL server listening port",
-		Value: node.DefaultGraphQLPort,
-	}
-	GraphQLCORSDomainFlag = cli.StringFlag{
-		Name:  "graphql.rpccorsdomain",
-		Usage: "Comma separated list of domains from which to accept cross origin requests (browser enforced)",
-		Value: "",
-	}
-	GraphQLVirtualHostsFlag = cli.StringFlag{
-		Name:  "graphql.rpcvhosts",
-		Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.",
-		Value: strings.Join(node.DefaultConfig.HTTPVirtualHosts, ","),
-	}
 	RPCCORSDomainFlag = cli.StringFlag{
 		Name:  "rpccorsdomain",
 		Usage: "Comma separated list of domains from which to accept cross origin requests (browser enforced)",
@@ -527,14 +551,6 @@ var (
 		Usage: "API's offered over the HTTP-RPC interface",
 		Value: "",
 	}
-	IPCDisabledFlag = cli.BoolFlag{
-		Name:  "ipcdisable",
-		Usage: "Disable the IPC-RPC server",
-	}
-	IPCPathFlag = DirectoryFlag{
-		Name:  "ipcpath",
-		Usage: "Filename for IPC socket/pipe within the datadir (explicit paths escape it)",
-	}
 	WSEnabledFlag = cli.BoolFlag{
 		Name:  "ws",
 		Usage: "Enable the WS-RPC server",
@@ -559,6 +575,30 @@ var (
 		Usage: "Origins from which to accept websockets requests",
 		Value: "",
 	}
+	GraphQLEnabledFlag = cli.BoolFlag{
+		Name:  "graphql",
+		Usage: "Enable the GraphQL server",
+	}
+	GraphQLListenAddrFlag = cli.StringFlag{
+		Name:  "graphql.addr",
+		Usage: "GraphQL server listening interface",
+		Value: node.DefaultGraphQLHost,
+	}
+	GraphQLPortFlag = cli.IntFlag{
+		Name:  "graphql.port",
+		Usage: "GraphQL server listening port",
+		Value: node.DefaultGraphQLPort,
+	}
+	GraphQLCORSDomainFlag = cli.StringFlag{
+		Name:  "graphql.corsdomain",
+		Usage: "Comma separated list of domains from which to accept cross origin requests (browser enforced)",
+		Value: "",
+	}
+	GraphQLVirtualHostsFlag = cli.StringFlag{
+		Name:  "graphql.vhosts",
+		Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.",
+		Value: strings.Join(node.DefaultConfig.GraphQLVirtualHosts, ","),
+	}
 	ExecFlag = cli.StringFlag{
 		Name:  "exec",
 		Usage: "Execute JavaScript statement",
@@ -874,7 +914,6 @@ func setHTTP(ctx *cli.Context, cfg *node.Config) {
 			cfg.HTTPHost = ctx.GlobalString(RPCListenAddrFlag.Name)
 		}
 	}
-
 	if ctx.GlobalIsSet(RPCPortFlag.Name) {
 		cfg.HTTPPort = ctx.GlobalInt(RPCPortFlag.Name)
 	}
@@ -916,7 +955,6 @@ func setWS(ctx *cli.Context, cfg *node.Config) {
 			cfg.WSHost = ctx.GlobalString(WSListenAddrFlag.Name)
 		}
 	}
-
 	if ctx.GlobalIsSet(WSPortFlag.Name) {
 		cfg.WSPort = ctx.GlobalInt(WSPortFlag.Name)
 	}
@@ -931,7 +969,7 @@ func setWS(ctx *cli.Context, cfg *node.Config) {
 // setIPC creates an IPC path configuration from the set command line flags,
 // returning an empty string if IPC was explicitly disabled, or the set path.
 func setIPC(ctx *cli.Context, cfg *node.Config) {
-	checkExclusive(ctx, IPCDisabledFlag, IPCPathFlag)
+	CheckExclusive(ctx, IPCDisabledFlag, IPCPathFlag)
 	switch {
 	case ctx.GlobalBool(IPCDisabledFlag.Name):
 		cfg.IPCPath = ""
@@ -940,37 +978,38 @@ func setIPC(ctx *cli.Context, cfg *node.Config) {
 	}
 }
 
-// SetULC setup ULC config from file if given.
+// setLes configures the les server and ultra light client settings from the command line flags.
-func SetULC(ctx *cli.Context, cfg *eth.Config) {
+func setLes(ctx *cli.Context, cfg *eth.Config) {
-	// ULC config isn't loaded from global config and ULC config and ULC trusted nodes are not defined.
+	if ctx.GlobalIsSet(LightLegacyServFlag.Name) {
-	if cfg.ULC == nil && !(ctx.GlobalIsSet(ULCModeConfigFlag.Name) || ctx.GlobalIsSet(ULCTrustedNodesFlag.Name)) {
+		cfg.LightServ = ctx.GlobalInt(LightLegacyServFlag.Name)
-		return
 	}
-	cfg.ULC = &eth.ULCConfig{}
+	if ctx.GlobalIsSet(LightServeFlag.Name) {
-
+		cfg.LightServ = ctx.GlobalInt(LightServeFlag.Name)
-	path := ctx.GlobalString(ULCModeConfigFlag.Name)
-	if path != "" {
-		cfgData, err := ioutil.ReadFile(path)
-		if err != nil {
-			Fatalf("Failed to unmarshal ULC configuration: %v", err)
-		}
-
-		err = json.Unmarshal(cfgData, &cfg.ULC)
-		if err != nil {
-			Fatalf("Failed to unmarshal ULC configuration: %s", err.Error())
-		}
 	}
-
+	if ctx.GlobalIsSet(LightIngressFlag.Name) {
-	if trustedNodes := ctx.GlobalString(ULCTrustedNodesFlag.Name); trustedNodes != "" {
+		cfg.LightIngress = ctx.GlobalInt(LightIngressFlag.Name)
-		cfg.ULC.TrustedServers = strings.Split(trustedNodes, ",")
 	}
-
+	if ctx.GlobalIsSet(LightEgressFlag.Name) {
-	if trustedFraction := ctx.GlobalInt(ULCMinTrustedFractionFlag.Name); trustedFraction > 0 {
+		cfg.LightEgress = ctx.GlobalInt(LightEgressFlag.Name)
-		cfg.ULC.MinTrustedFraction = trustedFraction
 	}
-	if cfg.ULC.MinTrustedFraction <= 0 && cfg.ULC.MinTrustedFraction > 100 {
+	if ctx.GlobalIsSet(LightLegacyPeersFlag.Name) {
-		log.Error("MinTrustedFraction is invalid", "MinTrustedFraction", cfg.ULC.MinTrustedFraction, "Changed to default", eth.DefaultULCMinTrustedFraction)
+		cfg.LightPeers = ctx.GlobalInt(LightLegacyPeersFlag.Name)
-		cfg.ULC.MinTrustedFraction = eth.DefaultULCMinTrustedFraction
+	}
+	if ctx.GlobalIsSet(LightMaxPeersFlag.Name) {
+		cfg.LightPeers = ctx.GlobalInt(LightMaxPeersFlag.Name)
+	}
+	if ctx.GlobalIsSet(UltraLightServersFlag.Name) {
+		cfg.UltraLightServers = strings.Split(ctx.GlobalString(UltraLightServersFlag.Name), ",")
+	}
+	if ctx.GlobalIsSet(UltraLightFractionFlag.Name) {
+		cfg.UltraLightFraction = ctx.GlobalInt(UltraLightFractionFlag.Name)
+	}
+	if cfg.UltraLightFraction <= 0 && cfg.UltraLightFraction > 100 {
+		log.Error("Ultra light fraction is invalid", "had", cfg.UltraLightFraction, "updated", eth.DefaultConfig.UltraLightFraction)
+		cfg.UltraLightFraction = eth.DefaultConfig.UltraLightFraction
+	}
+	if ctx.GlobalIsSet(UltraLightOnlyAnnounceFlag.Name) {
+		cfg.UltraLightOnlyAnnounce = ctx.GlobalBool(UltraLightOnlyAnnounceFlag.Name)
 	}
 }
 
@@ -1064,19 +1103,22 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) {
 	setBootstrapNodesV5(ctx, cfg)
 
 	lightClient := ctx.GlobalString(SyncModeFlag.Name) == "light"
-	lightServer := ctx.GlobalInt(LightServFlag.Name) != 0
+	lightServer := (ctx.GlobalInt(LightLegacyServFlag.Name) != 0 || ctx.GlobalInt(LightServeFlag.Name) != 0)
-	lightPeers := ctx.GlobalInt(LightPeersFlag.Name)
 
+	lightPeers := ctx.GlobalInt(LightLegacyPeersFlag.Name)
+	if ctx.GlobalIsSet(LightMaxPeersFlag.Name) {
+		lightPeers = ctx.GlobalInt(LightMaxPeersFlag.Name)
+	}
 	if ctx.GlobalIsSet(MaxPeersFlag.Name) {
 		cfg.MaxPeers = ctx.GlobalInt(MaxPeersFlag.Name)
-		if lightServer && !ctx.GlobalIsSet(LightPeersFlag.Name) {
+		if lightServer && !ctx.GlobalIsSet(LightLegacyPeersFlag.Name) && !ctx.GlobalIsSet(LightMaxPeersFlag.Name) {
 			cfg.MaxPeers += lightPeers
 		}
 	} else {
 		if lightServer {
 			cfg.MaxPeers += lightPeers
 		}
-		if lightClient && ctx.GlobalIsSet(LightPeersFlag.Name) && cfg.MaxPeers < lightPeers {
+		if lightClient && (ctx.GlobalIsSet(LightLegacyPeersFlag.Name) || ctx.GlobalIsSet(LightMaxPeersFlag.Name)) && cfg.MaxPeers < lightPeers {
 			cfg.MaxPeers = lightPeers
 		}
 	}
@@ -1178,11 +1220,11 @@ func setDataDir(ctx *cli.Context, cfg *node.Config) {
 		cfg.DataDir = ctx.GlobalString(DataDirFlag.Name)
 	case ctx.GlobalBool(DeveloperFlag.Name):
 		cfg.DataDir = "" // unless explicitly requested, use memory databases
-	case ctx.GlobalBool(TestnetFlag.Name):
+	case ctx.GlobalBool(TestnetFlag.Name) && cfg.DataDir == node.DefaultDataDir():
 		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "testnet")
-	case ctx.GlobalBool(RinkebyFlag.Name):
+	case ctx.GlobalBool(RinkebyFlag.Name) && cfg.DataDir == node.DefaultDataDir():
 		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby")
-	case ctx.GlobalBool(GoerliFlag.Name):
+	case ctx.GlobalBool(GoerliFlag.Name) && cfg.DataDir == node.DefaultDataDir():
 		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "goerli")
 	}
 }
@@ -1316,10 +1358,10 @@ func setWhitelist(ctx *cli.Context, cfg *eth.Config) {
 	}
 }
 
-// checkExclusive verifies that only a single instance of the provided flags was
+// CheckExclusive verifies that only a single instance of the provided flags was
 // set by the user. Each flag might optionally be followed by a string type to
 // specialize it further.
-func checkExclusive(ctx *cli.Context, args ...interface{}) {
+func CheckExclusive(ctx *cli.Context, args ...interface{}) {
 	set := make([]string, 0, 1)
 	for i := 0; i < len(args); i++ {
 		// Make sure the next argument is a flag and skip if not set
@@ -1373,10 +1415,10 @@ func SetShhConfig(ctx *cli.Context, stack *node.Node, cfg *whisper.Config) {
 // SetEthConfig applies eth-related command line flags to the config.
 func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 	// Avoid conflicting network flags
-	checkExclusive(ctx, DeveloperFlag, TestnetFlag, RinkebyFlag, GoerliFlag)
+	CheckExclusive(ctx, DeveloperFlag, TestnetFlag, RinkebyFlag, GoerliFlag)
-	checkExclusive(ctx, LightServFlag, SyncModeFlag, "light")
+	CheckExclusive(ctx, LightLegacyServFlag, LightServeFlag, SyncModeFlag, "light")
-	// Can't use both ephemeral unlocked and external signer
+	CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer
-	checkExclusive(ctx, DeveloperFlag, ExternalSignerFlag)
+
 	var ks *keystore.KeyStore
 	if keystores := stack.AccountManager().Backends(keystore.KeyStoreType); len(keystores) > 0 {
 		ks = keystores[0].(*keystore.KeyStore)
@@ -1387,21 +1429,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 	setEthash(ctx, cfg)
 	setMiner(ctx, &cfg.Miner)
 	setWhitelist(ctx, cfg)
+	setLes(ctx, cfg)
 
 	if ctx.GlobalIsSet(SyncModeFlag.Name) {
 		cfg.SyncMode = *GlobalTextMarshaler(ctx, SyncModeFlag.Name).(*downloader.SyncMode)
 	}
-	if ctx.GlobalIsSet(LightServFlag.Name) {
-		cfg.LightServ = ctx.GlobalInt(LightServFlag.Name)
-	}
-	cfg.LightBandwidthIn = ctx.GlobalInt(LightBandwidthInFlag.Name)
-	cfg.LightBandwidthOut = ctx.GlobalInt(LightBandwidthOutFlag.Name)
-	if ctx.GlobalIsSet(LightPeersFlag.Name) {
-		cfg.LightPeers = ctx.GlobalInt(LightPeersFlag.Name)
-	}
-	if ctx.GlobalIsSet(OnlyAnnounceModeFlag.Name) {
-		cfg.OnlyAnnounce = ctx.GlobalBool(OnlyAnnounceModeFlag.Name)
-	}
 	if ctx.GlobalIsSet(NetworkIdFlag.Name) {
 		cfg.NetworkId = ctx.GlobalUint64(NetworkIdFlag.Name)
 	}
@@ -1546,12 +1578,33 @@ func RegisterEthStatsService(stack *node.Node, url string) {
 		var lesServ *les.LightEthereum
 		ctx.Service(&lesServ)
 
+		// Let ethstats use whichever is not nil
 		return ethstats.New(url, ethServ, lesServ)
 	}); err != nil {
 		Fatalf("Failed to register the Ethereum Stats service: %v", err)
 	}
 }
 
+// RegisterGraphQLService is a utility function to construct a new service and register it against a node.
+func RegisterGraphQLService(stack *node.Node, endpoint string, cors, vhosts []string, timeouts rpc.HTTPTimeouts) {
+	if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
+		// Try to construct the GraphQL service backed by a full node
+		var ethServ *eth.Ethereum
+		if err := ctx.Service(&ethServ); err == nil {
+			return graphql.New(ethServ.APIBackend, endpoint, cors, vhosts, timeouts)
+		}
+		// Try to construct the GraphQL service backed by a light node
+		var lesServ *les.LightEthereum
+		if err := ctx.Service(&lesServ); err == nil {
+			return graphql.New(lesServ.ApiBackend, endpoint, cors, vhosts, timeouts)
+		}
+		// Well, this should not have happened, bail out
+		return nil, errors.New("no Ethereum service")
+	}); err != nil {
+		Fatalf("Failed to register the GraphQL service: %v", err)
+	}
+}
+
 func SetupMetrics(ctx *cli.Context) {
 	if metrics.Enabled {
 		log.Info("Enabling metrics collection")

cmd/wnode/main.go

@@ -356,7 +356,7 @@ func configureNode() {
 		if len(symPass) == 0 {
 			symPass, err = console.Stdin.PromptPassword("Please enter the password for symmetric encryption: ")
 			if err != nil {
-				utils.Fatalf("Failed to read passphrase: %v", err)
+				utils.Fatalf("Failed to read password: %v", err)
 			}
 		}
 

common/bitutil/bitutil_test.go

@@ -190,6 +190,8 @@ func benchmarkBaseOR(b *testing.B, size int) {
 	}
 }
 
+var GloBool bool // Exported global will not be dead-code eliminated, at least not yet.
+
 // Benchmarks the potentially optimized bit testing performance.
 func BenchmarkFastTest1KB(b *testing.B) { benchmarkFastTest(b, 1024) }
 func BenchmarkFastTest2KB(b *testing.B) { benchmarkFastTest(b, 2048) }
@@ -197,9 +199,11 @@ func BenchmarkFastTest4KB(b *testing.B) { benchmarkFastTest(b, 4096) }
 
 func benchmarkFastTest(b *testing.B, size int) {
 	p := make([]byte, size)
+	a := false
 	for i := 0; i < b.N; i++ {
-		TestBytes(p)
+		a = a != TestBytes(p)
 	}
+	GloBool = a // Use of benchmark "result" to prevent total dead code elimination.
 }
 
 // Benchmarks the baseline bit testing performance.
@@ -209,7 +213,9 @@ func BenchmarkBaseTest4KB(b *testing.B) { benchmarkBaseTest(b, 4096) }
 
 func benchmarkBaseTest(b *testing.B, size int) {
 	p := make([]byte, size)
+	a := false
 	for i := 0; i < b.N; i++ {
-		safeTestBytes(p)
+		a = a != safeTestBytes(p)
 	}
+	GloBool = a // Use of benchmark "result" to prevent total dead code elimination.
 }

common/bytes.go

@@ -43,10 +43,8 @@ func ToHexArray(b [][]byte) []string {
 // FromHex returns the bytes represented by the hexadecimal string s.
 // s may be prefixed with "0x".
 func FromHex(s string) []byte {
-	if len(s) > 1 {
+	if has0xPrefix(s) {
-		if s[0:2] == "0x" || s[0:2] == "0X" {
+		s = s[2:]
-			s = s[2:]
-		}
 	}
 	if len(s)%2 == 1 {
 		s = "0" + s
@@ -65,8 +63,8 @@ func CopyBytes(b []byte) (copiedBytes []byte) {
 	return
 }
 
-// hasHexPrefix validates str begins with '0x' or '0X'.
+// has0xPrefix validates str begins with '0x' or '0X'.
-func hasHexPrefix(str string) bool {
+func has0xPrefix(str string) bool {
 	return len(str) >= 2 && str[0] == '0' && (str[1] == 'x' || str[1] == 'X')
 }
 

common/compiler/helpers.go

@@ -27,9 +27,10 @@ var versionRegexp = regexp.MustCompile(`([0-9]+)\.([0-9]+)\.([0-9]+)`)
 
 // Contract contains information about a compiled contract, alongside its code and runtime code.
 type Contract struct {
-	Code        string       `json:"code"`
+	Code        string            `json:"code"`
-	RuntimeCode string       `json:"runtime-code"`
+	RuntimeCode string            `json:"runtime-code"`
-	Info        ContractInfo `json:"info"`
+	Info        ContractInfo      `json:"info"`
+	Hashes      map[string]string `json:"hashes"`
 }
 
 // ContractInfo contains information about a compiled contract, including access

common/compiler/solidity.go

@@ -1,4 +1,4 @@
-// Copyright 2019 The go-ethereum Authors
+// Copyright 2015 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
@@ -39,6 +39,7 @@ type solcOutput struct {
 		BinRuntime                                  string `json:"bin-runtime"`
 		SrcMapRuntime                               string `json:"srcmap-runtime"`
 		Bin, SrcMap, Abi, Devdoc, Userdoc, Metadata string
+		Hashes                                      map[string]string
 	}
 	Version string
 }
@@ -46,10 +47,11 @@ type solcOutput struct {
 func (s *Solidity) makeArgs() []string {
 	p := []string{
 		"--combined-json", "bin,bin-runtime,srcmap,srcmap-runtime,abi,userdoc,devdoc",
-		"--optimize", // code optimizer switched on
+		"--optimize",                  // code optimizer switched on
+		"--allow-paths", "., ./, ../", // default to support relative paths
 	}
 	if s.Major > 0 || s.Minor > 4 || s.Patch > 6 {
-		p[1] += ",metadata"
+		p[1] += ",metadata,hashes"
 	}
 	return p
 }
@@ -141,7 +143,6 @@ func ParseCombinedJSON(combinedJSON []byte, source string, languageVersion strin
 	if err := json.Unmarshal(combinedJSON, &output); err != nil {
 		return nil, err
 	}
-
 	// Compilation succeeded, assemble and return the contracts.
 	contracts := make(map[string]*Contract)
 	for name, info := range output.Contracts {
@@ -150,17 +151,14 @@ func ParseCombinedJSON(combinedJSON []byte, source string, languageVersion strin
 		if err := json.Unmarshal([]byte(info.Abi), &abi); err != nil {
 			return nil, fmt.Errorf("solc: error reading abi definition (%v)", err)
 		}
-		var userdoc interface{}
+		var userdoc, devdoc interface{}
-		if err := json.Unmarshal([]byte(info.Userdoc), &userdoc); err != nil {
+		json.Unmarshal([]byte(info.Userdoc), &userdoc)
-			return nil, fmt.Errorf("solc: error reading user doc: %v", err)
+		json.Unmarshal([]byte(info.Devdoc), &devdoc)
-		}
+
-		var devdoc interface{}
-		if err := json.Unmarshal([]byte(info.Devdoc), &devdoc); err != nil {
-			return nil, fmt.Errorf("solc: error reading dev doc: %v", err)
-		}
 		contracts[name] = &Contract{
 			Code:        "0x" + info.Bin,
 			RuntimeCode: "0x" + info.BinRuntime,
+			Hashes:      info.Hashes,
 			Info: ContractInfo{
 				Source:          source,
 				Language:        "Solidity",

common/compiler/solidity_test.go

@@ -1,4 +1,4 @@
-// Copyright 2019 The go-ethereum Authors
+// Copyright 2015 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify

common/fdlimit/fdlimit_darwin.go

@@ -1,4 +1,4 @@
-// Copyright 2019 The go-ethereum Authors
+// Copyright 2016 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify

common/mclock/mclock.go

@@ -42,6 +42,12 @@ type Clock interface {
 	Now() AbsTime
 	Sleep(time.Duration)
 	After(time.Duration) <-chan time.Time
+	AfterFunc(d time.Duration, f func()) Event
+}
+
+// Event represents a cancellable event returned by AfterFunc
+type Event interface {
+	Cancel() bool
 }
 
 // System implements Clock using the system clock.
@@ -61,3 +67,16 @@ func (System) Sleep(d time.Duration) {
 func (System) After(d time.Duration) <-chan time.Time {
 	return time.After(d)
 }
+
+// AfterFunc implements Clock.
+func (System) AfterFunc(d time.Duration, f func()) Event {
+	return (*SystemEvent)(time.AfterFunc(d, f))
+}
+
+// SystemEvent implements Event using time.Timer.
+type SystemEvent time.Timer
+
+// Cancel implements Event.
+func (e *SystemEvent) Cancel() bool {
+	return (*time.Timer)(e).Stop()
+}

common/mclock/simclock.go

@@ -35,30 +35,44 @@ type Simulated struct {
 	scheduled []event
 	mu        sync.RWMutex
 	cond      *sync.Cond
+	lastId    uint64
 }
 
 type event struct {
 	do func()
 	at AbsTime
+	id uint64
+}
+
+// SimulatedEvent implements Event for a virtual clock.
+type SimulatedEvent struct {
+	at AbsTime
+	id uint64
+	s  *Simulated
 }
 
 // Run moves the clock by the given duration, executing all timers before that duration.
 func (s *Simulated) Run(d time.Duration) {
 	s.mu.Lock()
-	defer s.mu.Unlock()
 	s.init()
 
 	end := s.now + AbsTime(d)
+	var do []func()
 	for len(s.scheduled) > 0 {
 		ev := s.scheduled[0]
 		if ev.at > end {
 			break
 		}
 		s.now = ev.at
-		ev.do()
+		do = append(do, ev.do)
 		s.scheduled = s.scheduled[1:]
 	}
 	s.now = end
+	s.mu.Unlock()
+
+	for _, fn := range do {
+		fn()
+	}
 }
 
 func (s *Simulated) ActiveTimers() int {
@@ -94,23 +108,26 @@ func (s *Simulated) Sleep(d time.Duration) {
 // After implements Clock.
 func (s *Simulated) After(d time.Duration) <-chan time.Time {
 	after := make(chan time.Time, 1)
-	s.insert(d, func() {
+	s.AfterFunc(d, func() {
 		after <- (time.Time{}).Add(time.Duration(s.now))
 	})
 	return after
 }
 
-func (s *Simulated) insert(d time.Duration, do func()) {
+// AfterFunc implements Clock.
+func (s *Simulated) AfterFunc(d time.Duration, do func()) Event {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	s.init()
 
 	at := s.now + AbsTime(d)
+	s.lastId++
+	id := s.lastId
 	l, h := 0, len(s.scheduled)
 	ll := h
 	for l != h {
 		m := (l + h) / 2
-		if at < s.scheduled[m].at {
+		if (at < s.scheduled[m].at) || ((at == s.scheduled[m].at) && (id < s.scheduled[m].id)) {
 			h = m
 		} else {
 			l = m + 1
@@ -118,8 +135,10 @@ func (s *Simulated) insert(d time.Duration, do func()) {
 	}
 	s.scheduled = append(s.scheduled, event{})
 	copy(s.scheduled[l+1:], s.scheduled[l:ll])
-	s.scheduled[l] = event{do: do, at: at}
+	e := event{do: do, at: at, id: id}
+	s.scheduled[l] = e
 	s.cond.Broadcast()
+	return &SimulatedEvent{at: at, id: id, s: s}
 }
 
 func (s *Simulated) init() {
@@ -127,3 +146,31 @@ func (s *Simulated) init() {
 		s.cond = sync.NewCond(&s.mu)
 	}
 }
+
+// Cancel implements Event.
+func (e *SimulatedEvent) Cancel() bool {
+	s := e.s
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	l, h := 0, len(s.scheduled)
+	ll := h
+	for l != h {
+		m := (l + h) / 2
+		if e.id == s.scheduled[m].id {
+			l = m
+			break
+		}
+		if (e.at < s.scheduled[m].at) || ((e.at == s.scheduled[m].at) && (e.id < s.scheduled[m].id)) {
+			h = m
+		} else {
+			l = m + 1
+		}
+	}
+	if l >= ll || s.scheduled[l].id != e.id {
+		return false
+	}
+	copy(s.scheduled[l:ll-1], s.scheduled[l+1:])
+	s.scheduled = s.scheduled[:ll-1]
+	return true
+}

common/prque/lazyqueue.go

@@ -0,0 +1,182 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package prque
+
+import (
+	"container/heap"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common/mclock"
+)
+
+// LazyQueue is a priority queue data structure where priorities can change over
+// time and are only evaluated on demand.
+// Two callbacks are required:
+// - priority evaluates the actual priority of an item
+// - maxPriority gives an upper estimate for the priority in any moment between
+//   now and the given absolute time
+// If the upper estimate is exceeded then Update should be called for that item.
+// A global Refresh function should also be called periodically.
+type LazyQueue struct {
+	clock mclock.Clock
+	// Items are stored in one of two internal queues ordered by estimated max
+	// priority until the next and the next-after-next refresh. Update and Refresh
+	// always places items in queue[1].
+	queue       [2]*sstack
+	popQueue    *sstack
+	period      time.Duration
+	maxUntil    mclock.AbsTime
+	indexOffset int
+	setIndex    SetIndexCallback
+	priority    PriorityCallback
+	maxPriority MaxPriorityCallback
+}
+
+type (
+	PriorityCallback    func(data interface{}, now mclock.AbsTime) int64   // actual priority callback
+	MaxPriorityCallback func(data interface{}, until mclock.AbsTime) int64 // estimated maximum priority callback
+)
+
+// NewLazyQueue creates a new lazy queue
+func NewLazyQueue(setIndex SetIndexCallback, priority PriorityCallback, maxPriority MaxPriorityCallback, clock mclock.Clock, refreshPeriod time.Duration) *LazyQueue {
+	q := &LazyQueue{
+		popQueue:    newSstack(nil),
+		setIndex:    setIndex,
+		priority:    priority,
+		maxPriority: maxPriority,
+		clock:       clock,
+		period:      refreshPeriod}
+	q.Reset()
+	q.Refresh()
+	return q
+}
+
+// Reset clears the contents of the queue
+func (q *LazyQueue) Reset() {
+	q.queue[0] = newSstack(q.setIndex0)
+	q.queue[1] = newSstack(q.setIndex1)
+}
+
+// Refresh should be called at least with the frequency specified by the refreshPeriod parameter
+func (q *LazyQueue) Refresh() {
+	q.maxUntil = q.clock.Now() + mclock.AbsTime(q.period)
+	for q.queue[0].Len() != 0 {
+		q.Push(heap.Pop(q.queue[0]).(*item).value)
+	}
+	q.queue[0], q.queue[1] = q.queue[1], q.queue[0]
+	q.indexOffset = 1 - q.indexOffset
+	q.maxUntil += mclock.AbsTime(q.period)
+}
+
+// Push adds an item to the queue
+func (q *LazyQueue) Push(data interface{}) {
+	heap.Push(q.queue[1], &item{data, q.maxPriority(data, q.maxUntil)})
+}
+
+// Update updates the upper priority estimate for the item with the given queue index
+func (q *LazyQueue) Update(index int) {
+	q.Push(q.Remove(index))
+}
+
+// Pop removes and returns the item with the greatest actual priority
+func (q *LazyQueue) Pop() (interface{}, int64) {
+	var (
+		resData interface{}
+		resPri  int64
+	)
+	q.MultiPop(func(data interface{}, priority int64) bool {
+		resData = data
+		resPri = priority
+		return false
+	})
+	return resData, resPri
+}
+
+// peekIndex returns the index of the internal queue where the item with the
+// highest estimated priority is or -1 if both are empty
+func (q *LazyQueue) peekIndex() int {
+	if q.queue[0].Len() != 0 {
+		if q.queue[1].Len() != 0 && q.queue[1].blocks[0][0].priority > q.queue[0].blocks[0][0].priority {
+			return 1
+		}
+		return 0
+	}
+	if q.queue[1].Len() != 0 {
+		return 1
+	}
+	return -1
+}
+
+// MultiPop pops multiple items from the queue and is more efficient than calling
+// Pop multiple times. Popped items are passed to the callback. MultiPop returns
+// when the callback returns false or there are no more items to pop.
+func (q *LazyQueue) MultiPop(callback func(data interface{}, priority int64) bool) {
+	now := q.clock.Now()
+	nextIndex := q.peekIndex()
+	for nextIndex != -1 {
+		data := heap.Pop(q.queue[nextIndex]).(*item).value
+		heap.Push(q.popQueue, &item{data, q.priority(data, now)})
+		nextIndex = q.peekIndex()
+		for q.popQueue.Len() != 0 && (nextIndex == -1 || q.queue[nextIndex].blocks[0][0].priority < q.popQueue.blocks[0][0].priority) {
+			i := heap.Pop(q.popQueue).(*item)
+			if !callback(i.value, i.priority) {
+				for q.popQueue.Len() != 0 {
+					q.Push(heap.Pop(q.popQueue).(*item).value)
+				}
+				return
+			}
+		}
+	}
+}
+
+// PopItem pops the item from the queue only, dropping the associated priority value.
+func (q *LazyQueue) PopItem() interface{} {
+	i, _ := q.Pop()
+	return i
+}
+
+// Remove removes removes the item with the given index.
+func (q *LazyQueue) Remove(index int) interface{} {
+	if index < 0 {
+		return nil
+	}
+	return heap.Remove(q.queue[index&1^q.indexOffset], index>>1).(*item).value
+}
+
+// Empty checks whether the priority queue is empty.
+func (q *LazyQueue) Empty() bool {
+	return q.queue[0].Len() == 0 && q.queue[1].Len() == 0
+}
+
+// Size returns the number of items in the priority queue.
+func (q *LazyQueue) Size() int {
+	return q.queue[0].Len() + q.queue[1].Len()
+}
+
+// setIndex0 translates internal queue item index to the virtual index space of LazyQueue
+func (q *LazyQueue) setIndex0(data interface{}, index int) {
+	if index == -1 {
+		q.setIndex(data, -1)
+	} else {
+		q.setIndex(data, index+index)
+	}
+}
+
+// setIndex1 translates internal queue item index to the virtual index space of LazyQueue
+func (q *LazyQueue) setIndex1(data interface{}, index int) {
+	q.setIndex(data, index+index+1)
+}

common/prque/lazyqueue_test.go

@@ -0,0 +1,119 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package prque
+
+import (
+	"math/rand"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common/mclock"
+)
+
+const (
+	testItems        = 1000
+	testPriorityStep = 100
+	testSteps        = 1000000
+	testStepPeriod   = time.Millisecond
+	testQueueRefresh = time.Second
+	testAvgRate      = float64(testPriorityStep) / float64(testItems) / float64(testStepPeriod)
+)
+
+type lazyItem struct {
+	p, maxp int64
+	last    mclock.AbsTime
+	index   int
+}
+
+func testPriority(a interface{}, now mclock.AbsTime) int64 {
+	return a.(*lazyItem).p
+}
+
+func testMaxPriority(a interface{}, until mclock.AbsTime) int64 {
+	i := a.(*lazyItem)
+	dt := until - i.last
+	i.maxp = i.p + int64(float64(dt)*testAvgRate)
+	return i.maxp
+}
+
+func testSetIndex(a interface{}, i int) {
+	a.(*lazyItem).index = i
+}
+
+func TestLazyQueue(t *testing.T) {
+	rand.Seed(time.Now().UnixNano())
+	clock := &mclock.Simulated{}
+	q := NewLazyQueue(testSetIndex, testPriority, testMaxPriority, clock, testQueueRefresh)
+
+	var (
+		items  [testItems]lazyItem
+		maxPri int64
+	)
+
+	for i := range items[:] {
+		items[i].p = rand.Int63n(testPriorityStep * 10)
+		if items[i].p > maxPri {
+			maxPri = items[i].p
+		}
+		items[i].index = -1
+		q.Push(&items[i])
+	}
+
+	var lock sync.Mutex
+	stopCh := make(chan chan struct{})
+	go func() {
+		for {
+			select {
+			case <-clock.After(testQueueRefresh):
+				lock.Lock()
+				q.Refresh()
+				lock.Unlock()
+			case stop := <-stopCh:
+				close(stop)
+				return
+			}
+		}
+	}()
+
+	for c := 0; c < testSteps; c++ {
+		i := rand.Intn(testItems)
+		lock.Lock()
+		items[i].p += rand.Int63n(testPriorityStep*2-1) + 1
+		if items[i].p > maxPri {
+			maxPri = items[i].p
+		}
+		items[i].last = clock.Now()
+		if items[i].p > items[i].maxp {
+			q.Update(items[i].index)
+		}
+		if rand.Intn(100) == 0 {
+			p := q.PopItem().(*lazyItem)
+			if p.p != maxPri {
+				t.Fatalf("incorrect item (best known priority %d, popped %d)", maxPri, p.p)
+			}
+			q.Push(p)
+		}
+		lock.Unlock()
+		clock.Run(testStepPeriod)
+		clock.WaitForTimers(1)
+	}
+
+	stop := make(chan struct{})
+	stopCh <- stop
+	<-stop
+}

common/types.go

@@ -193,7 +193,7 @@ func HexToAddress(s string) Address { return BytesToAddress(FromHex(s)) }
 // IsHexAddress verifies whether a string can represent a valid hex-encoded
 // Ethereum address or not.
 func IsHexAddress(s string) bool {
-	if hasHexPrefix(s) {
+	if has0xPrefix(s) {
 		s = s[2:]
 	}
 	return len(s) == 2*AddressLength && isHex(s)

consensus/clique/clique.go

@@ -55,8 +55,8 @@ const (
 var (
 	epochLength = uint64(30000) // Default number of blocks after which to checkpoint and reset the pending votes
 
-	extraVanity = 32 // Fixed number of extra-data prefix bytes reserved for signer vanity
+	extraVanity = 32                     // Fixed number of extra-data prefix bytes reserved for signer vanity
-	extraSeal   = 65 // Fixed number of extra-data suffix bytes reserved for signer seal
+	extraSeal   = crypto.SignatureLength // Fixed number of extra-data suffix bytes reserved for signer seal
 
 	nonceAuthVote = hexutil.MustDecode("0xffffffffffffffff") // Magic nonce number to vote on adding a new signer
 	nonceDropVote = hexutil.MustDecode("0x0000000000000000") // Magic nonce number to vote on removing a signer.
@@ -728,7 +728,7 @@ func encodeSigHeader(w io.Writer, header *types.Header) {
 		header.GasLimit,
 		header.GasUsed,
 		header.Time,
-		header.Extra[:len(header.Extra)-65], // Yes, this will panic if extra is too short
+		header.Extra[:len(header.Extra)-crypto.SignatureLength], // Yes, this will panic if extra is too short
 		header.MixDigest,
 		header.Nonce,
 	})

consensus/ethash/sealer_test.go

@@ -1,3 +1,19 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
 package ethash
 
 import (

console/bridge.go

@@ -59,14 +59,14 @@ func (b *bridge) NewAccount(call otto.FunctionCall) (response otto.Value) {
 	switch {
 	// No password was specified, prompt the user for it
 	case len(call.ArgumentList) == 0:
-		if password, err = b.prompter.PromptPassword("Passphrase: "); err != nil {
+		if password, err = b.prompter.PromptPassword("Password: "); err != nil {
 			throwJSException(err.Error())
 		}
-		if confirm, err = b.prompter.PromptPassword("Repeat passphrase: "); err != nil {
+		if confirm, err = b.prompter.PromptPassword("Repeat password: "); err != nil {
 			throwJSException(err.Error())
 		}
 		if password != confirm {
-			throwJSException("passphrases don't match!")
+			throwJSException("passwords don't match!")
 		}
 
 	// A single string password was specified, use that
@@ -180,7 +180,7 @@ func (b *bridge) OpenWallet(call otto.FunctionCall) (response otto.Value) {
 func (b *bridge) readPassphraseAndReopenWallet(call otto.FunctionCall) (otto.Value, error) {
 	var passwd otto.Value
 	wallet := call.Argument(0)
-	if input, err := b.prompter.PromptPassword("Please enter your passphrase: "); err != nil {
+	if input, err := b.prompter.PromptPassword("Please enter your password: "); err != nil {
 		throwJSException(err.Error())
 	} else {
 		passwd, _ = otto.ToValue(input)
@@ -223,7 +223,7 @@ func (b *bridge) UnlockAccount(call otto.FunctionCall) (response otto.Value) {
 
 	if call.Argument(1).IsUndefined() || call.Argument(1).IsNull() {
 		fmt.Fprintf(b.printer, "Unlock account %s\n", account)
-		if input, err := b.prompter.PromptPassword("Passphrase: "); err != nil {
+		if input, err := b.prompter.PromptPassword("Password: "); err != nil {
 			throwJSException(err.Error())
 		} else {
 			passwd, _ = otto.ToValue(input)
@@ -270,7 +270,7 @@ func (b *bridge) Sign(call otto.FunctionCall) (response otto.Value) {
 	// if the password is not given or null ask the user and ensure password is a string
 	if passwd.IsUndefined() || passwd.IsNull() {
 		fmt.Fprintf(b.printer, "Give password for account %s\n", account)
-		if input, err := b.prompter.PromptPassword("Passphrase: "); err != nil {
+		if input, err := b.prompter.PromptPassword("Password: "); err != nil {
 			throwJSException(err.Error())
 		} else {
 			passwd, _ = otto.ToValue(input)

contracts/checkpointoracle/contract/oracle.go

@@ -0,0 +1,415 @@
+// Code generated - DO NOT EDIT.
+// This file is a generated binding and any manual changes will be lost.
+
+package contract
+
+import (
+	"math/big"
+	"strings"
+
+	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/accounts/abi"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/event"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var (
+	_ = big.NewInt
+	_ = strings.NewReader
+	_ = ethereum.NotFound
+	_ = abi.U256
+	_ = bind.Bind
+	_ = common.Big1
+	_ = types.BloomLookup
+	_ = event.NewSubscription
+)
+
+// CheckpointOracleABI is the input ABI used to generate the binding from.
+const CheckpointOracleABI = "[{\"constant\":true,\"inputs\":[],\"name\":\"GetAllAdmin\",\"outputs\":[{\"name\":\"\",\"type\":\"address[]\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":true,\"inputs\":[],\"name\":\"GetLatestCheckpoint\",\"outputs\":[{\"name\":\"\",\"type\":\"uint64\"},{\"name\":\"\",\"type\":\"bytes32\"},{\"name\":\"\",\"type\":\"uint256\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"_recentNumber\",\"type\":\"uint256\"},{\"name\":\"_recentHash\",\"type\":\"bytes32\"},{\"name\":\"_hash\",\"type\":\"bytes32\"},{\"name\":\"_sectionIndex\",\"type\":\"uint64\"},{\"name\":\"v\",\"type\":\"uint8[]\"},{\"name\":\"r\",\"type\":\"bytes32[]\"},{\"name\":\"s\",\"type\":\"bytes32[]\"}],\"name\":\"SetCheckpoint\",\"outputs\":[{\"name\":\"\",\"type\":\"bool\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"name\":\"_adminlist\",\"type\":\"address[]\"},{\"name\":\"_sectionSize\",\"type\":\"uint256\"},{\"name\":\"_processConfirms\",\"type\":\"uint256\"},{\"name\":\"_threshold\",\"type\":\"uint256\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"constructor\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"name\":\"index\",\"type\":\"uint64\"},{\"indexed\":false,\"name\":\"checkpointHash\",\"type\":\"bytes32\"},{\"indexed\":false,\"name\":\"v\",\"type\":\"uint8\"},{\"indexed\":false,\"name\":\"r\",\"type\":\"bytes32\"},{\"indexed\":false,\"name\":\"s\",\"type\":\"bytes32\"}],\"name\":\"NewCheckpointVote\",\"type\":\"event\"}]"
+
+// CheckpointOracleBin is the compiled bytecode used for deploying new contracts.
+const CheckpointOracleBin = `0x608060405234801561001057600080fd5b506040516108153803806108158339818101604052608081101561003357600080fd5b81019080805164010000000081111561004b57600080fd5b8201602081018481111561005e57600080fd5b815185602082028301116401000000008211171561007b57600080fd5b505060208201516040830151606090930151919450925060005b84518110156101415760016000808784815181106100af57fe5b60200260200101516001600160a01b03166001600160a01b0316815260200190815260200160002060006101000a81548160ff02191690831515021790555060018582815181106100fc57fe5b60209081029190910181015182546001808201855560009485529290932090920180546001600160a01b0319166001600160a01b039093169290921790915501610095565b50600592909255600655600755506106b78061015e6000396000f3fe608060405234801561001057600080fd5b50600436106100415760003560e01c806345848dfc146100465780634d6a304c1461009e578063d459fc46146100cf575b600080fd5b61004e6102b0565b60408051602080825283518183015283519192839290830191858101910280838360005b8381101561008a578181015183820152602001610072565b505050509050019250505060405180910390f35b6100a661034f565b6040805167ffffffffffffffff9094168452602084019290925282820152519081900360600190f35b61029c600480360360e08110156100e557600080fd5b81359160208101359160408201359167ffffffffffffffff6060820135169181019060a08101608082013564010000000081111561012257600080fd5b82018360208201111561013457600080fd5b8035906020019184602083028401116401000000008311171561015657600080fd5b91908080602002602001604051908101604052809392919081815260200183836020028082843760009201919091525092959493602081019350359150506401000000008111156101a657600080fd5b8201836020820111156101b857600080fd5b803590602001918460208302840111640100000000831117156101da57600080fd5b919080806020026020016040519081016040528093929190818152602001838360200280828437600092019190915250929594936020810193503591505064010000000081111561022a57600080fd5b82018360208201111561023c57600080fd5b8035906020019184602083028401116401000000008311171561025e57600080fd5b91908080602002602001604051908101604052809392919081815260200183836020028082843760009201919091525092955061036a945050505050565b604080519115158252519081900360200190f35b6060806001805490506040519080825280602002602001820160405280156102e2578160200160208202803883390190505b50905060005b60015481101561034957600181815481106102ff57fe5b9060005260206000200160009054906101000a90046001600160a01b031682828151811061032957fe5b6001600160a01b03909216602092830291909101909101526001016102e8565b50905090565b60025460045460035467ffffffffffffffff90921691909192565b3360009081526020819052604081205460ff1661038657600080fd5b8688401461039357600080fd5b82518451146103a157600080fd5b81518451146103af57600080fd5b6006546005548660010167ffffffffffffffff1602014310156103d457506000610677565b60025467ffffffffffffffff90811690861610156103f457506000610677565b60025467ffffffffffffffff8681169116148015610426575067ffffffffffffffff8516151580610426575060035415155b1561043357506000610677565b8561044057506000610677565b60408051601960f81b6020808301919091526000602183018190523060601b60228401526001600160c01b031960c08a901b166036840152603e8084018b905284518085039091018152605e909301909352815191012090805b86518110156106715760006001848984815181106104b457fe5b60200260200101518985815181106104c857fe5b60200260200101518986815181106104dc57fe5b602002602001015160405160008152602001604052604051808581526020018460ff1660ff1681526020018381526020018281526020019450505050506020604051602081039080840390855afa15801561053b573d6000803e3d6000fd5b505060408051601f1901516001600160a01b03811660009081526020819052919091205490925060ff16905061057057600080fd5b826001600160a01b0316816001600160a01b03161161058e57600080fd5b8092508867ffffffffffffffff167fce51ffa16246bcaf0899f6504f473cd0114f430f566cef71ab7e03d3dde42a418b8a85815181106105ca57fe5b60200260200101518a86815181106105de57fe5b60200260200101518a87815181106105f257fe5b6020026020010151604051808581526020018460ff1660ff16815260200183815260200182815260200194505050505060405180910390a260075482600101106106685750505060048790555050436003556002805467ffffffffffffffff191667ffffffffffffffff86161790556001610677565b5060010161049a565b50600080fd5b97965050505050505056fea265627a7a723058207f6a191ce575596a2f1e907c8c0a01003d16b69fb2c4f432d10878e8c0a99a0264736f6c634300050a0032`
+
+// DeployCheckpointOracle deploys a new Ethereum contract, binding an instance of CheckpointOracle to it.
+func DeployCheckpointOracle(auth *bind.TransactOpts, backend bind.ContractBackend, _adminlist []common.Address, _sectionSize *big.Int, _processConfirms *big.Int, _threshold *big.Int) (common.Address, *types.Transaction, *CheckpointOracle, error) {
+	parsed, err := abi.JSON(strings.NewReader(CheckpointOracleABI))
+	if err != nil {
+		return common.Address{}, nil, nil, err
+	}
+	address, tx, contract, err := bind.DeployContract(auth, parsed, common.FromHex(CheckpointOracleBin), backend, _adminlist, _sectionSize, _processConfirms, _threshold)
+	if err != nil {
+		return common.Address{}, nil, nil, err
+	}
+	return address, tx, &CheckpointOracle{CheckpointOracleCaller: CheckpointOracleCaller{contract: contract}, CheckpointOracleTransactor: CheckpointOracleTransactor{contract: contract}, CheckpointOracleFilterer: CheckpointOracleFilterer{contract: contract}}, nil
+}
+
+// CheckpointOracle is an auto generated Go binding around an Ethereum contract.
+type CheckpointOracle struct {
+	CheckpointOracleCaller     // Read-only binding to the contract
+	CheckpointOracleTransactor // Write-only binding to the contract
+	CheckpointOracleFilterer   // Log filterer for contract events
+}
+
+// CheckpointOracleCaller is an auto generated read-only Go binding around an Ethereum contract.
+type CheckpointOracleCaller struct {
+	contract *bind.BoundContract // Generic contract wrapper for the low level calls
+}
+
+// CheckpointOracleTransactor is an auto generated write-only Go binding around an Ethereum contract.
+type CheckpointOracleTransactor struct {
+	contract *bind.BoundContract // Generic contract wrapper for the low level calls
+}
+
+// CheckpointOracleFilterer is an auto generated log filtering Go binding around an Ethereum contract events.
+type CheckpointOracleFilterer struct {
+	contract *bind.BoundContract // Generic contract wrapper for the low level calls
+}
+
+// CheckpointOracleSession is an auto generated Go binding around an Ethereum contract,
+// with pre-set call and transact options.
+type CheckpointOracleSession struct {
+	Contract     *CheckpointOracle // Generic contract binding to set the session for
+	CallOpts     bind.CallOpts     // Call options to use throughout this session
+	TransactOpts bind.TransactOpts // Transaction auth options to use throughout this session
+}
+
+// CheckpointOracleCallerSession is an auto generated read-only Go binding around an Ethereum contract,
+// with pre-set call options.
+type CheckpointOracleCallerSession struct {
+	Contract *CheckpointOracleCaller // Generic contract caller binding to set the session for
+	CallOpts bind.CallOpts           // Call options to use throughout this session
+}
+
+// CheckpointOracleTransactorSession is an auto generated write-only Go binding around an Ethereum contract,
+// with pre-set transact options.
+type CheckpointOracleTransactorSession struct {
+	Contract     *CheckpointOracleTransactor // Generic contract transactor binding to set the session for
+	TransactOpts bind.TransactOpts           // Transaction auth options to use throughout this session
+}
+
+// CheckpointOracleRaw is an auto generated low-level Go binding around an Ethereum contract.
+type CheckpointOracleRaw struct {
+	Contract *CheckpointOracle // Generic contract binding to access the raw methods on
+}
+
+// CheckpointOracleCallerRaw is an auto generated low-level read-only Go binding around an Ethereum contract.
+type CheckpointOracleCallerRaw struct {
+	Contract *CheckpointOracleCaller // Generic read-only contract binding to access the raw methods on
+}
+
+// CheckpointOracleTransactorRaw is an auto generated low-level write-only Go binding around an Ethereum contract.
+type CheckpointOracleTransactorRaw struct {
+	Contract *CheckpointOracleTransactor // Generic write-only contract binding to access the raw methods on
+}
+
+// NewCheckpointOracle creates a new instance of CheckpointOracle, bound to a specific deployed contract.
+func NewCheckpointOracle(address common.Address, backend bind.ContractBackend) (*CheckpointOracle, error) {
+	contract, err := bindCheckpointOracle(address, backend, backend, backend)
+	if err != nil {
+		return nil, err
+	}
+	return &CheckpointOracle{CheckpointOracleCaller: CheckpointOracleCaller{contract: contract}, CheckpointOracleTransactor: CheckpointOracleTransactor{contract: contract}, CheckpointOracleFilterer: CheckpointOracleFilterer{contract: contract}}, nil
+}
+
+// NewCheckpointOracleCaller creates a new read-only instance of CheckpointOracle, bound to a specific deployed contract.
+func NewCheckpointOracleCaller(address common.Address, caller bind.ContractCaller) (*CheckpointOracleCaller, error) {
+	contract, err := bindCheckpointOracle(address, caller, nil, nil)
+	if err != nil {
+		return nil, err
+	}
+	return &CheckpointOracleCaller{contract: contract}, nil
+}
+
+// NewCheckpointOracleTransactor creates a new write-only instance of CheckpointOracle, bound to a specific deployed contract.
+func NewCheckpointOracleTransactor(address common.Address, transactor bind.ContractTransactor) (*CheckpointOracleTransactor, error) {
+	contract, err := bindCheckpointOracle(address, nil, transactor, nil)
+	if err != nil {
+		return nil, err
+	}
+	return &CheckpointOracleTransactor{contract: contract}, nil
+}
+
+// NewCheckpointOracleFilterer creates a new log filterer instance of CheckpointOracle, bound to a specific deployed contract.
+func NewCheckpointOracleFilterer(address common.Address, filterer bind.ContractFilterer) (*CheckpointOracleFilterer, error) {
+	contract, err := bindCheckpointOracle(address, nil, nil, filterer)
+	if err != nil {
+		return nil, err
+	}
+	return &CheckpointOracleFilterer{contract: contract}, nil
+}
+
+// bindCheckpointOracle binds a generic wrapper to an already deployed contract.
+func bindCheckpointOracle(address common.Address, caller bind.ContractCaller, transactor bind.ContractTransactor, filterer bind.ContractFilterer) (*bind.BoundContract, error) {
+	parsed, err := abi.JSON(strings.NewReader(CheckpointOracleABI))
+	if err != nil {
+		return nil, err
+	}
+	return bind.NewBoundContract(address, parsed, caller, transactor, filterer), nil
+}
+
+// Call invokes the (constant) contract method with params as input values and
+// sets the output to result. The result type might be a single field for simple
+// returns, a slice of interfaces for anonymous returns and a struct for named
+// returns.
+func (_CheckpointOracle *CheckpointOracleRaw) Call(opts *bind.CallOpts, result interface{}, method string, params ...interface{}) error {
+	return _CheckpointOracle.Contract.CheckpointOracleCaller.contract.Call(opts, result, method, params...)
+}
+
+// Transfer initiates a plain transaction to move funds to the contract, calling
+// its default method if one is available.
+func (_CheckpointOracle *CheckpointOracleRaw) Transfer(opts *bind.TransactOpts) (*types.Transaction, error) {
+	return _CheckpointOracle.Contract.CheckpointOracleTransactor.contract.Transfer(opts)
+}
+
+// Transact invokes the (paid) contract method with params as input values.
+func (_CheckpointOracle *CheckpointOracleRaw) Transact(opts *bind.TransactOpts, method string, params ...interface{}) (*types.Transaction, error) {
+	return _CheckpointOracle.Contract.CheckpointOracleTransactor.contract.Transact(opts, method, params...)
+}
+
+// Call invokes the (constant) contract method with params as input values and
+// sets the output to result. The result type might be a single field for simple
+// returns, a slice of interfaces for anonymous returns and a struct for named
+// returns.
+func (_CheckpointOracle *CheckpointOracleCallerRaw) Call(opts *bind.CallOpts, result interface{}, method string, params ...interface{}) error {
+	return _CheckpointOracle.Contract.contract.Call(opts, result, method, params...)
+}
+
+// Transfer initiates a plain transaction to move funds to the contract, calling
+// its default method if one is available.
+func (_CheckpointOracle *CheckpointOracleTransactorRaw) Transfer(opts *bind.TransactOpts) (*types.Transaction, error) {
+	return _CheckpointOracle.Contract.contract.Transfer(opts)
+}
+
+// Transact invokes the (paid) contract method with params as input values.
+func (_CheckpointOracle *CheckpointOracleTransactorRaw) Transact(opts *bind.TransactOpts, method string, params ...interface{}) (*types.Transaction, error) {
+	return _CheckpointOracle.Contract.contract.Transact(opts, method, params...)
+}
+
+// GetAllAdmin is a free data retrieval call binding the contract method 0x45848dfc.
+//
+// Solidity: function GetAllAdmin() constant returns(address[])
+func (_CheckpointOracle *CheckpointOracleCaller) GetAllAdmin(opts *bind.CallOpts) ([]common.Address, error) {
+	var (
+		ret0 = new([]common.Address)
+	)
+	out := ret0
+	err := _CheckpointOracle.contract.Call(opts, out, "GetAllAdmin")
+	return *ret0, err
+}
+
+// GetAllAdmin is a free data retrieval call binding the contract method 0x45848dfc.
+//
+// Solidity: function GetAllAdmin() constant returns(address[])
+func (_CheckpointOracle *CheckpointOracleSession) GetAllAdmin() ([]common.Address, error) {
+	return _CheckpointOracle.Contract.GetAllAdmin(&_CheckpointOracle.CallOpts)
+}
+
+// GetAllAdmin is a free data retrieval call binding the contract method 0x45848dfc.
+//
+// Solidity: function GetAllAdmin() constant returns(address[])
+func (_CheckpointOracle *CheckpointOracleCallerSession) GetAllAdmin() ([]common.Address, error) {
+	return _CheckpointOracle.Contract.GetAllAdmin(&_CheckpointOracle.CallOpts)
+}
+
+// GetLatestCheckpoint is a free data retrieval call binding the contract method 0x4d6a304c.
+//
+// Solidity: function GetLatestCheckpoint() constant returns(uint64, bytes32, uint256)
+func (_CheckpointOracle *CheckpointOracleCaller) GetLatestCheckpoint(opts *bind.CallOpts) (uint64, [32]byte, *big.Int, error) {
+	var (
+		ret0 = new(uint64)
+		ret1 = new([32]byte)
+		ret2 = new(*big.Int)
+	)
+	out := &[]interface{}{
+		ret0,
+		ret1,
+		ret2,
+	}
+	err := _CheckpointOracle.contract.Call(opts, out, "GetLatestCheckpoint")
+	return *ret0, *ret1, *ret2, err
+}
+
+// GetLatestCheckpoint is a free data retrieval call binding the contract method 0x4d6a304c.
+//
+// Solidity: function GetLatestCheckpoint() constant returns(uint64, bytes32, uint256)
+func (_CheckpointOracle *CheckpointOracleSession) GetLatestCheckpoint() (uint64, [32]byte, *big.Int, error) {
+	return _CheckpointOracle.Contract.GetLatestCheckpoint(&_CheckpointOracle.CallOpts)
+}
+
+// GetLatestCheckpoint is a free data retrieval call binding the contract method 0x4d6a304c.
+//
+// Solidity: function GetLatestCheckpoint() constant returns(uint64, bytes32, uint256)
+func (_CheckpointOracle *CheckpointOracleCallerSession) GetLatestCheckpoint() (uint64, [32]byte, *big.Int, error) {
+	return _CheckpointOracle.Contract.GetLatestCheckpoint(&_CheckpointOracle.CallOpts)
+}
+
+// SetCheckpoint is a paid mutator transaction binding the contract method 0xd459fc46.
+//
+// Solidity: function SetCheckpoint(uint256 _recentNumber, bytes32 _recentHash, bytes32 _hash, uint64 _sectionIndex, uint8[] v, bytes32[] r, bytes32[] s) returns(bool)
+func (_CheckpointOracle *CheckpointOracleTransactor) SetCheckpoint(opts *bind.TransactOpts, _recentNumber *big.Int, _recentHash [32]byte, _hash [32]byte, _sectionIndex uint64, v []uint8, r [][32]byte, s [][32]byte) (*types.Transaction, error) {
+	return _CheckpointOracle.contract.Transact(opts, "SetCheckpoint", _recentNumber, _recentHash, _hash, _sectionIndex, v, r, s)
+}
+
+// SetCheckpoint is a paid mutator transaction binding the contract method 0xd459fc46.
+//
+// Solidity: function SetCheckpoint(uint256 _recentNumber, bytes32 _recentHash, bytes32 _hash, uint64 _sectionIndex, uint8[] v, bytes32[] r, bytes32[] s) returns(bool)
+func (_CheckpointOracle *CheckpointOracleSession) SetCheckpoint(_recentNumber *big.Int, _recentHash [32]byte, _hash [32]byte, _sectionIndex uint64, v []uint8, r [][32]byte, s [][32]byte) (*types.Transaction, error) {
+	return _CheckpointOracle.Contract.SetCheckpoint(&_CheckpointOracle.TransactOpts, _recentNumber, _recentHash, _hash, _sectionIndex, v, r, s)
+}
+
+// SetCheckpoint is a paid mutator transaction binding the contract method 0xd459fc46.
+//
+// Solidity: function SetCheckpoint(uint256 _recentNumber, bytes32 _recentHash, bytes32 _hash, uint64 _sectionIndex, uint8[] v, bytes32[] r, bytes32[] s) returns(bool)
+func (_CheckpointOracle *CheckpointOracleTransactorSession) SetCheckpoint(_recentNumber *big.Int, _recentHash [32]byte, _hash [32]byte, _sectionIndex uint64, v []uint8, r [][32]byte, s [][32]byte) (*types.Transaction, error) {
+	return _CheckpointOracle.Contract.SetCheckpoint(&_CheckpointOracle.TransactOpts, _recentNumber, _recentHash, _hash, _sectionIndex, v, r, s)
+}
+
+// CheckpointOracleNewCheckpointVoteIterator is returned from FilterNewCheckpointVote and is used to iterate over the raw logs and unpacked data for NewCheckpointVote events raised by the CheckpointOracle contract.
+type CheckpointOracleNewCheckpointVoteIterator struct {
+	Event *CheckpointOracleNewCheckpointVote // Event containing the contract specifics and raw log
+
+	contract *bind.BoundContract // Generic contract to use for unpacking event data
+	event    string              // Event name to use for unpacking event data
+
+	logs chan types.Log        // Log channel receiving the found contract events
+	sub  ethereum.Subscription // Subscription for errors, completion and termination
+	done bool                  // Whether the subscription completed delivering logs
+	fail error                 // Occurred error to stop iteration
+}
+
+// Next advances the iterator to the subsequent event, returning whether there
+// are any more events found. In case of a retrieval or parsing error, false is
+// returned and Error() can be queried for the exact failure.
+func (it *CheckpointOracleNewCheckpointVoteIterator) Next() bool {
+	// If the iterator failed, stop iterating
+	if it.fail != nil {
+		return false
+	}
+	// If the iterator completed, deliver directly whatever's available
+	if it.done {
+		select {
+		case log := <-it.logs:
+			it.Event = new(CheckpointOracleNewCheckpointVote)
+			if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil {
+				it.fail = err
+				return false
+			}
+			it.Event.Raw = log
+			return true
+
+		default:
+			return false
+		}
+	}
+	// Iterator still in progress, wait for either a data or an error event
+	select {
+	case log := <-it.logs:
+		it.Event = new(CheckpointOracleNewCheckpointVote)
+		if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil {
+			it.fail = err
+			return false
+		}
+		it.Event.Raw = log
+		return true
+
+	case err := <-it.sub.Err():
+		it.done = true
+		it.fail = err
+		return it.Next()
+	}
+}
+
+// Error returns any retrieval or parsing error occurred during filtering.
+func (it *CheckpointOracleNewCheckpointVoteIterator) Error() error {
+	return it.fail
+}
+
+// Close terminates the iteration process, releasing any pending underlying
+// resources.
+func (it *CheckpointOracleNewCheckpointVoteIterator) Close() error {
+	it.sub.Unsubscribe()
+	return nil
+}
+
+// CheckpointOracleNewCheckpointVote represents a NewCheckpointVote event raised by the CheckpointOracle contract.
+type CheckpointOracleNewCheckpointVote struct {
+	Index          uint64
+	CheckpointHash [32]byte
+	V              uint8
+	R              [32]byte
+	S              [32]byte
+	Raw            types.Log // Blockchain specific contextual infos
+}
+
+// FilterNewCheckpointVote is a free log retrieval operation binding the contract event 0xce51ffa16246bcaf0899f6504f473cd0114f430f566cef71ab7e03d3dde42a41.
+//
+// Solidity: event NewCheckpointVote(uint64 indexed index, bytes32 checkpointHash, uint8 v, bytes32 r, bytes32 s)
+func (_CheckpointOracle *CheckpointOracleFilterer) FilterNewCheckpointVote(opts *bind.FilterOpts, index []uint64) (*CheckpointOracleNewCheckpointVoteIterator, error) {
+
+	var indexRule []interface{}
+	for _, indexItem := range index {
+		indexRule = append(indexRule, indexItem)
+	}
+
+	logs, sub, err := _CheckpointOracle.contract.FilterLogs(opts, "NewCheckpointVote", indexRule)
+	if err != nil {
+		return nil, err
+	}
+	return &CheckpointOracleNewCheckpointVoteIterator{contract: _CheckpointOracle.contract, event: "NewCheckpointVote", logs: logs, sub: sub}, nil
+}
+
+// WatchNewCheckpointVote is a free log subscription operation binding the contract event 0xce51ffa16246bcaf0899f6504f473cd0114f430f566cef71ab7e03d3dde42a41.
+//
+// Solidity: event NewCheckpointVote(uint64 indexed index, bytes32 checkpointHash, uint8 v, bytes32 r, bytes32 s)
+func (_CheckpointOracle *CheckpointOracleFilterer) WatchNewCheckpointVote(opts *bind.WatchOpts, sink chan<- *CheckpointOracleNewCheckpointVote, index []uint64) (event.Subscription, error) {
+
+	var indexRule []interface{}
+	for _, indexItem := range index {
+		indexRule = append(indexRule, indexItem)
+	}
+
+	logs, sub, err := _CheckpointOracle.contract.WatchLogs(opts, "NewCheckpointVote", indexRule)
+	if err != nil {
+		return nil, err
+	}
+	return event.NewSubscription(func(quit <-chan struct{}) error {
+		defer sub.Unsubscribe()
+		for {
+			select {
+			case log := <-logs:
+				// New log arrived, parse the event and forward to the user
+				event := new(CheckpointOracleNewCheckpointVote)
+				if err := _CheckpointOracle.contract.UnpackLog(event, "NewCheckpointVote", log); err != nil {
+					return err
+				}
+				event.Raw = log
+
+				select {
+				case sink <- event:
+				case err := <-sub.Err():
+					return err
+				case <-quit:
+					return nil
+				}
+			case err := <-sub.Err():
+				return err
+			case <-quit:
+				return nil
+			}
+		}
+	}), nil
+}
+
+// ParseNewCheckpointVote is a log parse operation binding the contract event 0xce51ffa16246bcaf0899f6504f473cd0114f430f566cef71ab7e03d3dde42a41.
+//
+// Solidity: event NewCheckpointVote(uint64 indexed index, bytes32 checkpointHash, uint8 v, bytes32 r, bytes32 s)
+func (_CheckpointOracle *CheckpointOracleFilterer) ParseNewCheckpointVote(log types.Log) (*CheckpointOracleNewCheckpointVote, error) {
+	event := new(CheckpointOracleNewCheckpointVote)
+	if err := _CheckpointOracle.contract.UnpackLog(event, "NewCheckpointVote", log); err != nil {
+		return nil, err
+	}
+	return event, nil
+}

contracts/checkpointoracle/contract/oracle.sol

@@ -0,0 +1,174 @@
+pragma solidity ^0.5.10;
+
+/**
+ * @title CheckpointOracle
+ * @author Gary Rong<garyrong@ethereum.org>, Martin Swende <martin.swende@ethereum.org>
+ * @dev Implementation of the blockchain checkpoint registrar.
+ */
+contract CheckpointOracle {
+    /*
+        Events
+    */
+
+    // NewCheckpointVote is emitted when a new checkpoint proposal receives a vote.
+    event NewCheckpointVote(uint64 indexed index, bytes32 checkpointHash, uint8 v, bytes32 r, bytes32 s);
+
+    /*
+        Public Functions
+    */
+    constructor(address[] memory _adminlist, uint _sectionSize, uint _processConfirms, uint _threshold) public {
+        for (uint i = 0; i < _adminlist.length; i++) {
+            admins[_adminlist[i]] = true;
+            adminList.push(_adminlist[i]);
+        }
+        sectionSize = _sectionSize;
+        processConfirms = _processConfirms;
+        threshold = _threshold;
+    }
+
+    /**
+     * @dev Get latest stable checkpoint information.
+     * @return section index
+     * @return checkpoint hash
+     * @return block height associated with checkpoint
+     */
+    function GetLatestCheckpoint()
+    view
+    public
+    returns(uint64, bytes32, uint) {
+        return (sectionIndex, hash, height);
+    }
+
+    // SetCheckpoint sets  a new checkpoint. It accepts a list of signatures
+    // @_recentNumber: a recent blocknumber, for replay protection
+    // @_recentHash : the hash of `_recentNumber`
+    // @_hash : the hash to set at _sectionIndex
+    // @_sectionIndex : the section index to set
+    // @v : the list of v-values
+    // @r : the list or r-values
+    // @s : the list of s-values
+    function SetCheckpoint(
+        uint _recentNumber,
+        bytes32 _recentHash,
+        bytes32 _hash,
+        uint64 _sectionIndex,
+        uint8[] memory v,
+        bytes32[] memory r,
+        bytes32[] memory s)
+        public
+        returns (bool)
+    {
+        // Ensure the sender is authorized.
+        require(admins[msg.sender]);
+
+        // These checks replay protection, so it cannot be replayed on forks,
+        // accidentally or intentionally
+        require(blockhash(_recentNumber) == _recentHash);
+
+        // Ensure the batch of signatures are valid.
+        require(v.length == r.length);
+        require(v.length == s.length);
+
+        // Filter out "future" checkpoint.
+        if (block.number < (_sectionIndex+1)*sectionSize+processConfirms) {
+            return false;
+        }
+        // Filter out "old" announcement
+        if (_sectionIndex < sectionIndex) {
+            return false;
+        }
+        // Filter out "stale" announcement
+        if (_sectionIndex == sectionIndex && (_sectionIndex != 0 || height != 0)) {
+            return false;
+        }
+        // Filter out "invalid" announcement
+        if (_hash == ""){
+            return false;
+        }
+
+        // EIP 191 style signatures
+        //
+        // Arguments when calculating hash to validate
+        // 1: byte(0x19) - the initial 0x19 byte
+        // 2: byte(0) - the version byte (data with intended validator)
+        // 3: this - the validator address
+        // --  Application specific data
+        // 4 : checkpoint section_index(uint64)
+        // 5 : checkpoint hash (bytes32)
+        //     hash = keccak256(checkpoint_index, section_head, cht_root, bloom_root)
+        bytes32 signedHash = keccak256(abi.encodePacked(byte(0x19), byte(0), this, _sectionIndex, _hash));
+
+        address lastVoter = address(0);
+
+        // In order for us not to have to maintain a mapping of who has already
+        // voted, and we don't want to count a vote twice, the signatures must
+        // be submitted in strict ordering.
+        for (uint idx = 0; idx < v.length; idx++){
+            address signer = ecrecover(signedHash, v[idx], r[idx], s[idx]);
+            require(admins[signer]);
+            require(uint256(signer) > uint256(lastVoter));
+            lastVoter = signer;
+            emit NewCheckpointVote(_sectionIndex, _hash, v[idx], r[idx], s[idx]);
+
+            // Sufficient signatures present, update latest checkpoint.
+            if (idx+1 >= threshold){
+                hash = _hash;
+                height = block.number;
+                sectionIndex = _sectionIndex;
+                return true;
+            }
+        }
+        // We shouldn't wind up here, reverting un-emits the events
+        revert();
+    }
+
+    /**
+     * @dev Get all admin addresses
+     * @return address list
+     */
+    function GetAllAdmin()
+    public
+    view
+    returns(address[] memory)
+    {
+        address[] memory ret = new address[](adminList.length);
+        for (uint i = 0; i < adminList.length; i++) {
+            ret[i] = adminList[i];
+        }
+        return ret;
+    }
+
+    /*
+        Fields
+    */
+    // A map of admin users who have the permission to update CHT and bloom Trie root
+    mapping(address => bool) admins;
+
+    // A list of admin users so that we can obtain all admin users.
+    address[] adminList;
+
+    // Latest stored section id
+    uint64 sectionIndex;
+
+    // The block height associated with latest registered checkpoint.
+    uint height;
+
+    // The hash of latest registered checkpoint.
+    bytes32 hash;
+
+    // The frequency for creating a checkpoint
+    //
+    // The default value should be the same as the checkpoint size(32768) in the ethereum.
+    uint sectionSize;
+
+    // The number of confirmations needed before a checkpoint can be registered.
+    // We have to make sure the checkpoint registered will not be invalid due to
+    // chain reorg.
+    //
+    // The default value should be the same as the checkpoint process confirmations(256)
+    // in the ethereum.
+    uint processConfirms;
+
+    // The required signatures to finalize a stable checkpoint.
+    uint threshold;
+}

contracts/checkpointoracle/oracle.go

@@ -0,0 +1,90 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package checkpointoracle is a an on-chain light client checkpoint oracle.
+package checkpointoracle
+
+//go:generate abigen --sol contract/oracle.sol --pkg contract --out contract/oracle.go
+
+import (
+	"errors"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/contracts/checkpointoracle/contract"
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
+// CheckpointOracle is a Go wrapper around an on-chain light client checkpoint oracle.
+type CheckpointOracle struct {
+	contract *contract.CheckpointOracle
+}
+
+// NewCheckpointOracle binds checkpoint contract and returns a registrar instance.
+func NewCheckpointOracle(contractAddr common.Address, backend bind.ContractBackend) (*CheckpointOracle, error) {
+	c, err := contract.NewCheckpointOracle(contractAddr, backend)
+	if err != nil {
+		return nil, err
+	}
+	return &CheckpointOracle{contract: c}, nil
+}
+
+// Contract returns the underlying contract instance.
+func (oracle *CheckpointOracle) Contract() *contract.CheckpointOracle {
+	return oracle.contract
+}
+
+// LookupCheckpointEvents searches checkpoint event for specific section in the
+// given log batches.
+func (oracle *CheckpointOracle) LookupCheckpointEvents(blockLogs [][]*types.Log, section uint64, hash common.Hash) []*contract.CheckpointOracleNewCheckpointVote {
+	var votes []*contract.CheckpointOracleNewCheckpointVote
+
+	for _, logs := range blockLogs {
+		for _, log := range logs {
+			event, err := oracle.contract.ParseNewCheckpointVote(*log)
+			if err != nil {
+				continue
+			}
+			if event.Index == section && common.Hash(event.CheckpointHash) == hash {
+				votes = append(votes, event)
+			}
+		}
+	}
+	return votes
+}
+
+// RegisterCheckpoint registers the checkpoint with a batch of associated signatures
+// that are collected off-chain and sorted by lexicographical order.
+//
+// Notably all signatures given should be transformed to "ethereum style" which transforms
+// v from 0/1 to 27/28 according to the yellow paper.
+func (oracle *CheckpointOracle) RegisterCheckpoint(opts *bind.TransactOpts, index uint64, hash []byte, rnum *big.Int, rhash [32]byte, sigs [][]byte) (*types.Transaction, error) {
+	var (
+		r [][32]byte
+		s [][32]byte
+		v []uint8
+	)
+	for i := 0; i < len(sigs); i++ {
+		if len(sigs[i]) != 65 {
+			return nil, errors.New("invalid signature")
+		}
+		r = append(r, common.BytesToHash(sigs[i][:32]))
+		s = append(s, common.BytesToHash(sigs[i][32:64]))
+		v = append(v, sigs[i][64])
+	}
+	return oracle.contract.SetCheckpoint(opts, rnum, rhash, common.BytesToHash(hash), index, v, r, s)
+}

contracts/checkpointoracle/oracle_test.go

@@ -0,0 +1,336 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package checkpointoracle
+
+import (
+	"bytes"
+	"crypto/ecdsa"
+	"encoding/binary"
+	"errors"
+	"math/big"
+	"reflect"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind/backends"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/contracts/checkpointoracle/contract"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/params"
+)
+
+var (
+	emptyHash = [32]byte{}
+
+	checkpoint0 = params.TrustedCheckpoint{
+		SectionIndex: 0,
+		SectionHead:  common.HexToHash("0x7fa3c32f996c2bfb41a1a65b3d8ea3e0a33a1674cde43678ad6f4235e764d17d"),
+		CHTRoot:      common.HexToHash("0x98fc5d3de23a0fecebad236f6655533c157d26a1aedcd0852a514dc1169e6350"),
+		BloomRoot:    common.HexToHash("0x99b5adb52b337fe25e74c1c6d3835b896bd638611b3aebddb2317cce27a3f9fa"),
+	}
+	checkpoint1 = params.TrustedCheckpoint{
+		SectionIndex: 1,
+		SectionHead:  common.HexToHash("0x2d4dee68102125e59b0cc61b176bd89f0d12b3b91cfaf52ef8c2c82fb920c2d2"),
+		CHTRoot:      common.HexToHash("0x7d428008ece3b4c4ef5439f071930aad0bb75108d381308df73beadcd01ded95"),
+		BloomRoot:    common.HexToHash("0x652571f7736de17e7bbb427ac881474da684c6988a88bf51b10cca9a2ee148f4"),
+	}
+	checkpoint2 = params.TrustedCheckpoint{
+		SectionIndex: 2,
+		SectionHead:  common.HexToHash("0x61c0de578c0115b1dff8ef39aa600588c7c6ecb8a2f102003d7cf4c4146e9291"),
+		CHTRoot:      common.HexToHash("0x407a08a407a2bc3838b74ca3eb206903c9c8a186ccf5ef14af07794efff1970b"),
+		BloomRoot:    common.HexToHash("0x058b4161f558ce295a92925efc57f34f9210d5a30088d7475c183e0d3e58f5ac"),
+	}
+)
+
+var (
+	// The block frequency for creating checkpoint(only used in test)
+	sectionSize = big.NewInt(512)
+
+	// The number of confirmations needed to generate a checkpoint(only used in test).
+	processConfirms = big.NewInt(4)
+)
+
+// validateOperation executes the operation, watches and delivers all events fired by the backend and ensures the
+// correctness by assert function.
+func validateOperation(t *testing.T, c *contract.CheckpointOracle, backend *backends.SimulatedBackend, operation func(),
+	assert func(<-chan *contract.CheckpointOracleNewCheckpointVote) error, opName string) {
+	// Watch all events and deliver them to assert function
+	var (
+		sink   = make(chan *contract.CheckpointOracleNewCheckpointVote)
+		sub, _ = c.WatchNewCheckpointVote(nil, sink, nil)
+	)
+	defer func() {
+		// Close all subscribers
+		sub.Unsubscribe()
+	}()
+	operation()
+
+	// flush pending block
+	backend.Commit()
+	if err := assert(sink); err != nil {
+		t.Errorf("operation {%s} failed, err %s", opName, err)
+	}
+}
+
+// validateEvents checks that the correct number of contract events
+// fired by contract backend.
+func validateEvents(target int, sink interface{}) (bool, []reflect.Value) {
+	chanval := reflect.ValueOf(sink)
+	chantyp := chanval.Type()
+	if chantyp.Kind() != reflect.Chan || chantyp.ChanDir()&reflect.RecvDir == 0 {
+		return false, nil
+	}
+	count := 0
+	var recv []reflect.Value
+	timeout := time.After(1 * time.Second)
+	cases := []reflect.SelectCase{{Chan: chanval, Dir: reflect.SelectRecv}, {Chan: reflect.ValueOf(timeout), Dir: reflect.SelectRecv}}
+	for {
+		chose, v, _ := reflect.Select(cases)
+		if chose == 1 {
+			// Not enough event received
+			return false, nil
+		}
+		count += 1
+		recv = append(recv, v)
+		if count == target {
+			break
+		}
+	}
+	done := time.After(50 * time.Millisecond)
+	cases = cases[:1]
+	cases = append(cases, reflect.SelectCase{Chan: reflect.ValueOf(done), Dir: reflect.SelectRecv})
+	chose, _, _ := reflect.Select(cases)
+	// If chose equal 0, it means receiving redundant events.
+	return chose == 1, recv
+}
+
+func signCheckpoint(addr common.Address, privateKey *ecdsa.PrivateKey, index uint64, hash common.Hash) []byte {
+	// EIP 191 style signatures
+	//
+	// Arguments when calculating hash to validate
+	// 1: byte(0x19) - the initial 0x19 byte
+	// 2: byte(0) - the version byte (data with intended validator)
+	// 3: this - the validator address
+	// --  Application specific data
+	// 4 : checkpoint section_index(uint64)
+	// 5 : checkpoint hash (bytes32)
+	//     hash = keccak256(checkpoint_index, section_head, cht_root, bloom_root)
+	buf := make([]byte, 8)
+	binary.BigEndian.PutUint64(buf, index)
+	data := append([]byte{0x19, 0x00}, append(addr.Bytes(), append(buf, hash.Bytes()...)...)...)
+	sig, _ := crypto.Sign(crypto.Keccak256(data), privateKey)
+	sig[64] += 27 // Transform V from 0/1 to 27/28 according to the yellow paper
+	return sig
+}
+
+// assertSignature verifies whether the recovered signers are equal with expected.
+func assertSignature(addr common.Address, index uint64, hash [32]byte, r, s [32]byte, v uint8, expect common.Address) bool {
+	buf := make([]byte, 8)
+	binary.BigEndian.PutUint64(buf, index)
+	data := append([]byte{0x19, 0x00}, append(addr.Bytes(), append(buf, hash[:]...)...)...)
+	pubkey, err := crypto.Ecrecover(crypto.Keccak256(data), append(r[:], append(s[:], v-27)...))
+	if err != nil {
+		return false
+	}
+	var signer common.Address
+	copy(signer[:], crypto.Keccak256(pubkey[1:])[12:])
+	return bytes.Equal(signer.Bytes(), expect.Bytes())
+}
+
+type Account struct {
+	key  *ecdsa.PrivateKey
+	addr common.Address
+}
+type Accounts []Account
+
+func (a Accounts) Len() int           { return len(a) }
+func (a Accounts) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a Accounts) Less(i, j int) bool { return bytes.Compare(a[i].addr.Bytes(), a[j].addr.Bytes()) < 0 }
+
+func TestCheckpointRegister(t *testing.T) {
+	// Initialize test accounts
+	var accounts Accounts
+	for i := 0; i < 3; i++ {
+		key, _ := crypto.GenerateKey()
+		addr := crypto.PubkeyToAddress(key.PublicKey)
+		accounts = append(accounts, Account{key: key, addr: addr})
+	}
+	sort.Sort(accounts)
+
+	// Deploy registrar contract
+	contractBackend := backends.NewSimulatedBackend(core.GenesisAlloc{accounts[0].addr: {Balance: big.NewInt(1000000000)}, accounts[1].addr: {Balance: big.NewInt(1000000000)}, accounts[2].addr: {Balance: big.NewInt(1000000000)}}, 10000000)
+	defer contractBackend.Close()
+
+	transactOpts := bind.NewKeyedTransactor(accounts[0].key)
+
+	// 3 trusted signers, threshold 2
+	contractAddr, _, c, err := contract.DeployCheckpointOracle(transactOpts, contractBackend, []common.Address{accounts[0].addr, accounts[1].addr, accounts[2].addr}, sectionSize, processConfirms, big.NewInt(2))
+	if err != nil {
+		t.Error("Failed to deploy registrar contract", err)
+	}
+	contractBackend.Commit()
+
+	// getRecent returns block height and hash of the head parent.
+	getRecent := func() (*big.Int, common.Hash) {
+		parentNumber := new(big.Int).Sub(contractBackend.Blockchain().CurrentHeader().Number, big.NewInt(1))
+		parentHash := contractBackend.Blockchain().CurrentHeader().ParentHash
+		return parentNumber, parentHash
+	}
+	// collectSig generates specified number signatures.
+	collectSig := func(index uint64, hash common.Hash, n int, unauthorized *ecdsa.PrivateKey) (v []uint8, r [][32]byte, s [][32]byte) {
+		for i := 0; i < n; i++ {
+			sig := signCheckpoint(contractAddr, accounts[i].key, index, hash)
+			if unauthorized != nil {
+				sig = signCheckpoint(contractAddr, unauthorized, index, hash)
+			}
+			r = append(r, common.BytesToHash(sig[:32]))
+			s = append(s, common.BytesToHash(sig[32:64]))
+			v = append(v, sig[64])
+		}
+		return v, r, s
+	}
+	// insertEmptyBlocks inserts a batch of empty blocks to blockchain.
+	insertEmptyBlocks := func(number int) {
+		for i := 0; i < number; i++ {
+			contractBackend.Commit()
+		}
+	}
+	// assert checks whether the current contract status is same with
+	// the expected.
+	assert := func(index uint64, hash [32]byte, height *big.Int) error {
+		lindex, lhash, lheight, err := c.GetLatestCheckpoint(nil)
+		if err != nil {
+			return err
+		}
+		if lindex != index {
+			return errors.New("latest checkpoint index mismatch")
+		}
+		if !bytes.Equal(lhash[:], hash[:]) {
+			return errors.New("latest checkpoint hash mismatch")
+		}
+		if lheight.Cmp(height) != 0 {
+			return errors.New("latest checkpoint height mismatch")
+		}
+		return nil
+	}
+
+	// Test future checkpoint registration
+	validateOperation(t, c, contractBackend, func() {
+		number, hash := getRecent()
+		v, r, s := collectSig(0, checkpoint0.Hash(), 2, nil)
+		c.SetCheckpoint(transactOpts, number, hash, checkpoint0.Hash(), 0, v, r, s)
+	}, func(events <-chan *contract.CheckpointOracleNewCheckpointVote) error {
+		return assert(0, emptyHash, big.NewInt(0))
+	}, "test future checkpoint registration")
+
+	insertEmptyBlocks(int(sectionSize.Uint64() + processConfirms.Uint64()))
+
+	// Test transaction replay protection
+	validateOperation(t, c, contractBackend, func() {
+		number, _ := getRecent()
+		v, r, s := collectSig(0, checkpoint0.Hash(), 2, nil)
+		hash := common.HexToHash("deadbeef")
+		c.SetCheckpoint(transactOpts, number, hash, checkpoint0.Hash(), 0, v, r, s)
+	}, func(events <-chan *contract.CheckpointOracleNewCheckpointVote) error {
+		return assert(0, emptyHash, big.NewInt(0))
+	}, "test transaction replay protection")
+
+	// Test unauthorized signature checking
+	validateOperation(t, c, contractBackend, func() {
+		number, hash := getRecent()
+		u, _ := crypto.GenerateKey()
+		v, r, s := collectSig(0, checkpoint0.Hash(), 2, u)
+		c.SetCheckpoint(transactOpts, number, hash, checkpoint0.Hash(), 0, v, r, s)
+	}, func(events <-chan *contract.CheckpointOracleNewCheckpointVote) error {
+		return assert(0, emptyHash, big.NewInt(0))
+	}, "test unauthorized signature checking")
+
+	// Test un-multi-signature checkpoint registration
+	validateOperation(t, c, contractBackend, func() {
+		number, hash := getRecent()
+		v, r, s := collectSig(0, checkpoint0.Hash(), 1, nil)
+		c.SetCheckpoint(transactOpts, number, hash, checkpoint0.Hash(), 0, v, r, s)
+	}, func(events <-chan *contract.CheckpointOracleNewCheckpointVote) error {
+		return assert(0, emptyHash, big.NewInt(0))
+	}, "test un-multi-signature checkpoint registration")
+
+	// Test valid checkpoint registration
+	validateOperation(t, c, contractBackend, func() {
+		number, hash := getRecent()
+		v, r, s := collectSig(0, checkpoint0.Hash(), 2, nil)
+		c.SetCheckpoint(transactOpts, number, hash, checkpoint0.Hash(), 0, v, r, s)
+	}, func(events <-chan *contract.CheckpointOracleNewCheckpointVote) error {
+		if valid, recv := validateEvents(2, events); !valid {
+			return errors.New("receive incorrect number of events")
+		} else {
+			for i := 0; i < len(recv); i++ {
+				event := recv[i].Interface().(*contract.CheckpointOracleNewCheckpointVote)
+				if !assertSignature(contractAddr, event.Index, event.CheckpointHash, event.R, event.S, event.V, accounts[i].addr) {
+					return errors.New("recover signer failed")
+				}
+			}
+		}
+		number, _ := getRecent()
+		return assert(0, checkpoint0.Hash(), number.Add(number, big.NewInt(1)))
+	}, "test valid checkpoint registration")
+
+	distance := 3*sectionSize.Uint64() + processConfirms.Uint64() - contractBackend.Blockchain().CurrentHeader().Number.Uint64()
+	insertEmptyBlocks(int(distance))
+
+	// Test uncontinuous checkpoint registration
+	validateOperation(t, c, contractBackend, func() {
+		number, hash := getRecent()
+		v, r, s := collectSig(2, checkpoint2.Hash(), 2, nil)
+		c.SetCheckpoint(transactOpts, number, hash, checkpoint2.Hash(), 2, v, r, s)
+	}, func(events <-chan *contract.CheckpointOracleNewCheckpointVote) error {
+		if valid, recv := validateEvents(2, events); !valid {
+			return errors.New("receive incorrect number of events")
+		} else {
+			for i := 0; i < len(recv); i++ {
+				event := recv[i].Interface().(*contract.CheckpointOracleNewCheckpointVote)
+				if !assertSignature(contractAddr, event.Index, event.CheckpointHash, event.R, event.S, event.V, accounts[i].addr) {
+					return errors.New("recover signer failed")
+				}
+			}
+		}
+		number, _ := getRecent()
+		return assert(2, checkpoint2.Hash(), number.Add(number, big.NewInt(1)))
+	}, "test uncontinuous checkpoint registration")
+
+	// Test old checkpoint registration
+	validateOperation(t, c, contractBackend, func() {
+		number, hash := getRecent()
+		v, r, s := collectSig(1, checkpoint1.Hash(), 2, nil)
+		c.SetCheckpoint(transactOpts, number, hash, checkpoint1.Hash(), 1, v, r, s)
+	}, func(events <-chan *contract.CheckpointOracleNewCheckpointVote) error {
+		number, _ := getRecent()
+		return assert(2, checkpoint2.Hash(), number)
+	}, "test uncontinuous checkpoint registration")
+
+	// Test stale checkpoint registration
+	validateOperation(t, c, contractBackend, func() {
+		number, hash := getRecent()
+		v, r, s := collectSig(2, checkpoint2.Hash(), 2, nil)
+		c.SetCheckpoint(transactOpts, number, hash, checkpoint2.Hash(), 2, v, r, s)
+	}, func(events <-chan *contract.CheckpointOracleNewCheckpointVote) error {
+		number, _ := getRecent()
+		return assert(2, checkpoint2.Hash(), number.Sub(number, big.NewInt(1)))
+	}, "test stale checkpoint registration")
+}

core/bench_test.go

@@ -85,7 +85,7 @@ func genValueTx(nbytes int) func(int, *BlockGen) {
 	return func(i int, gen *BlockGen) {
 		toaddr := common.Address{}
 		data := make([]byte, nbytes)
-		gas, _ := IntrinsicGas(data, false, false)
+		gas, _ := IntrinsicGas(data, false, false, false)
 		tx, _ := types.SignTx(types.NewTransaction(gen.TxNonce(benchRootAddr), toaddr, big.NewInt(1), gas, nil, data), types.HomesteadSigner{}, benchRootKey)
 		gen.AddTx(tx)
 	}

core/blockchain.go

@@ -64,6 +64,8 @@ var (
 	blockValidationTimer = metrics.NewRegisteredTimer("chain/validation", nil)
 	blockExecutionTimer  = metrics.NewRegisteredTimer("chain/execution", nil)
 	blockWriteTimer      = metrics.NewRegisteredTimer("chain/write", nil)
+	blockReorgAddMeter   = metrics.NewRegisteredMeter("chain/reorg/drop", nil)
+	blockReorgDropMeter  = metrics.NewRegisteredMeter("chain/reorg/add", nil)
 
 	blockPrefetchExecuteTimer   = metrics.NewRegisteredTimer("chain/prefetch/executes", nil)
 	blockPrefetchInterruptMeter = metrics.NewRegisteredMeter("chain/prefetch/interrupts", nil)
@@ -75,6 +77,7 @@ const (
 	bodyCacheLimit      = 256
 	blockCacheLimit     = 256
 	receiptsCacheLimit  = 32
+	txLookupCacheLimit  = 1024
 	maxFutureBlocks     = 256
 	maxTimeFutureBlocks = 30
 	badBlockLimit       = 10
@@ -155,6 +158,7 @@ type BlockChain struct {
 	bodyRLPCache  *lru.Cache     // Cache for the most recent block bodies in RLP encoded format
 	receiptsCache *lru.Cache     // Cache for the most recent receipts per block
 	blockCache    *lru.Cache     // Cache for the most recent entire blocks
+	txLookupCache *lru.Cache     // Cache for the most recent transaction lookup data.
 	futureBlocks  *lru.Cache     // future blocks are blocks added for later processing
 
 	quit    chan struct{} // blockchain quit channel
@@ -189,6 +193,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
 	bodyRLPCache, _ := lru.New(bodyCacheLimit)
 	receiptsCache, _ := lru.New(receiptsCacheLimit)
 	blockCache, _ := lru.New(blockCacheLimit)
+	txLookupCache, _ := lru.New(txLookupCacheLimit)
 	futureBlocks, _ := lru.New(maxFutureBlocks)
 	badBlocks, _ := lru.New(badBlockLimit)
 
@@ -204,6 +209,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
 		bodyRLPCache:   bodyRLPCache,
 		receiptsCache:  receiptsCache,
 		blockCache:     blockCache,
+		txLookupCache:  txLookupCache,
 		futureBlocks:   futureBlocks,
 		engine:         engine,
 		vmConfig:       vmConfig,
@@ -440,6 +446,7 @@ func (bc *BlockChain) SetHead(head uint64) error {
 	bc.bodyRLPCache.Purge()
 	bc.receiptsCache.Purge()
 	bc.blockCache.Purge()
+	bc.txLookupCache.Purge()
 	bc.futureBlocks.Purge()
 
 	return bc.loadLastState()
@@ -921,15 +928,24 @@ func (bc *BlockChain) truncateAncient(head uint64) error {
 	bc.bodyRLPCache.Purge()
 	bc.receiptsCache.Purge()
 	bc.blockCache.Purge()
+	bc.txLookupCache.Purge()
 	bc.futureBlocks.Purge()
 
 	log.Info("Rewind ancient data", "number", head)
 	return nil
 }
 
+// numberHash is just a container for a number and a hash, to represent a block
+type numberHash struct {
+	number uint64
+	hash   common.Hash
+}
+
 // InsertReceiptChain attempts to complete an already existing header chain with
 // transaction and receipt data.
 func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain []types.Receipts, ancientLimit uint64) (int, error) {
+	// We don't require the chainMu here since we want to maximize the
+	// concurrency of header insertion and receipt insertion.
 	bc.wg.Add(1)
 	defer bc.wg.Done()
 
@@ -962,19 +978,21 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 	// updateHead updates the head fast sync block if the inserted blocks are better
 	// and returns a indicator whether the inserted blocks are canonical.
 	updateHead := func(head *types.Block) bool {
-		var isCanonical bool
 		bc.chainmu.Lock()
-		if td := bc.GetTd(head.Hash(), head.NumberU64()); td != nil { // Rewind may have occurred, skip in that case
+
-			currentFastBlock := bc.CurrentFastBlock()
+		// Rewind may have occurred, skip in that case.
+		if bc.CurrentHeader().Number.Cmp(head.Number()) >= 0 {
+			currentFastBlock, td := bc.CurrentFastBlock(), bc.GetTd(head.Hash(), head.NumberU64())
 			if bc.GetTd(currentFastBlock.Hash(), currentFastBlock.NumberU64()).Cmp(td) < 0 {
 				rawdb.WriteHeadFastBlockHash(bc.db, head.Hash())
 				bc.currentFastBlock.Store(head)
 				headFastBlockGauge.Update(int64(head.NumberU64()))
-				isCanonical = true
+				bc.chainmu.Unlock()
+				return true
 			}
 		}
 		bc.chainmu.Unlock()
-		return isCanonical
+		return false
 	}
 	// writeAncient writes blockchain and corresponding receipt chain into ancient store.
 	//
@@ -994,7 +1012,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 				}
 			}
 		}()
-		var deleted types.Blocks
+		var deleted []*numberHash
 		for i, block := range blockChain {
 			// Short circuit insertion if shutting down or processing failed
 			if atomic.LoadInt32(&bc.procInterrupt) == 1 {
@@ -1029,12 +1047,40 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 
 				// Always keep genesis block in active database.
 				if b.NumberU64() != 0 {
-					deleted = append(deleted, b)
+					deleted = append(deleted, &numberHash{b.NumberU64(), b.Hash()})
 				}
 				if time.Since(logged) > 8*time.Second {
 					log.Info("Migrating ancient blocks", "count", count, "elapsed", common.PrettyDuration(time.Since(start)))
 					logged = time.Now()
 				}
+				// Don't collect too much in-memory, write it out every 100K blocks
+				if len(deleted) > 100000 {
+
+					// Sync the ancient store explicitly to ensure all data has been flushed to disk.
+					if err := bc.db.Sync(); err != nil {
+						return 0, err
+					}
+					// Wipe out canonical block data.
+					for _, nh := range deleted {
+						rawdb.DeleteBlockWithoutNumber(batch, nh.hash, nh.number)
+						rawdb.DeleteCanonicalHash(batch, nh.number)
+					}
+					if err := batch.Write(); err != nil {
+						return 0, err
+					}
+					batch.Reset()
+					// Wipe out side chain too.
+					for _, nh := range deleted {
+						for _, hash := range rawdb.ReadAllHashes(bc.db, nh.number) {
+							rawdb.DeleteBlock(batch, hash, nh.number)
+						}
+					}
+					if err := batch.Write(); err != nil {
+						return 0, err
+					}
+					batch.Reset()
+					deleted = deleted[0:]
+				}
 			}
 			if count > 0 {
 				log.Info("Migrated ancient blocks", "count", count, "elapsed", common.PrettyDuration(time.Since(start)))
@@ -1062,7 +1108,11 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 		previous = nil // disable rollback explicitly
 
 		// Wipe out canonical block data.
-		for _, block := range append(deleted, blockChain...) {
+		for _, nh := range deleted {
+			rawdb.DeleteBlockWithoutNumber(batch, nh.hash, nh.number)
+			rawdb.DeleteCanonicalHash(batch, nh.number)
+		}
+		for _, block := range blockChain {
 			// Always keep genesis block in active database.
 			if block.NumberU64() != 0 {
 				rawdb.DeleteBlockWithoutNumber(batch, block.Hash(), block.NumberU64())
@@ -1075,7 +1125,12 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 		batch.Reset()
 
 		// Wipe out side chain too.
-		for _, block := range append(deleted, blockChain...) {
+		for _, nh := range deleted {
+			for _, hash := range rawdb.ReadAllHashes(bc.db, nh.number) {
+				rawdb.DeleteBlock(batch, hash, nh.number)
+			}
+		}
+		for _, block := range blockChain {
 			// Always keep genesis block in active database.
 			if block.NumberU64() != 0 {
 				for _, hash := range rawdb.ReadAllHashes(bc.db, block.NumberU64()) {
@@ -1695,6 +1750,11 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i
 			canonical := bc.GetBlockByNumber(number)
 			if canonical != nil && canonical.Hash() == block.Hash() {
 				// Not a sidechain block, this is a re-import of a canon block which has it's state pruned
+
+				// Collect the TD of the block. Since we know it's a canon one,
+				// we can get it directly, and not (like further below) use
+				// the parent and then add the block on top
+				externTd = bc.GetTd(block.Hash(), block.NumberU64())
 				continue
 			}
 			if canonical != nil && canonical.Root() == block.Root() {
@@ -1875,12 +1935,16 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
 	}
 	// Ensure the user sees large reorgs
 	if len(oldChain) > 0 && len(newChain) > 0 {
-		logFn := log.Debug
+		logFn := log.Info
+		msg := "Chain reorg detected"
 		if len(oldChain) > 63 {
+			msg = "Large chain reorg detected"
 			logFn = log.Warn
 		}
-		logFn("Chain split detected", "number", commonBlock.Number(), "hash", commonBlock.Hash(),
+		logFn(msg, "number", commonBlock.Number(), "hash", commonBlock.Hash(),
 			"drop", len(oldChain), "dropfrom", oldChain[0].Hash(), "add", len(newChain), "addfrom", newChain[0].Hash())
+		blockReorgAddMeter.Mark(int64(len(newChain)))
+		blockReorgDropMeter.Mark(int64(len(oldChain)))
 	} else {
 		log.Error("Impossible reorg, please file an issue", "oldnum", oldBlock.Number(), "oldhash", oldBlock.Hash(), "newnum", newBlock.Number(), "newhash", newBlock.Hash())
 	}
@@ -2099,7 +2163,23 @@ func (bc *BlockChain) GetHeaderByNumber(number uint64) *types.Header {
 	return bc.hc.GetHeaderByNumber(number)
 }
 
-// Config retrieves the blockchain's chain configuration.
+// GetTransactionLookup retrieves the lookup associate with the given transaction
+// hash from the cache or database.
+func (bc *BlockChain) GetTransactionLookup(hash common.Hash) *rawdb.LegacyTxLookupEntry {
+	// Short circuit if the txlookup already in the cache, retrieve otherwise
+	if lookup, exist := bc.txLookupCache.Get(hash); exist {
+		return lookup.(*rawdb.LegacyTxLookupEntry)
+	}
+	tx, blockHash, blockNumber, txIndex := rawdb.ReadTransaction(bc.db, hash)
+	if tx == nil {
+		return nil
+	}
+	lookup := &rawdb.LegacyTxLookupEntry{BlockHash: blockHash, BlockIndex: blockNumber, Index: txIndex}
+	bc.txLookupCache.Add(hash, lookup)
+	return lookup
+}
+
+// Config retrieves the chain's fork configuration.
 func (bc *BlockChain) Config() *params.ChainConfig { return bc.chainConfig }
 
 // Engine retrieves the blockchain's consensus engine.