diff --git a/.travis.yml b/.travis.yml index 7e2ce6c89f..9df63343f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,11 +3,10 @@ go_import_path: github.com/expanse-org/go-expanse sudo: false matrix: include: - - os: linux dist: trusty sudo: required - go: 1.8.1 + go: 1.8.3 script: - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install fuse - sudo modprobe fuse @@ -16,14 +15,13 @@ matrix: - go run build/ci.go install - os: osx - go: 1.8.1 + go: 1.8.3 sudo: required script: - brew update - brew install caskroom/cask/brew-cask - brew cask install osxfuse - go run build/ci.go install - install: - go get golang.org/x/tools/cmd/cover script: diff --git a/Dockerfile b/Dockerfile index 6ed68c6c47..1bb8428b25 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,11 +4,12 @@ ADD . /go-expanse RUN \ apk add --update git go make gcc musl-dev linux-headers && \ (cd go-expanse && make gexp) && \ - cp go-expanse/build/bin/gexp /gexp && \ + cp go-expanse/build/bin/gexp /usr/local/bin/ && \ apk del git go make gcc musl-dev linux-headers && \ rm -rf /go-expanse && rm -rf /var/cache/apk/* EXPOSE 9656 EXPOSE 42786 +EXPOSE 42786/udp -ENTRYPOINT ["/gexp"] +ENTRYPOINT ["gexp"] diff --git a/README.md b/README.md index d02cdd244d..8ec1b576ce 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ This command will: (via the trailing `console` subcommand) through which you can invoke all official [`web3` methods](https://github.com/expanse-org/wiki/wiki/JavaScript-API) as well as Gexp's own [management APIs](https://github.com/expanse-org/go-expanse/wiki/Management-APIs). This too is optional and if you leave it out you can always attach to an already running Gexp instance - with `gexp --attach`. + with `gexp attach`. ### Full node on the Expanse test network @@ -84,21 +84,24 @@ $ gexp --testnet --fast --cache=512 console ``` The `--fast`, `--cache` flags and `console` subcommand have the exact same meaning as above and they -are equially useful on the testnet too. Please see above for their explanations if you've skipped to +are equally useful on the testnet too. Please see above for their explanations if you've skipped to here. Specifying the `--testnet` flag however will reconfigure your Gexp instance a bit: * Instead of using the default data directory (`~/.expanse` on Linux for example), Gexp will nest - itself one level deeper into a `testnet` subfolder (`~/.expanse/testnet` on Linux). - * Instead of connecting the main Expanse network, the client will connect to the test network, + itself one level deeper into a `testnet` subfolder (`~/.expanse/testnet` on Linux). Note, on OSX + and Linux this also means that attaching to a running testnet node requires the use of a custom + endpoint since `gexp attach` will try to attach to a production node endpoint by default. E.g. + `gexp attach /testnet/gexp.ipc`. Windows users are not affected by this. + * Instead of connecting the main Ethereum network, the client will connect to the test network, +>>>>>>> 6171d01b1195abd7ac75044dcd507d4758d83cde which uses different P2P bootnodes, different network IDs and genesis states. *Note: Although there are some internal protective measures to prevent transactions from crossing -over between the main network and test network (different starting nonces), you should make sure to -always use separate accounts for play-money and real-money. Unless you manually move accounts, Gexp -will by default correctly separate the two networks and will not make any accounts available between -them.* +over between the main network and test network, you should make sure to always use separate accounts +for play-money and real-money. Unless you manually move accounts, Gexp will by default correctly +separate the two networks and will not make any accounts available between them.* #### Docker quick start @@ -162,6 +165,12 @@ and agree upon. This consists of a small JSON file (e.g. call it `genesis.json`) ```json { + "config": { + "chainId": 0, + "homesteadBlock": 0, + "eip155Block": 0, + "eip158Block": 0 + }, "alloc" : {}, "coinbase" : "0x0000000000000000000000000000000000000000", "difficulty" : "0x20000", diff --git a/VERSION b/VERSION index 9c6d6293b1..ec70f75560 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.6.1 +1.6.6 diff --git a/accounts/keystore/key.go b/accounts/keystore/key.go index 42038a37cc..3c464ce1ea 100644 --- a/accounts/keystore/key.go +++ b/accounts/keystore/key.go @@ -124,14 +124,13 @@ func (k *Key) UnmarshalJSON(j []byte) (err error) { if err != nil { return err } - - privkey, err := hex.DecodeString(keyJSON.PrivateKey) + privkey, err := crypto.HexToECDSA(keyJSON.PrivateKey) if err != nil { return err } k.Address = common.BytesToAddress(addr) - k.PrivateKey = crypto.ToECDSA(privkey) + k.PrivateKey = privkey return nil } diff --git a/accounts/keystore/keystore.go b/accounts/keystore/keystore.go index 28c891eea1..a9c921af24 100644 --- a/accounts/keystore/keystore.go +++ b/accounts/keystore/keystore.go @@ -450,7 +450,6 @@ func (ks *KeyStore) ImportECDSA(priv *ecdsa.PrivateKey, passphrase string) (acco if ks.cache.hasAddress(key.Address) { return accounts.Account{}, fmt.Errorf("account already exists") } - return ks.importKey(key, passphrase) } diff --git a/accounts/keystore/keystore_passphrase.go b/accounts/keystore/keystore_passphrase.go index dd414d21e0..e54165c746 100644 --- a/accounts/keystore/keystore_passphrase.go +++ b/accounts/keystore/keystore_passphrase.go @@ -182,7 +182,8 @@ func DecryptKey(keyjson []byte, auth string) (*Key, error) { if err != nil { return nil, err } - key := crypto.ToECDSA(keyBytes) + key := crypto.ToECDSAUnsafe(keyBytes) + return &Key{ Id: uuid.UUID(keyId), Address: crypto.PubkeyToAddress(key.PublicKey), diff --git a/accounts/keystore/keystore_passphrase_test.go b/accounts/keystore/keystore_passphrase_test.go index c4b85b519e..64b0ae9386 100644 --- a/accounts/keystore/keystore_passphrase_test.go +++ b/accounts/keystore/keystore_passphrase_test.go @@ -46,7 +46,7 @@ func TestKeyEncryptDecrypt(t *testing.T) { // Decrypt with the correct password key, err := DecryptKey(keyjson, password) if err != nil { - t.Errorf("test %d: json key failed to decrypt: %v", i, err) + t.Fatalf("test %d: json key failed to decrypt: %v", i, err) } if key.Address != address { t.Errorf("test %d: key address mismatch: have %x, want %x", i, key.Address, address) diff --git a/accounts/keystore/presale.go b/accounts/keystore/presale.go index dfc5418bd8..6bea28ef3d 100644 --- a/accounts/keystore/presale.go +++ b/accounts/keystore/presale.go @@ -74,7 +74,8 @@ func decryptPreSaleKey(fileContent []byte, password string) (key *Key, err error return nil, err } ethPriv := crypto.Keccak256(plainText) - ecKey := crypto.ToECDSA(ethPriv) + ecKey := crypto.ToECDSAUnsafe(ethPriv) + key = &Key{ Id: nil, Address: crypto.PubkeyToAddress(ecKey.PublicKey), diff --git a/appveyor.yml b/appveyor.yml index 85ac1cccdb..90cbb2f954 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -22,8 +22,8 @@ environment: install: - rmdir C:\go /s /q - - appveyor DownloadFile https://storage.googleapis.com/golang/go1.8.1.windows-%GETH_ARCH%.zip - - 7z x go1.8.1.windows-%GETH_ARCH%.zip -y -oC:\ > NUL + - appveyor DownloadFile https://storage.googleapis.com/golang/go1.8.3.windows-%GETH_ARCH%.zip + - 7z x go1.8.3.windows-%GETH_ARCH%.zip -y -oC:\ > NUL - go version - gcc --version diff --git a/build/ci.go b/build/ci.go index af889f14e5..af15f9b0ce 100644 --- a/build/ci.go +++ b/build/ci.go @@ -77,6 +77,7 @@ var ( executablePath("puppeth"), executablePath("rlpdump"), executablePath("swarm"), + executablePath("wnode"), } // A debian package is created for all executables listed here. @@ -109,6 +110,10 @@ var ( Name: "swarm", Description: "Expanse Swarm daemon and tools", }, + { + Name: "wnode", + Description: "Ethereum Whisper diagnostic tool", + }, } // Distros for which packages are created. diff --git a/build/update-license.go b/build/update-license.go index b4b3dbc2d8..a43740287c 100644 --- a/build/update-license.go +++ b/build/update-license.go @@ -47,11 +47,14 @@ var ( // boring stuff "vendor/", "tests/files/", "build/", // don't relicense vendored sources - "crypto/sha3/", "crypto/ecies/", "log/", - "crypto/secp256k1/curve.go", - "consensus/ethash/xor.go", - "internal/jsre/deps", "cmd/internal/browser", + "consensus/ethash/xor.go", + "crypto/bn256/", + "crypto/ecies/", + "crypto/secp256k1/curve.go", + "crypto/sha3/", + "internal/jsre/deps", + "log/", // don't license generated files "contracts/chequebook/contract/", "contracts/ens/contract/", diff --git a/cmd/bootnode/main.go b/cmd/bootnode/main.go index af1a2263e2..22e07bb6c3 100644 --- a/cmd/bootnode/main.go +++ b/cmd/bootnode/main.go @@ -68,6 +68,7 @@ func main() { if err = crypto.SaveECDSA(*genKey, nodeKey); err != nil { utils.Fatalf("%v", err) } + return case *nodeKeyFile == "" && *nodeKeyHex == "": utils.Fatalf("Use -nodekey or -nodekeyhex to specify a private key") case *nodeKeyFile != "" && *nodeKeyHex != "": diff --git a/cmd/evm/main.go b/cmd/evm/main.go index f6fb950653..72adac3648 100644 --- a/cmd/evm/main.go +++ b/cmd/evm/main.go @@ -35,6 +35,18 @@ var ( Name: "debug", Usage: "output full trace logs", } + MemProfileFlag = cli.StringFlag{ + Name: "memprofile", + Usage: "creates a memory profile at the given path", + } + CPUProfileFlag = cli.StringFlag{ + Name: "cpuprofile", + Usage: "creates a CPU profile at the given path", + } + StatDumpFlag = cli.BoolFlag{ + Name: "statdump", + Usage: "displays stack and heap memory information", + } CodeFlag = cli.StringFlag{ Name: "code", Usage: "EVM code", @@ -93,6 +105,9 @@ func init() { DumpFlag, InputFlag, DisableGasMeteringFlag, + MemProfileFlag, + CPUProfileFlag, + StatDumpFlag, } app.Commands = []cli.Command{ compileCommand, diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index 656d2f90d9..d8364d6ec1 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -21,6 +21,7 @@ import ( "fmt" "io/ioutil" "os" + "runtime/pprof" "time" goruntime "runtime" @@ -108,6 +109,19 @@ func runCmd(ctx *cli.Context) error { }, } + if cpuProfilePath := ctx.GlobalString(CPUProfileFlag.Name); cpuProfilePath != "" { + f, err := os.Create(cpuProfilePath) + if err != nil { + fmt.Println("could not create CPU profile: ", err) + os.Exit(1) + } + if err := pprof.StartCPUProfile(f); err != nil { + fmt.Println("could not start CPU profile: ", err) + os.Exit(1) + } + defer pprof.StopCPUProfile() + } + tstart := time.Now() if ctx.GlobalBool(CreateFlag.Name) { input := append(code, common.Hex2Bytes(ctx.GlobalString(InputFlag.Name))...) @@ -125,12 +139,27 @@ func runCmd(ctx *cli.Context) error { fmt.Println(string(statedb.Dump())) } + if memProfilePath := ctx.GlobalString(MemProfileFlag.Name); memProfilePath != "" { + f, err := os.Create(memProfilePath) + if err != nil { + fmt.Println("could not create memory profile: ", err) + os.Exit(1) + } + if err := pprof.WriteHeapProfile(f); err != nil { + fmt.Println("could not write memory profile: ", err) + os.Exit(1) + } + f.Close() + } + if ctx.GlobalBool(DebugFlag.Name) { fmt.Fprintln(os.Stderr, "#### TRACE ####") vm.WriteTrace(os.Stderr, logger.StructLogs()) fmt.Fprintln(os.Stderr, "#### LOGS ####") vm.WriteLogs(os.Stderr, statedb.Logs()) + } + if ctx.GlobalBool(StatDumpFlag.Name) { var mem goruntime.MemStats goruntime.ReadMemStats(&mem) fmt.Fprintf(os.Stderr, `evm execution time: %v diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index 2ee148b5c3..632afd04c8 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -27,10 +27,13 @@ import ( "fmt" "html/template" "io/ioutil" + "math" "math/big" "net/http" + "net/url" "os" "path/filepath" + "strconv" "strings" "sync" "time" @@ -60,12 +63,13 @@ var ( apiPortFlag = flag.Int("apiport", 8080, "Listener port for the HTTP API connection") ethPortFlag = flag.Int("ethport", 30303, "Listener port for the devp2p connection") bootFlag = flag.String("bootnodes", "", "Comma separated bootnode enode URLs to seed with") - netFlag = flag.Int("network", 0, "Network ID to use for the Ethereum protocol") + netFlag = flag.Uint64("network", 0, "Network ID to use for the Ethereum protocol") statsFlag = flag.String("ethstats", "", "Ethstats network monitoring auth string") netnameFlag = flag.String("faucet.name", "", "Network name to assign to the faucet") payoutFlag = flag.Int("faucet.amount", 1, "Number of Ethers to pay out per user request") minutesFlag = flag.Int("faucet.minutes", 1440, "Number of minutes to wait between funding rounds") + tiersFlag = flag.Int("faucet.tiers", 3, "Number of funding tiers to enable (x3 time, x2.5 funds)") accJSONFlag = flag.String("account.json", "", "Key json file to fund user requests with") accPassFlag = flag.String("account.pass", "", "Decryption password to access faucet funds") @@ -73,6 +77,9 @@ var ( githubUser = flag.String("github.user", "", "GitHub user to authenticate with for Gist access") githubToken = flag.String("github.token", "", "GitHub personal token to access Gists with") + captchaToken = flag.String("captcha.token", "", "Recaptcha site key to authenticate client side") + captchaSecret = flag.String("captcha.secret", "", "Recaptcha secret key to authenticate server side") + logFlag = flag.Int("loglevel", 3, "Log level to use for Ethereum and the faucet") ) @@ -85,21 +92,47 @@ func main() { flag.Parse() log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*logFlag), log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + // Construct the payout tiers + amounts := make([]string, *tiersFlag) + periods := make([]string, *tiersFlag) + for i := 0; i < *tiersFlag; i++ { + // Calculate the amount for the next tier and format it + amount := float64(*payoutFlag) * math.Pow(2.5, float64(i)) + amounts[i] = fmt.Sprintf("%s Ethers", strconv.FormatFloat(amount, 'f', -1, 64)) + if amount == 1 { + amounts[i] = strings.TrimSuffix(amounts[i], "s") + } + // Calculate the period for the next tier and format it + period := *minutesFlag * int(math.Pow(3, float64(i))) + periods[i] = fmt.Sprintf("%d mins", period) + if period%60 == 0 { + period /= 60 + periods[i] = fmt.Sprintf("%d hours", period) + + if period%24 == 0 { + period /= 24 + periods[i] = fmt.Sprintf("%d days", period) + } + } + if period == 1 { + periods[i] = strings.TrimSuffix(periods[i], "s") + } + } // Load up and render the faucet website tmpl, err := Asset("faucet.html") if err != nil { log.Crit("Failed to load the faucet template", "err", err) } - period := fmt.Sprintf("%d minute(s)", *minutesFlag) - if *minutesFlag%60 == 0 { - period = fmt.Sprintf("%d hour(s)", *minutesFlag/60) - } website := new(bytes.Buffer) - template.Must(template.New("").Parse(string(tmpl))).Execute(website, map[string]interface{}{ - "Network": *netnameFlag, - "Amount": *payoutFlag, - "Period": period, + err = template.Must(template.New("").Parse(string(tmpl))).Execute(website, map[string]interface{}{ + "Network": *netnameFlag, + "Amounts": amounts, + "Periods": periods, + "Recaptcha": *captchaToken, }) + if err != nil { + log.Crit("Failed to render the faucet template", "err", err) + } // Load and parse the genesis block requested by the user blob, err := ioutil.ReadFile(*genesisFlag) if err != nil { @@ -166,15 +199,15 @@ type faucet struct { nonce uint64 // Current pending nonce of the faucet price *big.Int // Current gas price to issue funds with - conns []*websocket.Conn // Currently live websocket connections - history map[string]time.Time // History of users and their funding requests - reqs []*request // Currently pending funding requests - update chan struct{} // Channel to signal request updates + conns []*websocket.Conn // Currently live websocket connections + timeouts map[string]time.Time // History of users and their funding timeouts + reqs []*request // Currently pending funding requests + update chan struct{} // Channel to signal request updates lock sync.RWMutex // Lock protecting the faucet's internals } -func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network int, stats string, ks *keystore.KeyStore, index []byte) (*faucet, error) { +func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network uint64, stats string, ks *keystore.KeyStore, index []byte) (*faucet, error) { // Assemble the raw devp2p protocol stack stack, err := node.New(&node.Config{ Name: "geth", @@ -236,7 +269,7 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network i index: index, keystore: ks, account: ks.Accounts()[0], - history: make(map[string]time.Time), + timeouts: make(map[string]time.Time), update: make(chan struct{}, 1), }, nil } @@ -290,14 +323,23 @@ func (f *faucet) apiHandler(conn *websocket.Conn) { "peers": f.stack.Server().PeerCount(), "requests": f.reqs, }) - header, _ := f.client.HeaderByNumber(context.Background(), nil) - websocket.JSON.Send(conn, header) + // Send the initial block to the client + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + header, err := f.client.HeaderByNumber(ctx, nil) + cancel() + if err != nil { + log.Error("Failed to retrieve latest header", "err", err) + } else { + websocket.JSON.Send(conn, header) + } // Keep reading requests from the websocket until the connection breaks for { // Fetch the next funding request and validate against github var msg struct { - URL string `json:"url"` + URL string `json:"url"` + Tier uint `json:"tier"` + Captcha string `json:"captcha"` } if err := websocket.JSON.Receive(conn, &msg); err != nil { return @@ -306,8 +348,39 @@ func (f *faucet) apiHandler(conn *websocket.Conn) { websocket.JSON.Send(conn, map[string]string{"error": "URL doesn't link to GitHub Gists"}) continue } - log.Info("Faucet funds requested", "gist", msg.URL) + if msg.Tier >= uint(*tiersFlag) { + websocket.JSON.Send(conn, map[string]string{"error": "Invalid funding tier requested"}) + continue + } + log.Info("Faucet funds requested", "gist", msg.URL, "tier", msg.Tier) + // If captcha verifications are enabled, make sure we're not dealing with a robot + if *captchaToken != "" { + form := url.Values{} + form.Add("secret", *captchaSecret) + form.Add("response", msg.Captcha) + + res, err := http.PostForm("https://www.google.com/recaptcha/api/siteverify", form) + if err != nil { + websocket.JSON.Send(conn, map[string]string{"error": err.Error()}) + continue + } + var result struct { + Success bool `json:"success"` + Errors json.RawMessage `json:"error-codes"` + } + err = json.NewDecoder(res.Body).Decode(&result) + res.Body.Close() + if err != nil { + websocket.JSON.Send(conn, map[string]string{"error": err.Error()}) + continue + } + if !result.Success { + log.Warn("Captcha verification failed", "err", string(result.Errors)) + websocket.JSON.Send(conn, map[string]string{"error": "Beep-bop, you're a robot!"}) + continue + } + } // Retrieve the gist from the GitHub Gist APIs parts := strings.Split(msg.URL, "/") req, _ := http.NewRequest("GET", "https://api.github.com/gists/"+parts[len(parts)-1], nil) @@ -334,7 +407,7 @@ func (f *faucet) apiHandler(conn *websocket.Conn) { continue } if gist.Owner.Login == "" { - websocket.JSON.Send(conn, map[string]string{"error": "Nice try ;)"}) + websocket.JSON.Send(conn, map[string]string{"error": "Anonymous Gists not allowed"}) continue } // Iterate over all the files and look for Ethereum addresses @@ -348,15 +421,30 @@ func (f *faucet) apiHandler(conn *websocket.Conn) { websocket.JSON.Send(conn, map[string]string{"error": "No Ethereum address found to fund"}) continue } + // Validate the user's existence since the API is unhelpful here + if res, err = http.Head("https://github.com/" + gist.Owner.Login); err != nil { + websocket.JSON.Send(conn, map[string]string{"error": err.Error()}) + continue + } + res.Body.Close() + + if res.StatusCode != 200 { + websocket.JSON.Send(conn, map[string]string{"error": "Invalid user... boom!"}) + continue + } // Ensure the user didn't request funds too recently f.lock.Lock() var ( fund bool - elapsed time.Duration + timeout time.Time ) - if elapsed = time.Since(f.history[gist.Owner.Login]); elapsed > time.Duration(*minutesFlag)*time.Minute { + if timeout = f.timeouts[gist.Owner.Login]; time.Now().After(timeout) { // User wasn't funded recently, create the funding transaction - tx := types.NewTransaction(f.nonce+uint64(len(f.reqs)), address, new(big.Int).Mul(big.NewInt(int64(*payoutFlag)), ether), big.NewInt(21000), f.price, nil) + amount := new(big.Int).Mul(big.NewInt(int64(*payoutFlag)), ether) + amount = new(big.Int).Mul(amount, new(big.Int).Exp(big.NewInt(5), big.NewInt(int64(msg.Tier)), nil)) + amount = new(big.Int).Div(amount, new(big.Int).Exp(big.NewInt(2), big.NewInt(int64(msg.Tier)), nil)) + + tx := types.NewTransaction(f.nonce+uint64(len(f.reqs)), address, amount, big.NewInt(21000), f.price, nil) signed, err := f.keystore.SignTx(f.account, tx, f.config.ChainId) if err != nil { websocket.JSON.Send(conn, map[string]string{"error": err.Error()}) @@ -375,14 +463,14 @@ func (f *faucet) apiHandler(conn *websocket.Conn) { Time: time.Now(), Tx: signed, }) - f.history[gist.Owner.Login] = time.Now() + f.timeouts[gist.Owner.Login] = time.Now().Add(time.Duration(*minutesFlag*int(math.Pow(3, float64(msg.Tier)))) * time.Minute) fund = true } f.lock.Unlock() // Send an error if too frequent funding, othewise a success if !fund { - websocket.JSON.Send(conn, map[string]string{"error": fmt.Sprintf("User already funded %s ago", common.PrettyDuration(elapsed))}) + websocket.JSON.Send(conn, map[string]string{"error": fmt.Sprintf("%s left until next allowance", common.PrettyDuration(timeout.Sub(time.Now())))}) continue } websocket.JSON.Send(conn, map[string]string{"success": fmt.Sprintf("Funding request accepted for %s into %s", gist.Owner.Login, address.Hex())}) diff --git a/cmd/faucet/faucet.html b/cmd/faucet/faucet.html index 570145ea2b..56dd376236 100644 --- a/cmd/faucet/faucet.html +++ b/cmd/faucet/faucet.html @@ -51,9 +51,13 @@
- + + -
+ {{if .Recaptcha}} +
{{end}}
@@ -76,8 +80,9 @@

How does this work?

-

This Ether faucet is running on the {{.Network}} network. To prevent malicious actors from exhausting all available funds or accumulating enough Ether to mount long running spam attacks, requests are tied to GitHub accounts. Anyone having a GitHub account may request funds within the permitted limit of {{.Amount}} Ether(s) / {{.Period}}.

+

This Ether faucet is running on the {{.Network}} network. To prevent malicious actors from exhausting all available funds or accumulating enough Ether to mount long running spam attacks, requests are tied to GitHub accounts. Anyone having a GitHub account may request funds within the permitted limits.

To request funds, simply create a GitHub Gist with your Ethereum address pasted into the contents (the file name doesn't matter), copy paste the gists URL into the above input box and fire away! You can track the current pending requests below the input field to see how much you have to wait until your turn comes.

+ {{if .Recaptcha}}The faucet is running invisible reCaptcha protection against bots.{{end}}
@@ -87,10 +92,12 @@ // Global variables to hold the current status of the faucet var attempt = 0; var server; + var tier = 0; // Define the function that submits a gist url to the server - var submit = function() { - server.send(JSON.stringify({url: $("#gist")[0].value})); + var submit = function({{if .Recaptcha}}captcha{{end}}) { + server.send(JSON.stringify({url: $("#gist")[0].value, tier: tier{{if .Recaptcha}}, captcha: captcha{{end}}}));{{if .Recaptcha}} + grecaptcha.reset();{{end}} }; // Define a method to reconnect upon server loss var reconnect = function() { @@ -134,10 +141,10 @@ } } server.onclose = function() { setTimeout(reconnect, 3000); }; - server.onerror = function() { setTimeout(reconnect, 3000); }; } // Establish a websocket connection to the API server reconnect(); - + {{if .Recaptcha}} + {{end}} diff --git a/cmd/faucet/website.go b/cmd/faucet/website.go index 32650fec40..3151ab5842 100644 --- a/cmd/faucet/website.go +++ b/cmd/faucet/website.go @@ -68,7 +68,7 @@ func (fi bindataFileInfo) Sys() interface{} { return nil } -var _faucetHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x58\xe1\x72\xdb\x36\x12\xfe\x2d\x3f\xc5\x86\x77\xad\xa5\xb1\x49\xca\x71\x26\xed\xc8\xa4\x3a\x99\x36\x97\xf6\xe6\xa6\xed\x5c\xd3\xb9\xeb\xb4\x9d\x1b\x90\x5c\x8a\xb0\x41\x80\x05\x16\x92\xd5\x8c\xde\xfd\x06\x00\x45\x51\x8a\x9d\xa4\x97\xde\x1f\x5b\x00\x16\xdf\x7e\xd8\x5d\xec\x2e\x98\x3d\xf9\xea\xbb\x2f\x5f\xff\xf4\xfd\x4b\x68\xa8\x15\xcb\xb3\xcc\xfd\x03\xc1\xe4\x2a\x8f\x50\x46\xcb\xb3\x49\xd6\x20\xab\x96\x67\x93\x49\xd6\x22\x31\x28\x1b\xa6\x0d\x52\x1e\x59\xaa\xe3\xcf\xa3\xc3\x42\x43\xd4\xc5\xf8\x9b\xe5\xeb\x3c\xfa\x77\xfc\xe3\x8b\xf8\x4b\xd5\x76\x8c\x78\x21\x30\x82\x52\x49\x42\x49\x79\xf4\xcd\xcb\x1c\xab\x15\x8e\xf6\x49\xd6\x62\x1e\xad\x39\x6e\x3a\xa5\x69\x24\xba\xe1\x15\x35\x79\x85\x6b\x5e\x62\xec\x07\x97\xc0\x25\x27\xce\x44\x6c\x4a\x26\x30\xbf\x8a\x96\x67\x0e\x87\x38\x09\x5c\xbe\x79\x93\x7c\x8b\xb4\x51\xfa\x6e\xb7\x5b\xc0\x2b\x4e\x5f\xdb\x02\xfe\xc6\x6c\x89\x94\xa5\x41\xc4\x4b\x0b\x2e\xef\xa0\xd1\x58\xe7\x91\xe3\x6c\x16\x69\x5a\x56\xf2\xd6\x24\xa5\x50\xb6\xaa\x05\xd3\x98\x94\xaa\x4d\xd9\x2d\xbb\x4f\x05\x2f\x4c\x4a\x1b\x4e\x84\x3a\x2e\x94\x22\x43\x9a\x75\xe9\x75\x72\x9d\x7c\x96\x96\xc6\xa4\xc3\x5c\xd2\x72\x99\x94\xc6\x44\xa0\x51\xe4\x91\xa1\xad\x40\xd3\x20\x52\x04\xe9\xf2\x7f\xd3\x5b\x2b\x49\x31\xdb\xa0\x51\x2d\xa6\xcf\x92\xcf\x92\xb9\x57\x39\x9e\x7e\xb7\x56\xa7\xd6\x94\x9a\x77\x04\x46\x97\x1f\xac\xf7\xf6\x37\x8b\x7a\x9b\x5e\x27\x57\xc9\x55\x3f\xf0\x7a\x6e\x4d\xb4\xcc\xd2\x00\xb8\xfc\x28\xec\x58\x2a\xda\xa6\x4f\x93\x67\xc9\x55\xda\xb1\xf2\x8e\xad\xb0\xda\x6b\x72\x4b\xc9\x7e\xf2\x4f\xd3\xfb\x98\x0f\x6f\x4f\x5d\xf8\x67\x28\x6b\x55\x8b\x92\x92\x5b\x93\x3e\x4d\xae\x3e\x4f\xe6\xfb\x89\xb7\xf1\xbd\x02\xe7\x34\xa7\x6a\x92\xac\x51\x13\x2f\x99\x88\x4b\x94\x84\x1a\xde\xb8\xd9\x49\xcb\x65\xdc\x20\x5f\x35\xb4\x80\xab\xf9\xfc\x93\x9b\x87\x66\xd7\x4d\x98\xae\xb8\xe9\x04\xdb\x2e\xa0\x16\x78\x1f\xa6\x98\xe0\x2b\x19\x73\xc2\xd6\x2c\x20\x20\xfb\x85\x9d\xd7\xd9\x69\xb5\xd2\x68\x4c\xaf\xac\x53\x86\x13\x57\x72\xe1\x22\x8a\x11\x5f\xe3\x43\xb2\xa6\x63\xf2\xad\x0d\xac\x30\x4a\x58\xc2\x13\x22\x85\x50\xe5\x5d\x98\xf3\xd7\x78\x7c\x88\x52\x09\xa5\x17\xb0\x69\x78\xbf\x0d\xbc\x22\xe8\x34\xf6\xf0\xd0\xb1\xaa\xe2\x72\xb5\x80\xe7\x5d\x7f\x1e\x68\x99\x5e\x71\xb9\x80\xf9\x61\x4b\x96\xee\xcd\x98\xa5\x21\x63\x9d\x4d\xb2\x42\x55\x5b\xef\xc3\x8a\xaf\xa1\x14\xcc\x98\x3c\x3a\x31\xb1\xcf\x44\x47\x02\x2e\x01\x31\x2e\xf7\x4b\x47\x6b\x5a\x6d\x22\xf0\x8a\xf2\x28\x90\x88\x0b\x45\xa4\xda\x05\x5c\x39\x7a\xfd\x96\x13\x3c\x11\x8b\x55\x7c\xf5\x74\xbf\x38\xc9\x9a\xab\x3d\x08\xe1\x3d\xc5\xde\x3f\x83\x67\xa2\x65\xc6\xf7\x7b\x6b\x06\x35\x8b\x0b\x46\x4d\x04\x4c\x73\x16\x37\xbc\xaa\x50\xe6\x11\x69\x8b\x2e\x8e\xf8\x12\xc6\x79\x6f\x9f\xf6\x5e\x58\x6a\x50\xba\x73\x12\x56\x7d\x12\x84\x53\xd8\x15\xa7\xc6\x16\x31\x13\xf4\x28\x78\x96\x36\x57\xfb\x23\xa5\x15\x5f\xf7\x16\x19\xfd\x3c\x31\xce\xe3\xe7\xff\x1c\xfa\x1f\xaa\xae\x0d\x52\x3c\x32\xc7\x48\x98\xcb\xce\x52\xbc\xd2\xca\x76\xc3\xfa\x24\xf3\xb3\xc0\xab\x3c\x5a\x71\x43\x11\xd0\xb6\xeb\x6d\x17\x0d\x47\x52\xba\x8d\x9d\xeb\xb4\x12\x11\x74\x82\x95\xd8\x28\x51\xa1\xce\xa3\xde\x26\xaf\xb8\x21\xf8\xf1\x9f\xff\x80\xde\xc1\x5c\xae\x60\xab\xac\x86\x97\xd4\xa0\x46\xdb\x02\xab\x2a\x17\xdc\x49\x92\x8c\x74\xfb\x48\x7f\x9b\x5d\x5c\x90\x3c\x48\x4d\xb2\xc2\x12\xa9\x41\xb0\x20\x09\x05\xc9\xb8\xc2\x9a\x59\x31\x30\x0e\x42\x11\x28\x59\x0a\x5e\xde\xe5\x91\xb1\x45\xcb\x69\x3a\x8b\x96\xaf\xf8\x1a\xa1\xc5\x40\xe6\x49\x96\x06\xd1\x03\x8d\xd4\xf1\x18\x2c\x76\x70\xc0\x07\xfa\xe5\x24\x68\x49\x75\x0b\xb8\x7e\x3a\x8a\xd8\x87\x5c\xf6\xfc\xc4\x65\xd7\x0f\x0a\x77\x4c\xa2\x00\xff\x37\x36\x2d\x13\xfb\xdf\xfb\xb3\x1f\xce\x70\xba\x29\x76\xf7\x73\xa0\x36\xdc\xf3\xf9\x0d\xa8\x35\xea\x5a\xa8\xcd\x02\x98\x25\x75\x03\x2d\xbb\x1f\x72\xdd\xf5\x7c\x3e\xe6\xed\xea\x3f\x2b\x04\xfa\xf0\xd0\xf8\x9b\x45\x43\x66\x08\x8b\xb0\xe4\xff\xba\xe8\xa8\x50\x1a\xac\x4e\xac\xe1\x34\xba\x70\xf7\x52\x23\x8b\x1f\x6c\xfc\x10\xf7\x5a\xa9\x21\x7d\x8c\x69\xf4\xd0\xa3\x4c\x17\x2d\x33\xd2\x07\xb9\x49\x46\xd5\x1f\xba\xfe\xda\x95\xf7\xc7\x6e\x7f\x88\x4f\x77\xf6\x0e\x51\x87\xda\xe2\x22\x05\xfc\x30\x4b\xa9\xfa\x08\xcd\x15\x23\x56\x30\x83\x1f\xa2\xde\x67\xf9\x83\x7a\x3f\xfc\x58\xfd\x0d\x32\x4d\x05\xb2\xc7\x13\xd4\x88\x40\x6d\x65\x35\x3a\xbf\xbf\x48\x1f\x4b\xc0\x4a\xbe\x46\x6d\x38\x6d\x3f\x94\x01\x56\x07\x0a\x61\x7c\x4c\x21\x4b\x49\xbf\x3b\xd6\xfe\x0f\x97\xfb\x7d\xe5\xe8\x7a\xf9\xb5\xda\x40\xa5\xd0\x00\x35\xdc\x80\x2b\x26\x5f\x64\x69\x73\x3d\x88\x74\xcb\xd7\x6e\xc1\x1b\x15\xea\x50\x4f\xb8\x01\x6d\xa5\xcf\xa3\x4a\x02\x35\x78\x5c\x8a\x64\xf8\x95\xc0\x6b\xe5\xca\xf9\x1a\x25\x41\xcb\x04\x2f\xb9\xb2\x06\x58\x49\x4a\x1b\xa8\xb5\x6a\x01\xef\x1b\x66\x0d\x39\x20\x97\x3e\xd8\x9a\x71\xe1\xef\x92\x77\x29\x28\x0d\xac\x2c\x6d\x6b\x5d\x3b\x22\x57\x80\x52\xd9\x55\xd3\x73\x21\x05\xad\xb2\x92\x40\x28\xb9\x1a\xf8\x98\x8e\xb5\xc0\x88\x58\x79\x67\x2e\x61\x9f\x15\x80\x69\x04\xe2\x58\xb9\x5d\x7d\x55\x60\x65\xe9\xb6\x9b\x04\x5e\xc8\xad\x92\x08\x0d\x5b\x7b\x22\x27\x02\xd0\xb2\xed\x1e\xa8\xe7\xb5\xe1\xd4\xf0\x70\xf0\x0e\x75\xeb\xfa\xcb\x0a\x04\x6f\x39\x81\xaa\x21\x33\xa4\x95\x5c\xb9\x67\xc9\x0b\xcf\x70\xb7\x0b\x94\xa7\x66\x06\xa9\x33\xd5\xf7\xa8\xb9\xaa\x76\x3b\xd7\xba\x78\xd1\x24\x4b\xbb\xb1\xc5\xd5\xb1\xc2\x4b\x30\xbc\xed\xc4\x16\x4a\x8d\x8c\x10\x18\x64\xec\xe4\x41\xe1\xca\x63\x12\xea\xba\x6f\x49\x23\x20\xa6\x57\xee\xb9\xf6\x1f\x56\x28\x4b\x8b\x42\x30\x79\xe7\xaa\xcd\x50\x12\xb3\x94\x2d\xfd\x51\x1e\x2e\x86\xd0\x31\xe3\xce\xc5\x25\x29\x7f\xd4\xfe\x7d\x66\x60\xea\x46\x35\x17\xe8\x9f\x70\x3e\x7a\xe4\xb9\xb3\x93\xeb\xb3\x67\x97\x50\xaa\x6e\x1b\x76\xfb\x7d\x8e\x9a\xf1\xf5\x77\x80\x62\x85\x5a\x23\x84\xe2\x5e\xa8\x7b\x60\xb2\x82\x9a\x6b\x04\xb6\x61\xdb\x27\xf0\x93\xb2\x50\x32\x09\xa4\x59\x79\x17\x74\x5b\xad\x5d\x18\x75\x28\x5d\xa9\x38\x38\xb6\x40\xa1\x36\x5e\x24\xa0\xd5\x1c\x85\xf7\xb2\x41\x84\x46\x6d\xa0\xb5\xa5\x3f\xa0\x73\x2f\xba\x85\x0d\xe3\x04\x56\x12\x17\xe1\xdc\x64\xb5\x84\x52\xb5\x68\x46\x5e\x78\xf0\xfa\x0d\xbf\xfa\x1f\x87\x37\x82\x5f\x4e\x53\x78\x25\x54\xc1\x04\xac\x5d\xc6\x28\x84\xbb\x54\x0a\x5c\x33\x72\x74\x06\x43\x8c\xac\x71\x91\xe2\xed\xe8\xaf\x94\xdb\xbf\x66\xda\x45\x2e\xb6\x1d\x41\xde\x77\xb8\x6e\xce\xa0\x5e\xbb\xbe\xbd\xd7\xf1\x15\xd6\x5c\x06\xcb\xd6\x56\x96\xae\x01\x07\x6a\x18\x41\x68\x29\x0c\x30\x6f\x71\xb0\x5a\x40\x6f\xee\x80\x30\xe0\x79\x39\xc8\x87\xed\xd3\x59\xdf\x71\x07\xb9\xc4\xa0\xac\xa6\x7f\xff\xe1\xbb\x6f\x13\x43\x9a\xcb\x15\xaf\xb7\xd3\x37\x56\x8b\x05\xfc\x75\x1a\xfd\xc5\x37\x62\xb3\x9f\xe7\xbf\x26\x6b\x26\x2c\xee\x66\xb3\xf0\x4c\xb8\x39\xe6\xc7\xa0\x45\x6a\x94\xf7\x85\xc6\x52\x49\x89\x25\x81\xed\x94\xec\xe9\x80\x50\xc6\xec\x39\x1d\x24\x1e\xa0\xc5\x6b\x98\xee\x0d\xf3\x09\x3c\x85\x3c\x87\xf9\x7e\xad\xe7\x0c\x39\x48\xdc\xc0\xbf\xb0\xf8\x41\x95\x77\x48\xd3\x68\x63\xdc\xb5\x88\xe0\x02\x84\x2a\x99\xc3\x4b\x1a\x65\x08\x2e\x20\x4a\x59\xc7\xa3\xc0\x7a\xb2\x03\x14\x06\xdf\x0f\xf6\x41\x58\xe1\xcd\x15\x98\x5e\x5c\x04\x8f\xed\x8d\xaa\x64\x8b\xc6\xb0\x15\x8e\x4f\xe8\x73\xe3\x70\x14\x67\x88\xd6\xac\x20\x07\x6f\xfc\x8e\x69\x83\x41\x24\x71\xf5\xb8\xd7\xe2\xcd\xe1\xc5\xf2\x1c\xa4\x15\x62\xd8\x3f\xd1\xe8\x82\xb9\x17\xdb\x9d\x1d\x89\x27\x21\x75\x3d\xc9\x73\x70\xc5\xc9\xf9\xa8\x3a\xec\x74\x8e\x0d\x65\x74\x96\xb8\xfa\x78\xd8\x31\x1b\xe0\xde\x42\xc3\xea\x7d\x70\x58\x9d\xe2\x61\xf5\x08\xa0\xef\x5a\xde\x85\x17\xba\x9c\x11\x9c\x9f\x78\x04\x4d\xda\xb6\x40\xfd\x2e\xb8\xd0\xb5\xf4\x70\xde\xd4\xdf\x48\x1a\xed\xbd\x84\xab\xe7\xb3\x47\xd0\x51\x6b\xf5\x28\xb8\x54\xb4\x9d\xbe\x11\x6c\xeb\xb2\x2e\x9c\x93\xea\xbe\xf4\x4d\xc6\xf9\x25\x38\x5d\x0b\x18\x10\x2e\xfd\xe3\x60\x01\xe7\x7e\x74\xbe\x7b\x44\x9b\xb1\x65\xe9\xf2\xf1\xc7\xe8\xeb\x31\x06\x8d\xfd\xf8\x51\x9d\x43\x7e\x3d\x52\x0a\x9f\x7e\x0a\x6f\xad\x1e\x87\xa0\x8b\xe1\xbe\x50\x40\x0e\x51\xd4\xc3\x4f\x6a\xa5\x61\xea\x16\x79\x3e\xbf\x01\x9e\x8d\x61\x12\x81\x72\x45\xcd\x0d\xf0\x8b\x8b\x03\xd2\x64\x0f\x73\x91\x43\xe4\xfa\xe8\x8c\xaa\xa5\xef\x67\x42\xd3\xf3\x4b\x54\xb0\xf2\xce\x3d\xc9\x64\xb5\x70\xd9\x6e\x7a\x7e\x28\x86\xa3\x3a\x78\x71\x44\xf9\x67\xfe\x6b\x62\x0d\x6a\x5f\xb9\x2e\x20\x4a\x3a\xb9\xfa\xc2\xf0\xdf\x31\x7f\xfe\xec\x7c\x76\x03\x07\xcc\xd8\xcd\x2e\xa0\x74\x2f\x92\x1b\x08\x5d\xbd\xef\xad\x60\x78\x8f\xf8\x51\xa1\x74\x85\x3a\xd6\xac\xe2\xd6\x2c\xe0\x59\x77\x7f\xf3\x8b\xeb\x04\x5d\x89\xf0\x1d\xa0\xe7\xdd\x69\x5c\x3e\xc4\x65\xdf\x64\x5c\x40\x94\xa5\x4e\x68\xbf\x65\x38\xe5\xf8\xcb\x09\x3c\xd0\xbb\xc2\xf0\x5d\xa3\x9f\x6f\x79\x55\x09\x74\x24\xbc\xc2\xf0\x01\xaa\xb2\xda\x27\xae\x69\x18\x4f\x4f\x79\x10\x6f\x71\x96\x58\xc9\xef\xa7\xb3\xb8\x97\xd9\x8f\x2f\xe1\xdc\xb8\xfc\x5c\x99\xf3\x59\xd2\xd8\x96\x49\xfe\x3b\x4e\x5d\x23\x3c\x0b\xbc\x1d\x63\xd7\xdd\x0e\xde\xde\x8d\x2e\xda\xf0\x32\x9b\x25\x0d\xb5\x62\x1a\x65\xe4\xbf\xce\x38\x72\x83\x8b\x3d\x4a\x98\x3e\x8e\xc8\xdd\x71\x0e\x2d\x85\x32\x78\x52\x23\xc0\x20\xbd\xe6\x2d\x2a\x4b\xd3\xa1\x8e\x5c\xba\xd7\xe2\x7c\x76\x03\xa1\x2e\x1d\x10\xc2\xdd\xfd\xe3\x08\xbb\xbe\xbc\xbd\x34\xae\x83\xe7\xa6\x01\x06\x1b\x2c\x8c\xaf\x10\xd0\xef\xf1\xb5\x38\xd4\xdc\x17\xdf\x7f\x33\xaa\xbb\x03\xea\xd4\x1f\x6f\xf4\x99\x31\x4b\xc3\xb7\xaa\x2c\x0d\xdf\xe1\xff\x1b\x00\x00\xff\xff\x97\x3f\x1d\xc4\x98\x17\x00\x00") +var _faucetHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x59\x6d\x6f\xdc\x36\x12\xfe\xec\xfc\x8a\xa9\x2e\xad\x77\x61\x4b\xb2\xe3\x20\x2d\xd6\xd2\x16\x41\x9a\x4b\x7b\x38\xb4\x45\x9b\xe2\xae\x68\x8b\x03\x25\xcd\x4a\x8c\x29\x52\x25\x87\xbb\xde\x1a\xfb\xdf\x0f\x24\x25\xad\x76\x6d\xa7\xb9\x4b\xf3\x61\x23\x92\x33\xcf\xbc\x51\xf3\x22\x67\x9f\x7c\xf5\xdd\xab\xb7\x3f\x7f\xff\x1a\x1a\x6a\xc5\xf2\x49\xe6\xfe\x03\xc1\x64\x9d\x47\x28\xa3\xe5\x93\x93\xac\x41\x56\x2d\x9f\x9c\x9c\x64\x2d\x12\x83\xb2\x61\xda\x20\xe5\x91\xa5\x55\xfc\x45\xb4\x3f\x68\x88\xba\x18\x7f\xb7\x7c\x9d\x47\xff\x8e\x7f\x7a\x19\xbf\x52\x6d\xc7\x88\x17\x02\x23\x28\x95\x24\x94\x94\x47\xdf\xbc\xce\xb1\xaa\x71\xc2\x27\x59\x8b\x79\xb4\xe6\xb8\xe9\x94\xa6\x09\xe9\x86\x57\xd4\xe4\x15\xae\x79\x89\xb1\x5f\x9c\x03\x97\x9c\x38\x13\xb1\x29\x99\xc0\xfc\x32\x5a\x3e\x71\x38\xc4\x49\xe0\xf2\xee\x2e\xf9\x16\x69\xa3\xf4\xcd\x6e\xb7\x80\x37\x9c\xbe\xb6\x05\xfc\x9d\xd9\x12\x29\x4b\x03\x89\xa7\x16\x5c\xde\x40\xa3\x71\x95\x47\x4e\x67\xb3\x48\xd3\xb2\x92\xef\x4c\x52\x0a\x65\xab\x95\x60\x1a\x93\x52\xb5\x29\x7b\xc7\x6e\x53\xc1\x0b\x93\xd2\x86\x13\xa1\x8e\x0b\xa5\xc8\x90\x66\x5d\x7a\x95\x5c\x25\x9f\xa7\xa5\x31\xe9\xb8\x97\xb4\x5c\x26\xa5\x31\x11\x68\x14\x79\x64\x68\x2b\xd0\x34\x88\x14\x41\xba\xfc\xff\xe4\xae\x94\xa4\x98\x6d\xd0\xa8\x16\xd3\xe7\xc9\xe7\xc9\x85\x17\x39\xdd\x7e\xbf\x54\x27\xd6\x94\x9a\x77\x04\x46\x97\x1f\x2c\xf7\xdd\xef\x16\xf5\x36\xbd\x4a\x2e\x93\xcb\x7e\xe1\xe5\xbc\x33\xd1\x32\x4b\x03\xe0\xf2\xa3\xb0\x63\xa9\x68\x9b\x3e\x4b\x9e\x27\x97\x69\xc7\xca\x1b\x56\x63\x35\x48\x72\x47\xc9\xb0\xf9\x97\xc9\x7d\x2c\x86\xef\x8e\x43\xf8\x57\x08\x6b\x55\x8b\x92\x92\x77\x26\x7d\x96\x5c\x7e\x91\x5c\x0c\x1b\xf7\xf1\xbd\x00\x17\x34\x27\xea\x24\x59\xa3\x26\x5e\x32\x11\x97\x28\x09\x35\xdc\xb9\xdd\x93\x96\xcb\xb8\x41\x5e\x37\xb4\x80\xcb\x8b\x8b\x4f\xaf\x1f\xda\x5d\x37\x61\xbb\xe2\xa6\x13\x6c\xbb\x80\x95\xc0\xdb\xb0\xc5\x04\xaf\x65\xcc\x09\x5b\xb3\x80\x80\xec\x0f\x76\x5e\x66\xa7\x55\xad\xd1\x98\x5e\x58\xa7\x0c\x27\xae\xe4\xc2\xdd\x28\x46\x7c\x8d\x0f\xd1\x9a\x8e\xc9\x7b\x0c\xac\x30\x4a\x58\xc2\x23\x45\x0a\xa1\xca\x9b\xb0\xe7\x5f\xe3\xa9\x11\xa5\x12\x4a\x2f\x60\xd3\xf0\x9e\x0d\xbc\x20\xe8\x34\xf6\xf0\xd0\xb1\xaa\xe2\xb2\x5e\xc0\x8b\xae\xb7\x07\x5a\xa6\x6b\x2e\x17\x70\xb1\x67\xc9\xd2\xc1\x8d\x59\x1a\x32\xd6\x93\x93\xac\x50\xd5\xd6\xc7\xb0\xe2\x6b\x28\x05\x33\x26\x8f\x8e\x5c\xec\x33\xd1\x01\x81\x4b\x40\x8c\xcb\xe1\xe8\xe0\x4c\xab\x4d\x04\x5e\x50\x1e\x05\x25\xe2\x42\x11\xa9\x76\x01\x97\x4e\xbd\x9e\xe5\x08\x4f\xc4\xa2\x8e\x2f\x9f\x0d\x87\x27\x59\x73\x39\x80\x10\xde\x52\xec\xe3\x33\x46\x26\x5a\x66\x7c\xe0\x5d\x31\x58\xb1\xb8\x60\xd4\x44\xc0\x34\x67\x71\xc3\xab\x0a\x65\x1e\x91\xb6\xe8\xee\x11\x5f\xc2\x34\xef\x0d\x69\xef\xa5\xa5\x06\xa5\xb3\x93\xb0\xea\x93\x20\x1c\xc3\xd6\x9c\x1a\x5b\xc4\x4c\xd0\xa3\xe0\x59\xda\x5c\x0e\x26\xa5\x15\x5f\xf7\x1e\x99\x3c\x1e\x39\xe7\x71\xfb\xbf\x80\xfe\x41\xad\x56\x06\x29\x9e\xb8\x63\x42\xcc\x65\x67\x29\xae\xb5\xb2\xdd\x78\x7e\x92\xf9\x5d\xe0\x55\x1e\xd5\xdc\x50\x04\xb4\xed\x7a\xdf\x45\xa3\x49\x4a\xb7\xb1\x0b\x9d\x56\x22\x82\x4e\xb0\x12\x1b\x25\x2a\xd4\x79\xd4\xfb\xe4\x0d\x37\x04\x3f\xfd\xf0\x4f\xe8\x03\xcc\x65\x0d\x5b\x65\x35\xbc\xa6\x06\x35\xda\x16\x58\x55\xb9\xcb\x9d\x24\xc9\x44\xb6\xbf\xe9\xf7\xb5\x8b\x0b\x92\x7b\xaa\x93\xac\xb0\x44\x6a\x24\x2c\x48\x42\x41\x32\xae\x70\xc5\xac\x20\xa8\xb4\xea\x2a\xb5\x91\x31\xa9\xba\x76\x05\x31\x58\x10\x98\x22\xa8\x18\xb1\xfe\x28\x8f\x06\xda\x21\x28\xcc\x74\xaa\xb3\x5d\x1f\x96\xb0\x89\xb7\x1d\x93\x15\x56\x2e\x94\xc2\x60\xb4\x7c\xc3\xd7\x08\x2d\x06\x5b\x4e\x8e\x23\x5d\x32\x8d\x14\x4f\x41\x1f\x88\x74\x50\x26\x98\x04\xfd\xbf\xcc\x8a\x01\x69\x34\xa1\x45\x69\xe1\x60\x15\x6b\x97\x85\xa2\xe5\xdd\x9d\x66\xb2\x46\x78\xca\xab\xdb\x73\x78\xca\x5a\x65\x25\xc1\x22\x87\xe4\xa5\x7f\x34\xbb\xdd\x01\x3a\x40\x26\xf8\x32\x63\xef\x7b\x19\x40\xc9\x52\xf0\xf2\x26\x8f\x88\xa3\xce\xef\xee\x1c\xf8\x6e\x77\x0d\x77\x77\x7c\x05\x4f\x93\x1f\xb0\x64\x1d\x95\x0d\xdb\xed\x6a\x3d\x3c\x27\x78\x8b\xa5\x25\x9c\xcd\xef\xee\x50\x18\xdc\xed\x8c\x2d\x5a\x4e\xb3\x81\xdd\xed\xcb\x6a\xb7\x73\x3a\xf7\x7a\xee\x76\x90\x3a\x50\x59\xe1\x2d\x3c\x4d\xbe\x47\xcd\x55\x65\x20\xd0\x67\x29\x5b\x66\xa9\xe0\xcb\x9e\xef\xd0\x49\xa9\x15\xfb\xfb\x92\xba\x0b\x33\x5e\x6d\xff\xa6\x78\x55\xa7\x9a\x3e\x70\xf1\xeb\x78\xd4\xbe\xbf\x0f\x86\x13\xde\xe0\x36\x8f\xee\xee\xa6\xbc\xfd\x69\xc9\x84\x28\x98\xf3\x4b\x30\x6d\x64\xfa\x03\xdd\x3d\x5d\x73\xe3\x3b\xaf\xe5\xa0\xc1\x5e\xed\x0f\x7c\x93\x8f\xd2\x1c\xa9\x6e\x01\x57\xcf\x26\x39\xee\xa1\x97\xfc\xc5\xd1\x4b\x7e\xf5\x20\x71\xc7\x24\x0a\xf0\xbf\xb1\x69\x99\x18\x9e\xfb\xb7\x65\xf2\xf2\x1d\x33\xc5\x2e\xa3\x8f\xaa\x8d\x95\xe1\xe2\x1a\xd4\x1a\xf5\x4a\xa8\xcd\x02\x98\x25\x75\x0d\x2d\xbb\x1d\xab\xe3\xd5\xc5\xc5\x54\x6f\xd7\x31\xb2\x42\xa0\x4f\x28\x1a\x7f\xb7\x68\xc8\x8c\x89\x24\x1c\xf9\x5f\x97\x4f\x2a\x94\x06\xab\x23\x6f\x38\x89\xce\xb5\x9e\x6a\x12\xfa\xd1\x99\x0f\xea\xbe\x52\x6a\x2c\x38\x53\x35\x7a\xe8\x49\x6d\x8c\x96\x19\xe9\x3d\xdd\x49\x46\xd5\xff\x54\x30\xb4\x6b\x08\x1f\xab\x17\x21\xa3\x39\xdb\x3b\x44\x1d\xba\x11\x77\x65\xc1\x2f\xb3\x94\xaa\x8f\x90\xec\x2e\x61\xc1\x0c\x7e\x88\x78\xdf\x17\xec\xc5\xfb\xe5\xc7\xca\x6f\x90\x69\x2a\x90\x3d\x5e\xd2\x26\x0a\xac\xac\xac\x26\xf6\xfb\xdc\xf9\xb1\x0a\x58\xc9\xd7\xa8\x0d\xa7\xed\x87\x6a\x80\xd5\x5e\x85\xb0\x3e\x54\x21\x4b\x49\xbf\xff\xae\x4d\x17\x7f\xd1\xcb\xfd\x67\x0d\xcc\xd5\xf2\x6b\xb5\x81\x4a\xa1\x01\x6a\xb8\x01\xd7\x7e\x7c\x99\xa5\xcd\xd5\x48\xd2\x2d\xdf\xba\x03\xef\x54\x58\x85\x0e\x84\x1b\xd0\x56\xfa\xca\xab\x24\x50\x83\x87\xcd\x8b\x0c\x4f\x09\xbc\x55\xae\x01\x5c\xa3\x24\x68\x99\xe0\x25\x57\xd6\x00\x2b\x49\x69\x03\x2b\xad\x5a\xc0\xdb\x86\x59\x43\x0e\xc8\xa5\x0f\xb6\x66\x5c\xf8\x77\xc9\x87\x14\x94\x06\x56\x96\xb6\xb5\xae\x81\x95\x35\xa0\x54\xb6\x6e\x7a\x5d\x48\x41\x28\x4c\x42\xc9\x7a\xd4\xc7\x74\xac\x05\x46\xc4\xca\x1b\x73\x0e\x43\x56\x00\xa6\x11\x88\x63\xe5\xb8\xfa\x3e\x82\x95\xa5\x2f\x66\x09\xbc\x94\x5b\x25\x11\x1a\xb6\xf6\x8a\x1c\x11\x40\xcb\xb6\x03\x50\xaf\xd7\x86\x53\xc3\x83\xe1\x1d\xea\xd6\x4d\x24\x15\x08\xde\x72\x32\x49\x96\x76\x53\xdf\xa9\x43\xd6\x73\x30\xbc\xed\xc4\x16\x4a\x8d\x8c\x10\x18\x64\xec\x68\x98\x74\xad\x51\x12\x7a\x3a\x3f\x8e\x44\x40\x4c\xd7\x6e\x54\xff\x0f\x2b\x94\xa5\x45\x21\x98\xbc\x71\xad\xc2\xd8\x0e\xb9\xb2\xe6\x95\x7a\xb8\x11\x82\x8e\x19\xa7\x21\x97\xa4\xbc\xd2\xfd\x6c\x6e\x60\xe6\x56\x2b\x2e\xd0\x8f\xef\xfe\x1e\xc8\x53\x67\xb1\x9b\xb1\xe6\xe7\x50\xaa\x6e\x1b\xb8\x3d\x9f\x53\xcd\xf8\xde\x6b\x84\x62\x85\x5a\x23\x84\xc6\xae\x50\xb7\xc0\x64\x05\x2b\xae\x11\xd8\x86\x6d\x3f\x81\x9f\x95\x85\x92\x49\x20\xcd\xca\x9b\x20\xdb\x6a\xed\x2e\x44\x87\xd2\x25\xfd\x7d\x88\x0a\x14\x6a\xe3\x49\x02\xda\x8a\xa3\xf0\xf1\x32\x88\xd0\xa8\x0d\xb4\xb6\xf4\x06\xba\x40\xa1\x3b\xd8\x30\x4e\x60\x25\x71\x11\xec\x26\xab\x25\x94\xaa\xc5\x83\x28\xdc\xab\xda\x19\xb6\xcb\xb7\xce\xee\x7b\x97\x79\xac\xb7\xa0\xf1\x55\x20\x87\x4e\x2b\xc2\xd2\x0d\x46\xc0\x6a\xc6\xa5\x71\x76\xfa\x38\x63\xfb\x01\xf5\x78\x7c\xea\x1f\xf6\x93\xa8\x3f\x4e\x53\x78\x23\x54\xc1\x04\xac\x5d\x96\x29\x84\x7b\x11\x15\xb8\x96\xf7\xc0\x5b\x86\x18\x59\x03\x6a\xe5\x77\x83\xe6\x8e\x7f\xcd\xb4\xbb\xed\xd8\x76\x04\x79\x3f\x47\xb9\x3d\x83\x7a\xdd\x4f\x87\x6e\xe9\x7a\xae\x70\xde\x0b\xfd\x0a\x57\x5c\x86\xa0\xae\xac\x0c\xe6\x51\xc3\x08\x42\x17\x62\x80\xf9\x60\x83\xd5\x02\xfa\x48\x07\xc8\x51\x80\xa7\x83\x7c\x64\x9f\xdd\xf3\x73\xff\xd0\xfb\x68\xde\xcf\x81\x01\x26\x31\x28\xab\xd9\x3f\x7e\xfc\xee\xdb\xc4\x90\xe6\xb2\xe6\xab\xed\xec\xce\x6a\xb1\x80\xa7\xb3\xe8\x6f\x7e\x3c\x98\xff\x72\xf1\x5b\xb2\x66\xc2\xe2\xb9\x37\x60\xe1\x7f\xef\x89\x39\x87\xfe\x71\x01\x87\x12\x77\xf3\xf9\xf5\xc3\x2d\xdb\xa4\xc3\xd4\x68\x90\x66\x8e\x70\x8c\xe4\xee\xfa\xd0\x49\x0c\x5a\xa4\x46\xf9\xbb\xa8\xb1\x54\x52\x62\x49\x60\x3b\x25\x7b\x9f\x80\x50\xc6\x0c\x8e\xd9\x53\x4c\x7c\x33\x18\xcf\x57\x30\x1b\xc2\xf5\x29\x3c\x83\x3c\x87\x8b\xe1\xac\xf7\x0c\xe4\x20\x71\x03\xff\xc2\xe2\x47\x55\xde\x20\xcd\xa2\x8d\x71\x69\x21\x82\x33\x10\xaa\x64\x0e\x2f\x69\x94\x21\x38\x83\x28\x65\x1d\x8f\xe6\x61\x9a\xde\x81\x6b\x91\xff\x1c\xec\x83\xb0\xc2\xf7\x86\xa0\xe9\xd9\x59\xb8\x36\x43\xe8\x94\x6c\xd1\x18\x56\xe3\xd4\x42\x9f\xe5\x47\x53\x9c\x23\x5a\x53\x43\x0e\x3e\xc4\x1d\xd3\x06\x03\x49\xe2\x3a\x8b\x5e\x8a\x77\x87\x27\xcb\x73\x90\x56\x88\x91\xff\x44\xa3\x7b\x99\x7b\xb2\xdd\x93\x03\xf2\x24\x24\xe1\x4f\xf2\x1c\x5c\x99\x75\x31\xaa\xf6\x9c\xee\xfa\x84\x86\x60\x9e\xb8\x4a\xbf\xe7\x98\x8f\x70\xf7\xd0\xb0\xfa\x33\x38\xac\x8e\xf1\xb0\x7a\x04\xd0\xf7\x5f\xef\xc3\x0b\xfd\xda\x04\xce\x6f\x3c\x82\x26\x6d\x5b\xa0\x7e\x1f\x5c\xe8\xbf\x7a\x38\xef\xea\x6f\x24\x4d\x78\xcf\xe1\xf2\xc5\xfc\x11\x74\xd4\x5a\x3d\x0a\x2e\x15\x6d\x67\x77\x82\x6d\x5d\xd5\x81\x53\x52\xdd\x2b\xdf\x2e\x9d\x9e\x83\x93\xb5\x80\x11\xe1\xdc\x0f\xc2\x0b\x38\xf5\xab\xd3\xdd\x23\xd2\x8c\x2d\x4b\x57\x8f\x3e\x46\x5e\x8f\x31\x4a\xec\xd7\x8f\xca\x1c\xeb\xcb\x81\x50\xf8\xec\x33\xb8\x77\x7a\x78\x05\xdd\x1d\xee\x0b\x25\xe4\x10\x45\x3d\xfc\xc9\x4a\x69\x98\xb9\x43\x9e\x5f\x5c\x03\xcf\xa6\x30\x89\x40\x59\x53\x73\x0d\xfc\xec\x6c\x8f\x74\x32\xc0\x9c\xe5\x10\xb9\x89\x20\xa3\x6a\xe9\x3b\xb3\xd0\xbe\xfd\x1a\xb9\x09\xb0\xd6\xca\xca\x6a\xe1\x52\xee\xec\x74\xdf\x0c\x4c\xfa\x80\xb3\x03\x95\x7f\xe1\xbf\x25\xd6\xa0\xf6\x95\xfb\x0c\xa2\xa4\x93\xf5\x97\x7e\x6e\x7c\xf1\xfc\x74\x7e\x0d\x7b\x4c\x3f\x4d\x2e\xa0\x74\xb3\xd5\x35\x84\xf9\xc4\x77\x89\x30\x4e\x56\x7e\x55\x28\x5d\xa1\x8e\x35\xab\xb8\x35\x0b\x78\xde\xdd\x5e\xff\x3a\x4c\x9e\xbe\x97\xf5\x7a\x77\x1a\x97\x0f\xe9\x32\xb4\x4b\x67\x10\x65\xa9\x23\x1a\x58\x46\x2b\xa7\x5f\x0d\xe1\x81\x2e\x1c\xc6\x6f\x7a\xfd\x7e\xcb\xab\x4a\xa0\x53\xc2\x0b\x0c\x1f\x5f\x2b\xab\x7d\xe2\x9a\x85\xf5\xec\x58\x0f\xe2\x2d\xce\x13\x2b\xf9\xed\x6c\x1e\xf7\x34\xc3\xfa\x1c\x4e\x8d\xcb\xcf\x95\x39\x9d\x27\x8d\x6d\x99\xe4\x7f\xe0\xcc\xb5\xf4\xf3\xa0\xb7\xd3\xd8\xf5\xe9\x63\xb4\x77\x93\x17\x6d\x9c\x31\xe7\x49\x43\xad\x98\x45\x19\xf9\x2f\x93\x4e\xb9\x31\xc4\x1e\x25\x6c\x1f\xde\xc8\xdd\x61\x0e\x2d\x85\x32\x78\x54\x23\xc0\x20\xbd\xe5\x2d\x2a\x4b\xb3\xb1\x8e\x9c\xbb\xb9\xf7\x62\x7e\x0d\xbb\xfd\x07\xdc\x34\x85\xd7\xc6\x4d\x12\xdc\x34\xc0\x60\x83\x85\xf1\xf9\x1d\x7a\x1e\x5f\xce\x43\xd9\x7e\xf9\xfd\x37\x93\xd2\x3d\xa2\xce\xbc\x72\xe3\x07\xec\x87\xea\xe4\x83\x5f\xcc\x37\x9b\x4d\x52\x2b\x55\x8b\xf0\xad\x7c\x2c\xa4\xae\x7a\x24\xef\xdc\xb8\x6a\xb6\xb2\x84\x0a\x57\xa8\x97\x13\xf8\xbe\xba\x66\x69\xf8\x96\x9b\xa5\xe1\xef\x54\xff\x0d\x00\x00\xff\xff\x71\x50\x77\xf3\xb8\x1a\x00\x00") func faucetHtmlBytes() ([]byte, error) { return bindataRead( diff --git a/cmd/gexp/accountcmd.go b/cmd/gexp/accountcmd.go index 349b40d2c3..411eacbfb1 100644 --- a/cmd/gexp/accountcmd.go +++ b/cmd/gexp/accountcmd.go @@ -40,32 +40,39 @@ var ( will prompt for your password and imports your ether presale account. It can be used non-interactively with the --password option taking a -passwordfile as argument containing the wallet password in plaintext. - -`, +passwordfile as argument containing the wallet password in plaintext.`, Subcommands: []cli.Command{ { - Action: importWallet, + Name: "import", Usage: "Import Expanse presale wallet", ArgsUsage: "", + Action: utils.MigrateFlags(importWallet), + Category: "ACCOUNT COMMANDS", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.KeyStoreDirFlag, + utils.PasswordFileFlag, + utils.LightKDFFlag, + }, Description: ` -TODO: Please write this -`, + geth wallet [options] /path/to/my/presale.wallet + +will prompt for your password and imports your ether presale account. +It can be used non-interactively with the --password option taking a +passwordfile as argument containing the wallet password in plaintext.`, }, }, } - accountCommand = cli.Command{ - Action: accountList, - Name: "account", - Usage: "Manage accounts", - ArgsUsage: "", - Category: "ACCOUNT COMMANDS", - Description: ` -Manage accounts lets you create new accounts, list all existing accounts, -import a private key into a new account. -' help' shows a list of subcommands or help for one subcommand. + accountCommand = cli.Command{ + Name: "account", + Usage: "Manage accounts", + Category: "ACCOUNT COMMANDS", + Description: ` + +Manage accounts, list all existing accounts, import a private key into a new +account, create a new account or update an existing account. It supports interactive mode, when you are prompted for password as well as non-interactive mode where passwords are supplied via a given password file. @@ -80,36 +87,34 @@ Note that exporting your key in unencrypted format is NOT supported. Keys are stored under /keystore. It is safe to transfer the entire directory or the individual keys therein between expanse nodes by simply copying. -Make sure you backup your keys regularly. -In order to use your account to send transactions, you need to unlock them using -the '--unlock' option. The argument is a space separated list of addresses or -indexes. If used non-interactively with a passwordfile, the file should contain -the respective passwords one per line. If you unlock n accounts and the password -file contains less than n entries, then the last password is meant to apply to -all remaining accounts. - -And finally. DO NOT FORGET YOUR PASSWORD. -`, +Make sure you backup your keys regularly.`, Subcommands: []cli.Command{ { - Action: accountList, - Name: "list", - Usage: "Print account addresses", - ArgsUsage: " ", + Name: "list", + Usage: "Print summary of existing accounts", + Action: utils.MigrateFlags(accountList), + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.KeyStoreDirFlag, + }, Description: ` -TODO: Please write this -`, +Print a short summary of all accounts`, }, { - Action: accountCreate, - Name: "new", - Usage: "Create a new account", - ArgsUsage: " ", + Name: "new", + Usage: "Create a new account", + Action: utils.MigrateFlags(accountCreate), + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.KeyStoreDirFlag, + utils.PasswordFileFlag, + utils.LightKDFFlag, + }, Description: ` gexp account new -Creates a new account. Prints the address. +Creates a new account and prints the address. The account is saved in encrypted format, you are prompted for a passphrase. @@ -117,17 +122,20 @@ You must remember this passphrase to unlock your account in the future. For non-interactive use the passphrase can be specified with the --password flag: - gexp --password account new - Note, this is meant to be used for testing only, it is a bad idea to save your password to file or expose in any other way. `, }, { - Action: accountUpdate, Name: "update", Usage: "Update an existing account", + Action: utils.MigrateFlags(accountUpdate), ArgsUsage: "
", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.KeyStoreDirFlag, + utils.LightKDFFlag, + }, Description: ` gexp account update
@@ -141,16 +149,22 @@ format to the newest format or change the password for an account. For non-interactive use the passphrase can be specified with the --password flag: - gexp --password account update
+ gexp account update [options]
Since only one password can be given, only format update can be performed, changing your password is only possible interactively. `, }, { - Action: accountImport, - Name: "import", - Usage: "Import a private key into a new account", + Name: "import", + Usage: "Import a private key into a new account", + Action: utils.MigrateFlags(accountImport), + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.KeyStoreDirFlag, + utils.PasswordFileFlag, + utils.LightKDFFlag, + }, ArgsUsage: "", Description: ` gexp account import @@ -166,7 +180,7 @@ You must remember this passphrase to unlock your account in the future. For non-interactive use the passphrase can be specified with the -password flag: - gexp --password account import + gexp account import [options] Note: As you can directly copy your encrypted accounts to another expanse instance, @@ -298,10 +312,12 @@ func accountUpdate(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) - account, oldPassword := unlockAccount(ctx, ks, ctx.Args().First(), 0, nil) - newPassword := getPassPhrase("Please give a new password. Do not forget this password.", true, 0, nil) - if err := ks.Update(account, oldPassword, newPassword); err != nil { - utils.Fatalf("Could not update the account: %v", err) + for _, addr := range ctx.Args() { + account, oldPassword := unlockAccount(ctx, ks, addr, 0, nil) + newPassword := getPassPhrase("Please give a new password. Do not forget this password.", true, 0, nil) + if err := ks.Update(account, oldPassword, newPassword); err != nil { + utils.Fatalf("Could not update the account: %v", err) + } } return nil } diff --git a/cmd/gexp/accountcmd_test.go b/cmd/gexp/accountcmd_test.go index 56178634f5..dba98aaddf 100644 --- a/cmd/gexp/accountcmd_test.go +++ b/cmd/gexp/accountcmd_test.go @@ -43,13 +43,13 @@ func tmpDatadirWithKeystore(t *testing.T) string { } func TestAccountListEmpty(t *testing.T) { - gexp := runGeth(t, "account") + gexp := runGeth(t, "account", "list") gexp.expectExit() } func TestAccountList(t *testing.T) { datadir := tmpDatadirWithKeystore(t) - gexp := runGeth(t, "--datadir", datadir, "account") + gexp := runGeth(t, "account", "list", "--datadir", datadir) defer gexp.expectExit() if runtime.GOOS == "windows" { gexp.expect(` @@ -67,7 +67,7 @@ Account #2: {289d485d9771714cce91d3393d764e1311907acc} keystore://{{.Datadir}}/k } func TestAccountNew(t *testing.T) { - gexp := runGeth(t, "--lightkdf", "account", "new") + gexp := runGeth(t, "account", "new", "--lightkdf") defer gexp.expectExit() gexp.expect(` Your new account is locked with a password. Please give a password. Do not forget this password. @@ -79,7 +79,7 @@ Repeat passphrase: {{.InputLine "foobar"}} } func TestAccountNewBadRepeat(t *testing.T) { - gexp := runGeth(t, "--lightkdf", "account", "new") + gexp := runGeth(t, "account", "new", "--lightkdf") defer gexp.expectExit() gexp.expect(` Your new account is locked with a password. Please give a password. Do not forget this password. @@ -92,9 +92,9 @@ Fatal: Passphrases do not match func TestAccountUpdate(t *testing.T) { datadir := tmpDatadirWithKeystore(t) - gexp := runGeth(t, + gexp := runGeth(t, "account", "update", "--datadir", datadir, "--lightkdf", - "account", "update", "f466859ead1932d743d622cb74fc058882e8648a") + "f466859ead1932d743d622cb74fc058882e8648a") defer gexp.expectExit() gexp.expect(` Unlocking account f466859ead1932d743d622cb74fc058882e8648a | Attempt 1/3 @@ -107,7 +107,7 @@ Repeat passphrase: {{.InputLine "foobar2"}} } func TestWalletImport(t *testing.T) { - gexp := runGeth(t, "--lightkdf", "wallet", "import", "testdata/guswallet.json") + gexp := runGeth(t, "wallet", "import", "--lightkdf", "testdata/guswallet.json") defer gexp.expectExit() gexp.expect(` !! Unsupported terminal, password will be echoed. @@ -122,7 +122,8 @@ Address: {d4584b5f6229b7be90727b0fc8c6b91bb427821f} } func TestWalletImportBadPassword(t *testing.T) { - gexp := runGeth(t, "--lightkdf", "wallet", "import", "testdata/guswallet.json") + + gexp := runGeth(t, "wallet", "import", "--lightkdf", "testdata/guswallet.json") defer gexp.expectExit() gexp.expect(` !! Unsupported terminal, password will be echoed. diff --git a/cmd/gexp/bugcmd.go b/cmd/gexp/bugcmd.go index b6234413ba..b0f4253581 100644 --- a/cmd/gexp/bugcmd.go +++ b/cmd/gexp/bugcmd.go @@ -29,11 +29,12 @@ import ( "github.com/expanse-org/go-expanse/cmd/internal/browser" "github.com/expanse-org/go-expanse/params" + "github.com/expanse-org/go-expanse/cmd/utils" cli "gopkg.in/urfave/cli.v1" ) var bugCommand = cli.Command{ - Action: reportBug, + Action: utils.MigrateFlags(reportBug), Name: "bug", Usage: "opens a window to report a bug on the gexp repo", ArgsUsage: " ", diff --git a/cmd/gexp/chaincmd.go b/cmd/gexp/chaincmd.go index 7865930fd4..92d30b78bb 100644 --- a/cmd/gexp/chaincmd.go +++ b/cmd/gexp/chaincmd.go @@ -40,80 +40,98 @@ import ( var ( initCommand = cli.Command{ - Action: initGenesis, + Action: utils.MigrateFlags(initGenesis), Name: "init", Usage: "Bootstrap and initialize a new genesis block", ArgsUsage: "", - Category: "BLOCKCHAIN COMMANDS", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.LightModeFlag, + }, + Category: "BLOCKCHAIN COMMANDS", Description: ` The init command initializes a new genesis block and definition for the network. This is a destructive action and changes the network in which you will be participating. -`, + +It expects the genesis file as argument.`, } importCommand = cli.Command{ - Action: importChain, + Action: utils.MigrateFlags(importChain), Name: "import", Usage: "Import a blockchain file", ArgsUsage: " ( ... ) ", - Category: "BLOCKCHAIN COMMANDS", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.CacheFlag, + utils.LightModeFlag, + }, + Category: "BLOCKCHAIN COMMANDS", Description: ` -The import command imports blocks from an RLP-encoded form. The form can be one file -with several RLP-encoded blocks, or several files can be used. -If only one file is used, import error will result in failure. If several files are used, -processing will proceed even if an individual RLP-file import failure occurs. -`, +The import command imports blocks from an RLP-encoded form. The form can be one file +with several RLP-encoded blocks, or several files can be used. + +If only one file is used, import error will result in failure. If several files are used, +processing will proceed even if an individual RLP-file import failure occurs.`, } exportCommand = cli.Command{ - Action: exportChain, + Action: utils.MigrateFlags(exportChain), Name: "export", Usage: "Export blockchain into file", ArgsUsage: " [ ]", - Category: "BLOCKCHAIN COMMANDS", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.CacheFlag, + utils.LightModeFlag, + }, + Category: "BLOCKCHAIN COMMANDS", Description: ` Requires a first argument of the file to write to. Optional second and third arguments control the first and last block to write. In this mode, the file will be appended -if already existing. -`, +if already existing.`, } removedbCommand = cli.Command{ - Action: removeDB, + Action: utils.MigrateFlags(removeDB), Name: "removedb", Usage: "Remove blockchain and state databases", ArgsUsage: " ", - Category: "BLOCKCHAIN COMMANDS", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.LightModeFlag, + }, + Category: "BLOCKCHAIN COMMANDS", Description: ` -TODO: Please write this -`, +Remove blockchain and state databases`, } dumpCommand = cli.Command{ - Action: dump, + Action: utils.MigrateFlags(dump), Name: "dump", Usage: "Dump a specific block from storage", ArgsUsage: "[ | ]...", - Category: "BLOCKCHAIN COMMANDS", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.CacheFlag, + utils.LightModeFlag, + }, + Category: "BLOCKCHAIN COMMANDS", Description: ` The arguments are interpreted as block numbers or hashes. -Use "expanse dump 0" to dump the genesis block. -`, +Use "expanse dump 0" to dump the genesis block.`, } ) // initGenesis will initialise the given JSON format genesis file and writes it as // the zero'd block (i.e. genesis) or will fail hard if it can't succeed. func initGenesis(ctx *cli.Context) error { + // Make sure we have a valid genesis JSON genesisPath := ctx.Args().First() if len(genesisPath) == 0 { - utils.Fatalf("must supply path to genesis JSON file") + utils.Fatalf("Must supply path to genesis JSON file") } - - stack := makeFullNode(ctx) - chaindb := utils.MakeChainDatabase(ctx, stack) - file, err := os.Open(genesisPath) if err != nil { - utils.Fatalf("failed to read genesis file: %v", err) + utils.Fatalf("Failed to read genesis file: %v", err) } defer file.Close() @@ -121,12 +139,19 @@ func initGenesis(ctx *cli.Context) error { if err := json.NewDecoder(file).Decode(genesis); err != nil { utils.Fatalf("invalid genesis file: %v", err) } - - _, hash, err := core.SetupGenesisBlock(chaindb, genesis) - if err != nil { - utils.Fatalf("failed to write genesis block: %v", err) + // Open an initialise both full and light databases + stack := makeFullNode(ctx) + for _, name := range []string{"chaindata", "lightchaindata"} { + chaindb, err := stack.OpenDatabase(name, 0, 0) + if err != nil { + utils.Fatalf("Failed to open database: %v", err) + } + _, hash, err := core.SetupGenesisBlock(chaindb, genesis) + if err != nil { + utils.Fatalf("Failed to write genesis block: %v", err) + } + log.Info("Successfully wrote genesis state", "database", name, "hash", hash) } - log.Info("Successfully wrote genesis state", "hash", hash) return nil } @@ -245,24 +270,29 @@ func exportChain(ctx *cli.Context) error { func removeDB(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) - dbdir := stack.ResolvePath(utils.ChainDbName(ctx)) - if !common.FileExist(dbdir) { - fmt.Println(dbdir, "does not exist") - return nil - } - fmt.Println(dbdir) - confirm, err := console.Stdin.PromptConfirm("Remove this database?") - switch { - case err != nil: - utils.Fatalf("%v", err) - case !confirm: - fmt.Println("Operation aborted") - default: - fmt.Println("Removing...") - start := time.Now() - os.RemoveAll(dbdir) - fmt.Printf("Removed in %v\n", time.Since(start)) + for _, name := range []string{"chaindata", "lightchaindata"} { + // Ensure the database exists in the first place + logger := log.New("database", name) + + dbdir := stack.ResolvePath(name) + if !common.FileExist(dbdir) { + logger.Info("Database doesn't exist, skipping", "path", dbdir) + continue + } + // Confirm removal and execute + fmt.Println(dbdir) + confirm, err := console.Stdin.PromptConfirm("Remove this database?") + switch { + case err != nil: + utils.Fatalf("%v", err) + case !confirm: + logger.Warn("Database deletion aborted") + default: + start := time.Now() + os.RemoveAll(dbdir) + logger.Info("Database successfully deleted", "elapsed", common.PrettyDuration(time.Since(start))) + } } return nil } diff --git a/cmd/gexp/config.go b/cmd/gexp/config.go index c711922c99..fa6445e8d0 100644 --- a/cmd/gexp/config.go +++ b/cmd/gexp/config.go @@ -38,10 +38,11 @@ import ( var ( dumpConfigCommand = cli.Command{ - Action: dumpConfig, + Action: utils.MigrateFlags(dumpConfig), Name: "dumpconfig", Usage: "Show configuration values", ArgsUsage: "", + Flags: append(nodeFlags, rpcFlags...), Category: "MISCELLANEOUS COMMANDS", Description: `The dumpconfig command shows configuration values.`, } diff --git a/cmd/gexp/consolecmd.go b/cmd/gexp/consolecmd.go index 2227648c42..1530909c63 100644 --- a/cmd/gexp/consolecmd.go +++ b/cmd/gexp/consolecmd.go @@ -29,23 +29,26 @@ import ( ) var ( + consoleFlags = []cli.Flag{utils.JSpathFlag, utils.ExecFlag, utils.PreloadJSFlag} + consoleCommand = cli.Command{ - Action: localConsole, - Name: "console", - Usage: "Start an interactive JavaScript environment", - ArgsUsage: "", // TODO: Write this! - Category: "CONSOLE COMMANDS", + Action: utils.MigrateFlags(localConsole), + Name: "console", + Usage: "Start an interactive JavaScript environment", + Flags: append(append(nodeFlags, rpcFlags...), consoleFlags...), + Category: "CONSOLE COMMANDS", Description: ` The Gexp console is an interactive shell for the JavaScript runtime environment which exposes a node admin interface as well as the Ðapp JavaScript API. -See https://github.com/expanse-org/go-expanse/wiki/Javascipt-Console -`, +See https://github.com/expanse-org/go-expanse/wiki/Javascipt-Console.`, } + attachCommand = cli.Command{ - Action: remoteConsole, + Action: utils.MigrateFlags(remoteConsole), Name: "attach", Usage: "Start an interactive JavaScript environment (connect to node)", - ArgsUsage: "", // TODO: Write this! + ArgsUsage: "[endpoint]", + Flags: append(consoleFlags, utils.DataDirFlag), Category: "CONSOLE COMMANDS", Description: ` The Gexp console is an interactive shell for the JavaScript runtime environment @@ -54,11 +57,13 @@ See https://github.com/expanse-org/go-expanse/wiki/Javascipt-Console. This command allows to open a console on a running gexp node. `, } + javascriptCommand = cli.Command{ - Action: ephemeralConsole, + Action: utils.MigrateFlags(ephemeralConsole), Name: "js", Usage: "Execute the specified JavaScript files", - ArgsUsage: "", // TODO: Write this! + ArgsUsage: " [jsfile...]", + Flags: append(nodeFlags, consoleFlags...), Category: "CONSOLE COMMANDS", Description: ` The JavaScript VM exposes a node admin interface as well as the Ðapp @@ -81,11 +86,12 @@ func localConsole(ctx *cli.Context) error { utils.Fatalf("Failed to attach to the inproc gexp: %v", err) } config := console.Config{ - DataDir: node.DataDir(), + DataDir: utils.MakeDataDir(ctx), DocRoot: ctx.GlobalString(utils.JSpathFlag.Name), Client: client, Preload: utils.MakeConsolePreloads(ctx), } + console, err := console.New(config) if err != nil { utils.Fatalf("Failed to start the JavaScript console: %v", err) @@ -118,17 +124,18 @@ func remoteConsole(ctx *cli.Context) error { Client: client, Preload: utils.MakeConsolePreloads(ctx), } + console, err := console.New(config) if err != nil { utils.Fatalf("Failed to start the JavaScript console: %v", err) } defer console.Stop(false) - // If only a short execution was requested, evaluate and return if script := ctx.GlobalString(utils.ExecFlag.Name); script != "" { console.Evaluate(script) return nil } + // Otherwise print the welcome screen and enter interactive mode console.Welcome() console.Interactive() @@ -151,7 +158,7 @@ func dialRPC(endpoint string) (*rpc.Client, error) { } // ephemeralConsole starts a new gexp node, attaches an ephemeral JavaScript -// console to it, and each of the files specified as arguments and tears the +// console to it, executes each of the files specified as arguments and tears // everything down. func ephemeralConsole(ctx *cli.Context) error { // Create and start the node based on the CLI flags @@ -165,11 +172,12 @@ func ephemeralConsole(ctx *cli.Context) error { utils.Fatalf("Failed to attach to the inproc gexp: %v", err) } config := console.Config{ - DataDir: node.DataDir(), + DataDir: utils.MakeDataDir(ctx), DocRoot: ctx.GlobalString(utils.JSpathFlag.Name), Client: client, Preload: utils.MakeConsolePreloads(ctx), } + console, err := console.New(config) if err != nil { utils.Fatalf("Failed to start the JavaScript console: %v", err) diff --git a/cmd/gexp/main.go b/cmd/gexp/main.go index 3bc8f70a15..17b81370c7 100644 --- a/cmd/gexp/main.go +++ b/cmd/gexp/main.go @@ -49,13 +49,89 @@ var ( relOracle = common.HexToAddress("0x926d69cc3bbf81d52cba6886d788df007a15a3cd") // The app that holds all commands and flags. app = utils.NewApp(gitCommit, "the go-expanse command line interface") + // flags that configure the node + nodeFlags = []cli.Flag{ + utils.IdentityFlag, + utils.UnlockedAccountFlag, + utils.PasswordFileFlag, + utils.BootnodesFlag, + utils.BootnodesV4Flag, + utils.BootnodesV5Flag, + utils.DataDirFlag, + utils.KeyStoreDirFlag, + utils.NoUSBFlag, + utils.EthashCacheDirFlag, + utils.EthashCachesInMemoryFlag, + utils.EthashCachesOnDiskFlag, + utils.EthashDatasetDirFlag, + utils.EthashDatasetsInMemoryFlag, + utils.EthashDatasetsOnDiskFlag, + utils.TxPoolPriceLimitFlag, + utils.TxPoolPriceBumpFlag, + utils.TxPoolAccountSlotsFlag, + utils.TxPoolGlobalSlotsFlag, + utils.TxPoolAccountQueueFlag, + utils.TxPoolGlobalQueueFlag, + utils.TxPoolLifetimeFlag, + utils.FastSyncFlag, + utils.LightModeFlag, + utils.SyncModeFlag, + utils.LightServFlag, + utils.LightPeersFlag, + utils.LightKDFFlag, + utils.CacheFlag, + utils.TrieCacheGenFlag, + utils.ListenPortFlag, + utils.MaxPeersFlag, + utils.MaxPendingPeersFlag, + utils.EtherbaseFlag, + utils.GasPriceFlag, + utils.MinerThreadsFlag, + utils.MiningEnabledFlag, + utils.TargetGasLimitFlag, + utils.NATFlag, + utils.NoDiscoverFlag, + utils.DiscoveryV5Flag, + utils.NetrestrictFlag, + utils.NodeKeyFileFlag, + utils.NodeKeyHexFlag, + utils.WhisperEnabledFlag, + utils.DevModeFlag, + utils.TestnetFlag, + utils.RinkebyFlag, + utils.VMEnableDebugFlag, + utils.NetworkIdFlag, + utils.RPCCORSDomainFlag, + utils.EthStatsURLFlag, + utils.MetricsEnabledFlag, + utils.FakePoWFlag, + utils.NoCompactionFlag, + utils.GpoBlocksFlag, + utils.GpoPercentileFlag, + utils.ExtraDataFlag, + configFileFlag, + } + + rpcFlags = []cli.Flag{ + utils.RPCEnabledFlag, + utils.RPCListenAddrFlag, + utils.RPCPortFlag, + utils.RPCApiFlag, + utils.WSEnabledFlag, + utils.WSListenAddrFlag, + utils.WSPortFlag, + utils.WSApiFlag, + utils.WSAllowedOriginsFlag, + utils.IPCDisabledFlag, + utils.IPCPathFlag, + } ) func init() { - // Initialize the CLI app and start Gexp + // Initialize the CLI app and start Geth app.Action = gexp app.HideVersion = true // we have a command to print the version - app.Copyright = "Copyright 2013-2017 The go-ethereum / go-expanse Authors" + app.Copyright = "Copyright 2013-2017 The go-ethereum Authors & the go-expanse Authors" app.Commands = []cli.Command{ // See chaincmd.go: initCommand, @@ -81,70 +157,9 @@ func init() { dumpConfigCommand, } - app.Flags = []cli.Flag{ - utils.IdentityFlag, - utils.UnlockedAccountFlag, - utils.PasswordFileFlag, - utils.BootnodesFlag, - utils.DataDirFlag, - utils.KeyStoreDirFlag, - utils.EthashCacheDirFlag, - utils.EthashCachesInMemoryFlag, - utils.EthashCachesOnDiskFlag, - utils.EthashDatasetDirFlag, - utils.EthashDatasetsInMemoryFlag, - utils.EthashDatasetsOnDiskFlag, - utils.FastSyncFlag, - utils.LightModeFlag, - utils.SyncModeFlag, - utils.LightServFlag, - utils.LightPeersFlag, - utils.LightKDFFlag, - utils.CacheFlag, - utils.TrieCacheGenFlag, - utils.JSpathFlag, - utils.ListenPortFlag, - utils.MaxPeersFlag, - utils.MaxPendingPeersFlag, - utils.EtherbaseFlag, - utils.GasPriceFlag, - utils.MinerThreadsFlag, - utils.MiningEnabledFlag, - utils.TargetGasLimitFlag, - utils.NATFlag, - utils.NoDiscoverFlag, - utils.DiscoveryV5Flag, - utils.NetrestrictFlag, - utils.NodeKeyFileFlag, - utils.NodeKeyHexFlag, - utils.RPCEnabledFlag, - utils.RPCListenAddrFlag, - utils.RPCPortFlag, - utils.RPCApiFlag, - utils.WSEnabledFlag, - utils.WSListenAddrFlag, - utils.WSPortFlag, - utils.WSApiFlag, - utils.WSAllowedOriginsFlag, - utils.IPCDisabledFlag, - utils.IPCPathFlag, - utils.ExecFlag, - utils.PreloadJSFlag, - utils.WhisperEnabledFlag, - utils.DevModeFlag, - utils.TestNetFlag, - utils.VMEnableDebugFlag, - utils.NetworkIdFlag, - utils.RPCCORSDomainFlag, - utils.EthStatsURLFlag, - utils.MetricsEnabledFlag, - utils.FakePoWFlag, - utils.NoCompactionFlag, - utils.GpoBlocksFlag, - utils.GpoPercentileFlag, - utils.ExtraDataFlag, - configFileFlag, - } + app.Flags = append(app.Flags, nodeFlags...) + app.Flags = append(app.Flags, rpcFlags...) + app.Flags = append(app.Flags, consoleFlags...) app.Flags = append(app.Flags, debug.Flags...) app.Before = func(ctx *cli.Context) error { diff --git a/cmd/gexp/misccmd.go b/cmd/gexp/misccmd.go index 1d9f16a78c..38c94c9c1d 100644 --- a/cmd/gexp/misccmd.go +++ b/cmd/gexp/misccmd.go @@ -34,7 +34,7 @@ import ( var ( makedagCommand = cli.Command{ - Action: makedag, + Action: utils.MigrateFlags(makedag), Name: "makedag", Usage: "Generate ethash DAG (for testing)", ArgsUsage: " ", @@ -47,7 +47,7 @@ Regular users do not need to execute it. `, } versionCommand = cli.Command{ - Action: version, + Action: utils.MigrateFlags(version), Name: "version", Usage: "Print version numbers", ArgsUsage: " ", @@ -57,7 +57,7 @@ The output of this command is supposed to be machine-readable. `, } licenseCommand = cli.Command{ - Action: license, + Action: utils.MigrateFlags(license), Name: "license", Usage: "Display license information", ArgsUsage: " ", @@ -103,7 +103,7 @@ func version(ctx *cli.Context) error { } fmt.Println("Architecture:", runtime.GOARCH) fmt.Println("Protocol Versions:", eth.ProtocolVersions) - fmt.Println("Network Id:", ctx.GlobalInt(utils.NetworkIdFlag.Name)) + fmt.Println("Network Id:", eth.DefaultConfig.NetworkId) fmt.Println("Go Version:", runtime.Version()) fmt.Println("Operating System:", runtime.GOOS) fmt.Printf("GOPATH=%s\n", os.Getenv("GOPATH")) diff --git a/cmd/gexp/monitorcmd.go b/cmd/gexp/monitorcmd.go index cf9d062078..53e2124956 100644 --- a/cmd/gexp/monitorcmd.go +++ b/cmd/gexp/monitorcmd.go @@ -49,7 +49,7 @@ var ( Usage: "Refresh interval in seconds", } monitorCommand = cli.Command{ - Action: monitor, + Action: utils.MigrateFlags(monitor), // keep track of migration progress Name: "monitor", Usage: "Monitor and visualize node metrics", ArgsUsage: " ", diff --git a/cmd/gexp/usage.go b/cmd/gexp/usage.go index f84af231a5..794b9ed97a 100644 --- a/cmd/gexp/usage.go +++ b/cmd/gexp/usage.go @@ -20,6 +20,7 @@ package main import ( "io" + "sort" "github.com/expanse-org/go-expanse/cmd/utils" "github.com/expanse-org/go-expanse/internal/debug" @@ -67,8 +68,10 @@ var AppHelpFlagGroups = []flagGroup{ configFileFlag, utils.DataDirFlag, utils.KeyStoreDirFlag, + utils.NoUSBFlag, utils.NetworkIdFlag, - utils.TestNetFlag, + utils.TestnetFlag, + utils.RinkebyFlag, utils.DevModeFlag, utils.SyncModeFlag, utils.EthStatsURLFlag, @@ -89,6 +92,18 @@ var AppHelpFlagGroups = []flagGroup{ utils.EthashDatasetsOnDiskFlag, }, }, + { + Name: "TRANSACTION POOL", + Flags: []cli.Flag{ + utils.TxPoolPriceLimitFlag, + utils.TxPoolPriceBumpFlag, + utils.TxPoolAccountSlotsFlag, + utils.TxPoolGlobalSlotsFlag, + utils.TxPoolAccountQueueFlag, + utils.TxPoolGlobalQueueFlag, + utils.TxPoolLifetimeFlag, + }, + }, { Name: "PERFORMANCE TUNING", Flags: []cli.Flag{ @@ -127,6 +142,8 @@ var AppHelpFlagGroups = []flagGroup{ Name: "NETWORKING", Flags: []cli.Flag{ utils.BootnodesFlag, + utils.BootnodesV4Flag, + utils.BootnodesV5Flag, utils.ListenPortFlag, utils.MaxPeersFlag, utils.MaxPendingPeersFlag, @@ -185,6 +202,39 @@ var AppHelpFlagGroups = []flagGroup{ }, } +// byCategory sorts an array of flagGroup by Name in the order +// defined in AppHelpFlagGroups. +type byCategory []flagGroup + +func (a byCategory) Len() int { return len(a) } +func (a byCategory) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a byCategory) Less(i, j int) bool { + iCat, jCat := a[i].Name, a[j].Name + iIdx, jIdx := len(AppHelpFlagGroups), len(AppHelpFlagGroups) // ensure non categorized flags come last + + for i, group := range AppHelpFlagGroups { + if iCat == group.Name { + iIdx = i + } + if jCat == group.Name { + jIdx = i + } + } + + return iIdx < jIdx +} + +func flagCategory(flag cli.Flag) string { + for _, category := range AppHelpFlagGroups { + for _, flg := range category.Flags { + if flg.GetName() == flag.GetName() { + return category.Name + } + } + } + return "MISC" +} + func init() { // Override the default app help template cli.AppHelpTemplate = AppHelpTemplate @@ -194,6 +244,7 @@ func init() { App interface{} FlagGroups []flagGroup } + // Override the default app help printer, but only for the global app help originalHelpPrinter := cli.HelpPrinter cli.HelpPrinter = func(w io.Writer, tmpl string, data interface{}) { @@ -223,6 +274,27 @@ func init() { } // Render out custom usage screen originalHelpPrinter(w, tmpl, helpData{data, AppHelpFlagGroups}) + } else if tmpl == utils.CommandHelpTemplate { + // Iterate over all command specific flags and categorize them + categorized := make(map[string][]cli.Flag) + for _, flag := range data.(cli.Command).Flags { + if _, ok := categorized[flag.String()]; !ok { + categorized[flagCategory(flag)] = append(categorized[flagCategory(flag)], flag) + } + } + + // sort to get a stable ordering + sorted := make([]flagGroup, 0, len(categorized)) + for cat, flgs := range categorized { + sorted = append(sorted, flagGroup{cat, flgs}) + } + sort.Sort(byCategory(sorted)) + + // add sorted array to data and render with default printer + originalHelpPrinter(w, tmpl, map[string]interface{}{ + "cmd": data, + "categorizedFlags": sorted, + }) } else { originalHelpPrinter(w, tmpl, data) } diff --git a/cmd/puppeth/module_faucet.go b/cmd/puppeth/module_faucet.go index 7ff0aa0743..f15725a5df 100644 --- a/cmd/puppeth/module_faucet.go +++ b/cmd/puppeth/module_faucet.go @@ -51,9 +51,10 @@ ADD account.pass /account.pass EXPOSE 8080 CMD [ \ - "/faucet", "--genesis", "/genesis.json", "--network", "{{.NetworkID}}", "--bootnodes", "{{.Bootnodes}}", "--ethstats", "{{.Ethstats}}", \ - "--ethport", "{{.EthPort}}", "--faucet.name", "{{.FaucetName}}", "--faucet.amount", "{{.FaucetAmount}}", "--faucet.minutes", "{{.FaucetMinutes}}", \ - "--github.user", "{{.GitHubUser}}", "--github.token", "{{.GitHubToken}}", "--account.json", "/account.json", "--account.pass", "/account.pass" \ + "/faucet", "--genesis", "/genesis.json", "--network", "{{.NetworkID}}", "--bootnodes", "{{.Bootnodes}}", "--ethstats", "{{.Ethstats}}", "--ethport", "{{.EthPort}}", \ + "--faucet.name", "{{.FaucetName}}", "--faucet.amount", "{{.FaucetAmount}}", "--faucet.minutes", "{{.FaucetMinutes}}", "--faucet.tiers", "{{.FaucetTiers}}", \ + "--github.user", "{{.GitHubUser}}", "--github.token", "{{.GitHubToken}}", "--account.json", "/account.json", "--account.pass", "/account.pass" \ + {{if .CaptchaToken}}, "--captcha.token", "{{.CaptchaToken}}", "--captcha.secret", "{{.CaptchaSecret}}"{{end}} \ ]` // faucetComposefile is the docker-compose.yml file required to deploy and maintain @@ -74,8 +75,11 @@ services: - ETH_NAME={{.EthName}} - FAUCET_AMOUNT={{.FaucetAmount}} - FAUCET_MINUTES={{.FaucetMinutes}} + - FAUCET_TIERS={{.FaucetTiers}} - GITHUB_USER={{.GitHubUser}} - - GITHUB_TOKEN={{.GitHubToken}}{{if .VHost}} + - GITHUB_TOKEN={{.GitHubToken}} + - CAPTCHA_TOKEN={{.CaptchaToken}} + - CAPTCHA_SECRET={{.CaptchaSecret}}{{if .VHost}} - VIRTUAL_HOST={{.VHost}} - VIRTUAL_PORT=8080{{end}} restart: always @@ -97,9 +101,12 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config "EthPort": config.node.portFull, "GitHubUser": config.githubUser, "GitHubToken": config.githubToken, + "CaptchaToken": config.captchaToken, + "CaptchaSecret": config.captchaSecret, "FaucetName": strings.Title(network), "FaucetAmount": config.amount, "FaucetMinutes": config.minutes, + "FaucetTiers": config.tiers, }) files[filepath.Join(workdir, "Dockerfile")] = dockerfile.Bytes() @@ -113,8 +120,11 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config "EthName": config.node.ethstats[:strings.Index(config.node.ethstats, ":")], "GitHubUser": config.githubUser, "GitHubToken": config.githubToken, + "CaptchaToken": config.captchaToken, + "CaptchaSecret": config.captchaSecret, "FaucetAmount": config.amount, "FaucetMinutes": config.minutes, + "FaucetTiers": config.tiers, }) files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes() @@ -135,18 +145,21 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config // faucetInfos is returned from an faucet status check to allow reporting various // configuration parameters. type faucetInfos struct { - node *nodeInfos - host string - port int - amount int - minutes int - githubUser string - githubToken string + node *nodeInfos + host string + port int + amount int + minutes int + tiers int + githubUser string + githubToken string + captchaToken string + captchaSecret string } // String implements the stringer interface. func (info *faucetInfos) String() string { - return fmt.Sprintf("host=%s, api=%d, eth=%d, amount=%d, minutes=%d, github=%s, ethstats=%s", info.host, info.port, info.node.portFull, info.amount, info.minutes, info.githubUser, info.node.ethstats) + return fmt.Sprintf("host=%s, api=%d, eth=%d, amount=%d, minutes=%d, tiers=%d, github=%s, captcha=%v, ethstats=%s", info.host, info.port, info.node.portFull, info.amount, info.minutes, info.tiers, info.githubUser, info.captchaToken != "", info.node.ethstats) } // checkFaucet does a health-check against an faucet server to verify whether @@ -177,6 +190,7 @@ func checkFaucet(client *sshClient, network string) (*faucetInfos, error) { } amount, _ := strconv.Atoi(infos.envvars["FAUCET_AMOUNT"]) minutes, _ := strconv.Atoi(infos.envvars["FAUCET_MINUTES"]) + tiers, _ := strconv.Atoi(infos.envvars["FAUCET_TIERS"]) // Retrieve the funding account informations var out []byte @@ -200,11 +214,14 @@ func checkFaucet(client *sshClient, network string) (*faucetInfos, error) { keyJSON: keyJSON, keyPass: keyPass, }, - host: host, - port: port, - amount: amount, - minutes: minutes, - githubUser: infos.envvars["GITHUB_USER"], - githubToken: infos.envvars["GITHUB_TOKEN"], + host: host, + port: port, + amount: amount, + minutes: minutes, + tiers: tiers, + githubUser: infos.envvars["GITHUB_USER"], + githubToken: infos.envvars["GITHUB_TOKEN"], + captchaToken: infos.envvars["CAPTCHA_TOKEN"], + captchaSecret: infos.envvars["CAPTCHA_SECRET"], }, nil } diff --git a/cmd/puppeth/module_node.go b/cmd/puppeth/module_node.go index bf1ec6b41f..fe8a616a93 100644 --- a/cmd/puppeth/module_node.go +++ b/cmd/puppeth/module_node.go @@ -40,7 +40,7 @@ ADD genesis.json /genesis.json RUN \ echo '/geth init /genesis.json' > geth.sh && \{{if .Unlock}} echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}} - echo $'/geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine{{end}}{{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}}' >> geth.sh + echo $'/geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .BootV4}}--bootnodesv4 {{.BootV4}}{{end}} {{if .BootV5}}--bootnodesv5 {{.BootV5}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine{{end}}{{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> geth.sh ENTRYPOINT ["/bin/sh", "geth.sh"] ` @@ -66,17 +66,20 @@ services: - LIGHT_PEERS={{.LightPeers}} - STATS_NAME={{.Ethstats}} - MINER_NAME={{.Etherbase}} + - GAS_TARGET={{.GasTarget}} + - GAS_PRICE={{.GasPrice}} restart: always ` // deployNode deploys a new Ethereum node container to a remote machine via SSH, // docker and docker-compose. If an instance with the specified network name // already exists there, it will be overwritten! -func deployNode(client *sshClient, network string, bootnodes []string, config *nodeInfos) ([]byte, error) { +func deployNode(client *sshClient, network string, bootv4, bootv5 []string, config *nodeInfos) ([]byte, error) { kind := "sealnode" if config.keyJSON == "" && config.etherbase == "" { kind = "bootnode" - bootnodes = make([]string, 0) + bootv4 = make([]string, 0) + bootv5 = make([]string, 0) } // Generate the content to upload to the server workdir := fmt.Sprintf("%d", rand.Int63()) @@ -92,9 +95,12 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n "Port": config.portFull, "Peers": config.peersTotal, "LightFlag": lightFlag, - "Bootnodes": strings.Join(bootnodes, ","), + "BootV4": strings.Join(bootv4, ","), + "BootV5": strings.Join(bootv5, ","), "Ethstats": config.ethstats, "Etherbase": config.etherbase, + "GasTarget": uint64(1000000 * config.gasTarget), + "GasPrice": uint64(1000000000 * config.gasPrice), "Unlock": config.keyJSON != "", }) files[filepath.Join(workdir, "Dockerfile")] = dockerfile.Bytes() @@ -111,6 +117,8 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n "LightPeers": config.peersLight, "Ethstats": config.ethstats[:strings.Index(config.ethstats, ":")], "Etherbase": config.etherbase, + "GasTarget": config.gasTarget, + "GasPrice": config.gasPrice, }) files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes() @@ -127,7 +135,7 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n } defer client.Run("rm -rf " + workdir) - // Build and deploy the bootnode service + // Build and deploy the boot or seal node service return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build", workdir, network)) } @@ -147,6 +155,8 @@ type nodeInfos struct { etherbase string keyJSON string keyPass string + gasTarget float64 + gasPrice float64 } // String implements the stringer interface. @@ -155,7 +165,8 @@ func (info *nodeInfos) String() string { if info.peersLight > 0 { discv5 = fmt.Sprintf(", portv5=%d", info.portLight) } - return fmt.Sprintf("port=%d%s, datadir=%s, peers=%d, lights=%d, ethstats=%s", info.portFull, discv5, info.datadir, info.peersTotal, info.peersLight, info.ethstats) + return fmt.Sprintf("port=%d%s, datadir=%s, peers=%d, lights=%d, ethstats=%s, gastarget=%0.3f MGas, gasprice=%0.3f GWei", + info.portFull, discv5, info.datadir, info.peersTotal, info.peersLight, info.ethstats, info.gasTarget, info.gasPrice) } // checkNode does a health-check against an boot or seal node server to verify @@ -176,6 +187,8 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error) // Resolve a few types from the environmental variables totalPeers, _ := strconv.Atoi(infos.envvars["TOTAL_PEERS"]) lightPeers, _ := strconv.Atoi(infos.envvars["LIGHT_PEERS"]) + gasTarget, _ := strconv.ParseFloat(infos.envvars["GAS_TARGET"], 64) + gasPrice, _ := strconv.ParseFloat(infos.envvars["GAS_PRICE"], 64) // Container available, retrieve its node ID and its genesis json var out []byte @@ -213,6 +226,8 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error) etherbase: infos.envvars["MINER_NAME"], keyJSON: keyJSON, keyPass: keyPass, + gasTarget: gasTarget, + gasPrice: gasPrice, } stats.enodeFull = fmt.Sprintf("enode://%s@%s:%d", id, client.address, stats.portFull) if stats.portLight != 0 { diff --git a/cmd/puppeth/ssh.go b/cmd/puppeth/ssh.go index 34263d06be..c142e71bbc 100644 --- a/cmd/puppeth/ssh.go +++ b/cmd/puppeth/ssh.go @@ -17,6 +17,8 @@ package main import ( + "bufio" + "bytes" "errors" "fmt" "io/ioutil" @@ -37,18 +39,26 @@ import ( type sshClient struct { server string // Server name or IP without port number address string // IP address of the remote server + pubkey []byte // RSA public key to authenticate the server client *ssh.Client logger log.Logger } // dial establishes an SSH connection to a remote node using the current user and -// the user's configured private RSA key. -func dial(server string) (*sshClient, error) { +// the user's configured private RSA key. If that fails, password authentication +// is fallen back to. The caller may override the login user via user@server:port. +func dial(server string, pubkey []byte) (*sshClient, error) { // Figure out a label for the server and a logger label := server if strings.Contains(label, ":") { label = label[:strings.Index(label, ":")] } + login := "" + if strings.Contains(server, "@") { + login = label[:strings.Index(label, "@")] + label = label[strings.Index(label, "@")+1:] + server = server[strings.Index(server, "@")+1:] + } logger := log.New("server", label) logger.Debug("Attempting to establish SSH connection") @@ -56,6 +66,9 @@ func dial(server string) (*sshClient, error) { if err != nil { return nil, err } + if login == "" { + login = user.Username + } // Configure the supported authentication methods (private key and password) var auths []ssh.AuthMethod @@ -71,7 +84,7 @@ func dial(server string) (*sshClient, error) { } } auths = append(auths, ssh.PasswordCallback(func() (string, error) { - fmt.Printf("What's the login password for %s at %s? (won't be echoed)\n> ", user.Username, server) + fmt.Printf("What's the login password for %s at %s? (won't be echoed)\n> ", login, server) blob, err := terminal.ReadPassword(int(syscall.Stdin)) fmt.Println() @@ -86,11 +99,36 @@ func dial(server string) (*sshClient, error) { return nil, errors.New("no IPs associated with domain") } // Try to dial in to the remote server - logger.Trace("Dialing remote SSH server", "user", user.Username, "key", path) + logger.Trace("Dialing remote SSH server", "user", login) if !strings.Contains(server, ":") { server += ":22" } - client, err := ssh.Dial("tcp", server, &ssh.ClientConfig{User: user.Username, Auth: auths}) + keycheck := func(hostname string, remote net.Addr, key ssh.PublicKey) error { + // If no public key is known for SSH, ask the user to confirm + if pubkey == nil { + fmt.Printf("The authenticity of host '%s (%s)' can't be established.\n", hostname, remote) + fmt.Printf("SSH key fingerprint is %s [MD5]\n", ssh.FingerprintLegacyMD5(key)) + fmt.Printf("Are you sure you want to continue connecting (yes/no)? ") + + text, err := bufio.NewReader(os.Stdin).ReadString('\n') + switch { + case err != nil: + return err + case strings.TrimSpace(text) == "yes": + pubkey = key.Marshal() + return nil + default: + return fmt.Errorf("unknown auth choice: %v", text) + } + } + // If a public key exists for this SSH server, check that it matches + if bytes.Compare(pubkey, key.Marshal()) == 0 { + return nil + } + // We have a mismatch, forbid connecting + return errors.New("ssh key mismatch, readd the machine to update") + } + client, err := ssh.Dial("tcp", server, &ssh.ClientConfig{User: login, Auth: auths, HostKeyCallback: keycheck}) if err != nil { return nil, err } @@ -98,6 +136,7 @@ func dial(server string) (*sshClient, error) { c := &sshClient{ server: label, address: addr[0], + pubkey: pubkey, client: client, logger: logger, } diff --git a/cmd/puppeth/wizard.go b/cmd/puppeth/wizard.go index 823e35cdb9..ad7a5b8f8e 100644 --- a/cmd/puppeth/wizard.go +++ b/cmd/puppeth/wizard.go @@ -44,14 +44,24 @@ type config struct { bootLight []string // Bootnodes to always connect to by light nodes ethstats string // Ethstats settings to cache for node deploys - Servers []string `json:"servers,omitempty"` + Servers map[string][]byte `json:"servers,omitempty"` +} + +// servers retrieves an alphabetically sorted list of servers. +func (c config) servers() []string { + servers := make([]string, 0, len(c.Servers)) + for server := range c.Servers { + servers = append(servers, server) + } + sort.Strings(servers) + + return servers } // flush dumps the contents of config to disk. func (c config) flush() { os.MkdirAll(filepath.Dir(c.path), 0755) - sort.Strings(c.Servers) out, _ := json.MarshalIndent(c, "", " ") if err := ioutil.WriteFile(c.path, out, 0644); err != nil { log.Warn("Failed to save puppeth configs", "file", c.path, "err", err) @@ -152,6 +162,48 @@ func (w *wizard) readDefaultInt(def int) int { } } +// readFloat reads a single line from stdin, trimming if from spaces, enforcing it +// to parse into a float. +func (w *wizard) readFloat() float64 { + for { + fmt.Printf("> ") + text, err := w.in.ReadString('\n') + if err != nil { + log.Crit("Failed to read user input", "err", err) + } + if text = strings.TrimSpace(text); text == "" { + continue + } + val, err := strconv.ParseFloat(strings.TrimSpace(text), 64) + if err != nil { + log.Error("Invalid input, expected float", "err", err) + continue + } + return val + } +} + +// readDefaultFloat reads a single line from stdin, trimming if from spaces, enforcing +// it to parse into a float. If an empty line is entered, the default value is returned. +func (w *wizard) readDefaultFloat(def float64) float64 { + for { + fmt.Printf("> ") + text, err := w.in.ReadString('\n') + if err != nil { + log.Crit("Failed to read user input", "err", err) + } + if text = strings.TrimSpace(text); text == "" { + return def + } + val, err := strconv.ParseFloat(strings.TrimSpace(text), 64) + if err != nil { + log.Error("Invalid input, expected float", "err", err) + continue + } + return val + } +} + // readPassword reads a single line from stdin, trimming it from the trailing new // line and returns it. The input will not be echoed. func (w *wizard) readPassword() string { diff --git a/cmd/puppeth/wizard_faucet.go b/cmd/puppeth/wizard_faucet.go index 92e0d0ec21..13d0cddbca 100644 --- a/cmd/puppeth/wizard_faucet.go +++ b/cmd/puppeth/wizard_faucet.go @@ -44,6 +44,7 @@ func (w *wizard) deployFaucet() { host: client.server, amount: 1, minutes: 1440, + tiers: 3, } } infos.node.genesis, _ = json.MarshalIndent(w.conf.genesis, "", " ") @@ -68,10 +69,17 @@ func (w *wizard) deployFaucet() { fmt.Printf("How many minutes to enforce between requests? (default = %d)\n", infos.minutes) infos.minutes = w.readDefaultInt(infos.minutes) + fmt.Println() + fmt.Printf("How many funding tiers to feature (x2.5 amounts, x3 timeout)? (default = %d)\n", infos.tiers) + infos.tiers = w.readDefaultInt(infos.tiers) + if infos.tiers == 0 { + log.Error("At least one funding tier must be set") + return + } // Accessing GitHub gists requires API authorization, retrieve it if infos.githubUser != "" { fmt.Println() - fmt.Printf("Reused previous (%s) GitHub API authorization (y/n)? (default = yes)\n", infos.githubUser) + fmt.Printf("Reuse previous (%s) GitHub API authorization (y/n)? (default = yes)\n", infos.githubUser) if w.readDefaultString("y") != "y" { infos.githubUser, infos.githubToken = "", "" } @@ -109,6 +117,29 @@ func (w *wizard) deployFaucet() { return } } + // Accessing the reCaptcha service requires API authorizations, request it + if infos.captchaToken != "" { + fmt.Println() + fmt.Println("Reuse previous reCaptcha API authorization (y/n)? (default = yes)") + if w.readDefaultString("y") != "y" { + infos.captchaToken, infos.captchaSecret = "", "" + } + } + if infos.captchaToken == "" { + // No previous authorization (or old one discarded) + fmt.Println() + fmt.Println("Enable reCaptcha protection against robots (y/n)? (default = no)") + if w.readDefaultString("n") == "y" { + // Captcha protection explicitly requested, read the site and secret keys + fmt.Println() + fmt.Printf("What is the reCaptcha site key to authenticate human users?\n") + infos.captchaToken = w.readString() + + fmt.Println() + fmt.Printf("What is the reCaptcha secret key to verify authentications? (won't be echoed)\n") + infos.captchaSecret = w.readPassword() + } + } // Figure out where the user wants to store the persistent data fmt.Println() if infos.node.datadir == "" { diff --git a/cmd/puppeth/wizard_genesis.go b/cmd/puppeth/wizard_genesis.go index 52067b3e08..933f8bd0e7 100644 --- a/cmd/puppeth/wizard_genesis.go +++ b/cmd/puppeth/wizard_genesis.go @@ -120,7 +120,7 @@ func (w *wizard) makeGenesis() { // Query the user for some custom extras fmt.Println() fmt.Println("Specify your chain/network ID if you want an explicit one (default = random)") - genesis.Config.ChainId = big.NewInt(int64(w.readDefaultInt(rand.Intn(65536)))) + genesis.Config.ChainId = new(big.Int).SetUint64(uint64(w.readDefaultInt(rand.Intn(65536)))) fmt.Println() fmt.Println("Anything fun to embed into the genesis block? (max 32 bytes)") diff --git a/cmd/puppeth/wizard_intro.go b/cmd/puppeth/wizard_intro.go index db3636fc83..5b32392eb1 100644 --- a/cmd/puppeth/wizard_intro.go +++ b/cmd/puppeth/wizard_intro.go @@ -31,7 +31,10 @@ import ( // makeWizard creates and returns a new puppeth wizard. func makeWizard(network string) *wizard { return &wizard{ - network: network, + network: network, + conf: config{ + Servers: make(map[string][]byte), + }, servers: make(map[string]*sshClient), services: make(map[string][]string), in: bufio.NewReader(os.Stdin), @@ -77,9 +80,9 @@ func (w *wizard) run() { } else if err := json.Unmarshal(blob, &w.conf); err != nil { log.Crit("Previous configuration corrupted", "path", w.conf.path, "err", err) } else { - for _, server := range w.conf.Servers { + for server, pubkey := range w.conf.Servers { log.Info("Dialing previously configured server", "server", server) - client, err := dial(server) + client, err := dial(server, pubkey) if err != nil { log.Error("Previous server unreachable", "server", server, "err", err) } diff --git a/cmd/puppeth/wizard_netstats.go b/cmd/puppeth/wizard_netstats.go index 4839ff4090..b9ae931031 100644 --- a/cmd/puppeth/wizard_netstats.go +++ b/cmd/puppeth/wizard_netstats.go @@ -39,16 +39,16 @@ func (w *wizard) networkStats(tips bool) { // Iterate over all the specified hosts and check their status stats := tablewriter.NewWriter(os.Stdout) stats.SetHeader([]string{"Server", "IP", "Status", "Service", "Details"}) - stats.SetColWidth(128) + stats.SetColWidth(100) - for _, server := range w.conf.Servers { + for server, pubkey := range w.conf.Servers { client := w.servers[server] logger := log.New("server", server) logger.Info("Starting remote server health-check") // If the server is not connected, try to connect again if client == nil { - conn, err := dial(server) + conn, err := dial(server, pubkey) if err != nil { logger.Error("Failed to establish remote connection", "err", err) stats.Append([]string{server, "", err.Error(), "", ""}) diff --git a/cmd/puppeth/wizard_network.go b/cmd/puppeth/wizard_network.go index e24f9d2a84..d52e8e4dcf 100644 --- a/cmd/puppeth/wizard_network.go +++ b/cmd/puppeth/wizard_network.go @@ -28,7 +28,9 @@ import ( func (w *wizard) manageServers() { // List all the servers we can disconnect, along with an entry to connect a new one fmt.Println() - for i, server := range w.conf.Servers { + + servers := w.conf.servers() + for i, server := range servers { fmt.Printf(" %d. Disconnect %s\n", i+1, server) } fmt.Printf(" %d. Connect another server\n", len(w.conf.Servers)+1) @@ -40,14 +42,14 @@ func (w *wizard) manageServers() { } // If the user selected an existing server, drop it if choice <= len(w.conf.Servers) { - server := w.conf.Servers[choice-1] + server := servers[choice-1] client := w.servers[server] delete(w.servers, server) if client != nil { client.Close() } - w.conf.Servers = append(w.conf.Servers[:choice-1], w.conf.Servers[choice:]...) + delete(w.conf.Servers, server) w.conf.flush() log.Info("Disconnected existing server", "server", server) @@ -73,14 +75,14 @@ func (w *wizard) makeServer() string { // Read and fial the server to ensure docker is present input := w.readString() - client, err := dial(input) + client, err := dial(input, nil) if err != nil { log.Error("Server not ready for puppeth", "err", err) return "" } // All checks passed, start tracking the server w.servers[input] = client - w.conf.Servers = append(w.conf.Servers, input) + w.conf.Servers[input] = client.pubkey w.conf.flush() return input @@ -93,7 +95,9 @@ func (w *wizard) selectServer() string { // List the available server to the user and wait for a choice fmt.Println() fmt.Println("Which server do you want to interact with?") - for i, server := range w.conf.Servers { + + servers := w.conf.servers() + for i, server := range servers { fmt.Printf(" %d. %s\n", i+1, server) } fmt.Printf(" %d. Connect another server\n", len(w.conf.Servers)+1) @@ -105,7 +109,7 @@ func (w *wizard) selectServer() string { } // If the user requested connecting to a new server, go for it if choice <= len(w.conf.Servers) { - return w.conf.Servers[choice-1] + return servers[choice-1] } return w.makeServer() } diff --git a/cmd/puppeth/wizard_node.go b/cmd/puppeth/wizard_node.go index 750ff8e871..2fd469dcd5 100644 --- a/cmd/puppeth/wizard_node.go +++ b/cmd/puppeth/wizard_node.go @@ -50,7 +50,7 @@ func (w *wizard) deployNode(boot bool) { if boot { infos = &nodeInfos{portFull: 30303, peersTotal: 512, peersLight: 256} } else { - infos = &nodeInfos{portFull: 30303, peersTotal: 50, peersLight: 0} + infos = &nodeInfos{portFull: 30303, peersTotal: 50, peersLight: 0, gasTarget: 4.7, gasPrice: 18} } } infos.genesis, _ = json.MarshalIndent(w.conf.genesis, "", " ") @@ -109,8 +109,7 @@ func (w *wizard) deployNode(boot bool) { } else if w.conf.genesis.Config.Clique != nil { // If a previous signer was already set, offer to reuse it if infos.keyJSON != "" { - var key keystore.Key - if err := json.Unmarshal([]byte(infos.keyJSON), &key); err != nil { + if key, err := keystore.DecryptKey([]byte(infos.keyJSON), infos.keyPass); err != nil { infos.keyJSON, infos.keyPass = "", "" } else { fmt.Println() @@ -136,9 +135,17 @@ func (w *wizard) deployNode(boot bool) { } } } + // Establish the gas dynamics to be enforced by the signer + fmt.Println() + fmt.Printf("What gas limit should empty blocks target (MGas)? (default = %0.3f)\n", infos.gasTarget) + infos.gasTarget = w.readDefaultFloat(infos.gasTarget) + + fmt.Println() + fmt.Printf("What gas price should the signer require (GWei)? (default = %0.3f)\n", infos.gasPrice) + infos.gasPrice = w.readDefaultFloat(infos.gasPrice) } // Try to deploy the full node on the host - if out, err := deployNode(client, w.network, w.conf.bootFull, infos); err != nil { + if out, err := deployNode(client, w.network, w.conf.bootFull, w.conf.bootLight, infos); err != nil { log.Error("Failed to deploy Ethereum node container", "err", err) if len(out) > 0 { fmt.Printf("%s\n", out) diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 26ec22469e..bacda9060b 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -67,6 +67,10 @@ var ( Name: "bzzaccount", Usage: "Swarm account key file", } + SwarmListenAddrFlag = cli.StringFlag{ + Name: "httpaddr", + Usage: "Swarm HTTP API listening interface", + } SwarmPortFlag = cli.StringFlag{ Name: "bzzport", Usage: "Swarm local http api port", @@ -249,6 +253,7 @@ Cleans database of corrupted entries. SwarmConfigPathFlag, SwarmSwapEnabledFlag, SwarmSyncEnabledFlag, + SwarmListenAddrFlag, SwarmPortFlag, SwarmAccountFlag, SwarmNetworkIdFlag, @@ -345,6 +350,9 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { if len(bzzport) > 0 { bzzconfig.Port = bzzport } + if bzzaddr := ctx.GlobalString(SwarmListenAddrFlag.Name); bzzaddr != "" { + bzzconfig.ListenAddr = bzzaddr + } swapEnabled := ctx.GlobalBool(SwarmSwapEnabledFlag.Name) syncEnabled := ctx.GlobalBoolT(SwarmSyncEnabledFlag.Name) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 484c0b56e8..9df2016c60 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -52,10 +52,23 @@ import ( "github.com/expanse-org/go-expanse/p2p/nat" "github.com/expanse-org/go-expanse/p2p/netutil" "github.com/expanse-org/go-expanse/params" - whisper "github.com/expanse-org/go-expanse/whisper/whisperv2" + whisper "github.com/expanse-org/go-expanse/whisper/whisperv5" "gopkg.in/urfave/cli.v1" ) +var ( + CommandHelpTemplate = `{{.cmd.Name}}{{if .cmd.Subcommands}} command{{end}}{{if .cmd.Flags}} [command options]{{end}} [arguments...] +{{if .cmd.Description}}{{.cmd.Description}} +{{end}}{{if .cmd.Subcommands}} +SUBCOMMANDS: + {{range .cmd.Subcommands}}{{.cmd.Name}}{{with .cmd.ShortName}}, {{.cmd}}{{end}}{{ "\t" }}{{.cmd.Usage}} + {{end}}{{end}}{{if .categorizedFlags}} +{{range $idx, $categorized := .categorizedFlags}}{{$categorized.Name}} OPTIONS: +{{range $categorized.Flags}}{{"\t"}}{{.}} +{{end}} +{{end}}{{end}}` +) + func init() { cli.AppHelpTemplate = `{{.Name}} {{if .Flags}}[global options] {{end}}command{{if .Flags}} [command options]{{end}} [arguments...] @@ -70,16 +83,7 @@ GLOBAL OPTIONS: {{end}}{{end}} ` - cli.CommandHelpTemplate = `{{.Name}}{{if .Subcommands}} command{{end}}{{if .Flags}} [command options]{{end}} [arguments...] -{{if .Description}}{{.Description}} -{{end}}{{if .Subcommands}} -SUBCOMMANDS: - {{range .Subcommands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}} - {{end}}{{end}}{{if .Flags}} -OPTIONS: - {{range .Flags}}{{.}} - {{end}}{{end}} -` + cli.CommandHelpTemplate = CommandHelpTemplate } // NewApp creates an app with sane defaults. @@ -115,44 +119,23 @@ var ( Name: "keystore", Usage: "Directory for the keystore (default = inside the datadir)", } - EthashCacheDirFlag = DirectoryFlag{ - Name: "ethash.cachedir", - Usage: "Directory to store the ethash verification caches (default = inside the datadir)", + NoUSBFlag = cli.BoolFlag{ + Name: "nousb", + Usage: "Disables monitoring for and managine USB hardware wallets", } - EthashCachesInMemoryFlag = cli.IntFlag{ - Name: "ethash.cachesinmem", - Usage: "Number of recent ethash caches to keep in memory (16MB each)", - Value: eth.DefaultConfig.EthashCachesInMem, - } - EthashCachesOnDiskFlag = cli.IntFlag{ - Name: "ethash.cachesondisk", - Usage: "Number of recent ethash caches to keep on disk (16MB each)", - Value: eth.DefaultConfig.EthashCachesOnDisk, - } - EthashDatasetDirFlag = DirectoryFlag{ - Name: "ethash.dagdir", - Usage: "Directory to store the ethash mining DAGs (default = inside home folder)", - Value: DirectoryString{eth.DefaultConfig.EthashDatasetDir}, - } - EthashDatasetsInMemoryFlag = cli.IntFlag{ - Name: "ethash.dagsinmem", - Usage: "Number of recent ethash mining DAGs to keep in memory (1+GB each)", - Value: eth.DefaultConfig.EthashDatasetsInMem, - } - EthashDatasetsOnDiskFlag = cli.IntFlag{ - Name: "ethash.dagsondisk", - Usage: "Number of recent ethash mining DAGs to keep on disk (1+GB each)", - Value: eth.DefaultConfig.EthashDatasetsOnDisk, - } - NetworkIdFlag = cli.IntFlag{ + NetworkIdFlag = cli.Uint64Flag{ Name: "networkid", - Usage: "Network identifier (integer, 1=Frontier, 2=Morden (disused), 3=Ropsten)", + Usage: "Network identifier (integer, 1=Frontier, 2=Morden (disused), 3=Ropsten, 4=Rinkeby)", Value: eth.DefaultConfig.NetworkId, } - TestNetFlag = cli.BoolFlag{ + TestnetFlag = cli.BoolFlag{ Name: "testnet", Usage: "Ropsten network: pre-configured proof-of-work test network", } + RinkebyFlag = cli.BoolFlag{ + Name: "rinkeby", + Usage: "Rinkeby network: pre-configured proof-of-authority test network", + } DevModeFlag = cli.BoolFlag{ Name: "dev", Usage: "Developer mode: pre-configured private network with several debugging flags", @@ -195,6 +178,72 @@ var ( Name: "lightkdf", Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength", } + // Ethash settings + EthashCacheDirFlag = DirectoryFlag{ + Name: "ethash.cachedir", + Usage: "Directory to store the ethash verification caches (default = inside the datadir)", + } + EthashCachesInMemoryFlag = cli.IntFlag{ + Name: "ethash.cachesinmem", + Usage: "Number of recent ethash caches to keep in memory (16MB each)", + Value: eth.DefaultConfig.EthashCachesInMem, + } + EthashCachesOnDiskFlag = cli.IntFlag{ + Name: "ethash.cachesondisk", + Usage: "Number of recent ethash caches to keep on disk (16MB each)", + Value: eth.DefaultConfig.EthashCachesOnDisk, + } + EthashDatasetDirFlag = DirectoryFlag{ + Name: "ethash.dagdir", + Usage: "Directory to store the ethash mining DAGs (default = inside home folder)", + Value: DirectoryString{eth.DefaultConfig.EthashDatasetDir}, + } + EthashDatasetsInMemoryFlag = cli.IntFlag{ + Name: "ethash.dagsinmem", + Usage: "Number of recent ethash mining DAGs to keep in memory (1+GB each)", + Value: eth.DefaultConfig.EthashDatasetsInMem, + } + EthashDatasetsOnDiskFlag = cli.IntFlag{ + Name: "ethash.dagsondisk", + Usage: "Number of recent ethash mining DAGs to keep on disk (1+GB each)", + Value: eth.DefaultConfig.EthashDatasetsOnDisk, + } + // Transaction pool settings + TxPoolPriceLimitFlag = cli.Uint64Flag{ + Name: "txpool.pricelimit", + Usage: "Minimum gas price limit to enforce for acceptance into the pool", + Value: eth.DefaultConfig.TxPool.PriceLimit, + } + TxPoolPriceBumpFlag = cli.Uint64Flag{ + Name: "txpool.pricebump", + Usage: "Price bump percentage to replace an already existing transaction", + Value: eth.DefaultConfig.TxPool.PriceBump, + } + TxPoolAccountSlotsFlag = cli.Uint64Flag{ + Name: "txpool.accountslots", + Usage: "Minimum number of executable transaction slots guaranteed per account", + Value: eth.DefaultConfig.TxPool.AccountSlots, + } + TxPoolGlobalSlotsFlag = cli.Uint64Flag{ + Name: "txpool.globalslots", + Usage: "Maximum number of executable transaction slots for all accounts", + Value: eth.DefaultConfig.TxPool.GlobalSlots, + } + TxPoolAccountQueueFlag = cli.Uint64Flag{ + Name: "txpool.accountqueue", + Usage: "Maximum number of non-executable transaction slots permitted per account", + Value: eth.DefaultConfig.TxPool.AccountQueue, + } + TxPoolGlobalQueueFlag = cli.Uint64Flag{ + Name: "txpool.globalqueue", + Usage: "Maximum number of non-executable transaction slots for all accounts", + Value: eth.DefaultConfig.TxPool.GlobalQueue, + } + TxPoolLifetimeFlag = cli.DurationFlag{ + Name: "txpool.lifetime", + Usage: "Maximum amount of time non-executable transaction are queued", + Value: eth.DefaultConfig.TxPool.Lifetime, + } // Performance tuning settings CacheFlag = cli.IntFlag{ Name: "cache", @@ -229,7 +278,7 @@ var ( GasPriceFlag = BigFlag{ Name: "gasprice", Usage: "Minimal gas price to accept for mining a transactions", - Value: big.NewInt(20 * params.Shannon), + Value: eth.DefaultConfig.GasPrice, } ExtraDataFlag = cli.StringFlag{ Name: "extradata", @@ -327,7 +376,7 @@ var ( } ExecFlag = cli.StringFlag{ Name: "exec", - Usage: "Execute JavaScript statement (only in combination with console/attach)", + Usage: "Execute JavaScript statement", } PreloadJSFlag = cli.StringFlag{ Name: "preload", @@ -352,7 +401,17 @@ var ( } BootnodesFlag = cli.StringFlag{ Name: "bootnodes", - Usage: "Comma separated enode URLs for P2P discovery bootstrap", + Usage: "Comma separated enode URLs for P2P discovery bootstrap (set v4+v5 instead for light servers)", + Value: "", + } + BootnodesV4Flag = cli.StringFlag{ + Name: "bootnodesv4", + Usage: "Comma separated enode URLs for P2P v4 discovery bootstrap (light server, full nodes)", + Value: "", + } + BootnodesV5Flag = cli.StringFlag{ + Name: "bootnodesv5", + Usage: "Comma separated enode URLs for P2P v5 discovery bootstrap (light server, light nodes)", Value: "", } NodeKeyFileFlag = cli.StringFlag{ @@ -411,10 +470,12 @@ var ( // the a subdirectory of the specified datadir will be used. func MakeDataDir(ctx *cli.Context) string { if path := ctx.GlobalString(DataDirFlag.Name); path != "" { - // TODO: choose a different location outside of the regular datadir. - if ctx.GlobalBool(TestNetFlag.Name) { + if ctx.GlobalBool(TestnetFlag.Name) { return filepath.Join(path, "testnet") } + if ctx.GlobalBool(RinkebyFlag.Name) { + return filepath.Join(path, "rinkeby") + } return path } Fatalf("Cannot determine default data directory, please set manually (--datadir)") @@ -458,10 +519,17 @@ func setNodeUserIdent(ctx *cli.Context, cfg *node.Config) { // flags, reverting to pre-configured ones if none have been specified. func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) { urls := params.MainnetBootnodes - if ctx.GlobalIsSet(BootnodesFlag.Name) { - urls = strings.Split(ctx.GlobalString(BootnodesFlag.Name), ",") - } else if ctx.GlobalBool(TestNetFlag.Name) { + switch { + case ctx.GlobalIsSet(BootnodesFlag.Name) || ctx.GlobalIsSet(BootnodesV4Flag.Name): + if ctx.GlobalIsSet(BootnodesV4Flag.Name) { + urls = strings.Split(ctx.GlobalString(BootnodesV4Flag.Name), ",") + } else { + urls = strings.Split(ctx.GlobalString(BootnodesFlag.Name), ",") + } + case ctx.GlobalBool(TestnetFlag.Name): urls = params.TestnetBootnodes + case ctx.GlobalBool(RinkebyFlag.Name): + urls = params.RinkebyBootnodes } cfg.BootstrapNodes = make([]*discover.Node, 0, len(urls)) @@ -479,9 +547,16 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) { // flags, reverting to pre-configured ones if none have been specified. func setBootstrapNodesV5(ctx *cli.Context, cfg *p2p.Config) { urls := params.DiscoveryV5Bootnodes - if ctx.GlobalIsSet(BootnodesFlag.Name) { - urls = strings.Split(ctx.GlobalString(BootnodesFlag.Name), ",") - } else if cfg.BootstrapNodesV5 == nil { + switch { + case ctx.GlobalIsSet(BootnodesFlag.Name) || ctx.GlobalIsSet(BootnodesV5Flag.Name): + if ctx.GlobalIsSet(BootnodesV5Flag.Name) { + urls = strings.Split(ctx.GlobalString(BootnodesV5Flag.Name), ",") + } else { + urls = strings.Split(ctx.GlobalString(BootnodesFlag.Name), ",") + } + case ctx.GlobalBool(RinkebyFlag.Name): + urls = params.RinkebyV5Bootnodes + case cfg.BootstrapNodesV5 != nil: return // already set, don't apply defaults. } @@ -643,7 +718,7 @@ func setEtherbase(ctx *cli.Context, ks *keystore.KeyStore, cfg *eth.Config) { } } -// MakePasswordList reads password lines from the file specified by --password. +// MakePasswordList reads password lines from the file specified by the global --password flag. func MakePasswordList(ctx *cli.Context) []string { path := ctx.GlobalString(PasswordFileFlag.Name) if path == "" { @@ -701,6 +776,7 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) { // --dev mode can't use p2p networking. cfg.MaxPeers = 0 cfg.ListenAddr = ":0" + cfg.DiscoveryV5Addr = ":0" cfg.NoDiscovery = true cfg.DiscoveryV5 = false } @@ -719,8 +795,10 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { cfg.DataDir = ctx.GlobalString(DataDirFlag.Name) case ctx.GlobalBool(DevModeFlag.Name): cfg.DataDir = filepath.Join(os.TempDir(), "ethereum_dev_mode") - case ctx.GlobalBool(TestNetFlag.Name): + case ctx.GlobalBool(TestnetFlag.Name): cfg.DataDir = filepath.Join(node.DefaultDataDir(), "testnet") + case ctx.GlobalBool(RinkebyFlag.Name): + cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby") } if ctx.GlobalIsSet(KeyStoreDirFlag.Name) { @@ -729,6 +807,9 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { if ctx.GlobalIsSet(LightKDFFlag.Name) { cfg.UseLightweightKDF = ctx.GlobalBool(LightKDFFlag.Name) } + if ctx.GlobalIsSet(NoUSBFlag.Name) { + cfg.NoUSB = ctx.GlobalBool(NoUSBFlag.Name) + } } func setGPO(ctx *cli.Context, cfg *gasprice.Config) { @@ -740,6 +821,30 @@ func setGPO(ctx *cli.Context, cfg *gasprice.Config) { } } +func setTxPool(ctx *cli.Context, cfg *core.TxPoolConfig) { + if ctx.GlobalIsSet(TxPoolPriceLimitFlag.Name) { + cfg.PriceLimit = ctx.GlobalUint64(TxPoolPriceLimitFlag.Name) + } + if ctx.GlobalIsSet(TxPoolPriceBumpFlag.Name) { + cfg.PriceBump = ctx.GlobalUint64(TxPoolPriceBumpFlag.Name) + } + if ctx.GlobalIsSet(TxPoolAccountSlotsFlag.Name) { + cfg.AccountSlots = ctx.GlobalUint64(TxPoolAccountSlotsFlag.Name) + } + if ctx.GlobalIsSet(TxPoolGlobalSlotsFlag.Name) { + cfg.GlobalSlots = ctx.GlobalUint64(TxPoolGlobalSlotsFlag.Name) + } + if ctx.GlobalIsSet(TxPoolAccountQueueFlag.Name) { + cfg.AccountQueue = ctx.GlobalUint64(TxPoolAccountQueueFlag.Name) + } + if ctx.GlobalIsSet(TxPoolGlobalQueueFlag.Name) { + cfg.GlobalQueue = ctx.GlobalUint64(TxPoolGlobalQueueFlag.Name) + } + if ctx.GlobalIsSet(TxPoolLifetimeFlag.Name) { + cfg.Lifetime = ctx.GlobalDuration(TxPoolLifetimeFlag.Name) + } +} + func setEthash(ctx *cli.Context, cfg *eth.Config) { if ctx.GlobalIsSet(EthashCacheDirFlag.Name) { cfg.EthashCacheDir = ctx.GlobalString(EthashCacheDirFlag.Name) @@ -776,12 +881,13 @@ func checkExclusive(ctx *cli.Context, flags ...cli.Flag) { // SetEthConfig applies eth-related command line flags to the config. func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { // Avoid conflicting network flags - checkExclusive(ctx, DevModeFlag, TestNetFlag) + checkExclusive(ctx, DevModeFlag, TestnetFlag, RinkebyFlag) checkExclusive(ctx, FastSyncFlag, LightModeFlag, SyncModeFlag) ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) setEtherbase(ctx, ks, cfg) setGPO(ctx, &cfg.GPO) + setTxPool(ctx, &cfg.TxPool) setEthash(ctx, cfg) switch { @@ -799,7 +905,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { cfg.LightPeers = ctx.GlobalInt(LightPeersFlag.Name) } if ctx.GlobalIsSet(NetworkIdFlag.Name) { - cfg.NetworkId = ctx.GlobalInt(NetworkIdFlag.Name) + cfg.NetworkId = ctx.GlobalUint64(NetworkIdFlag.Name) } // Ethereum needs to know maxPeers to calculate the light server peer ratio. @@ -828,13 +934,18 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { cfg.EnablePreimageRecording = ctx.GlobalBool(VMEnableDebugFlag.Name) } - // Override any default configs for --dev and --testnet. + // Override any default configs for hard coded networks. switch { - case ctx.GlobalBool(TestNetFlag.Name): + case ctx.GlobalBool(TestnetFlag.Name): if !ctx.GlobalIsSet(NetworkIdFlag.Name) { cfg.NetworkId = 3 } cfg.Genesis = core.DefaultTestnetGenesisBlock() + case ctx.GlobalBool(RinkebyFlag.Name): + if !ctx.GlobalIsSet(NetworkIdFlag.Name) { + cfg.NetworkId = 4 + } + cfg.Genesis = core.DefaultRinkebyGenesisBlock() case ctx.GlobalBool(DevModeFlag.Name): cfg.Genesis = core.DevGenesisBlock() if !ctx.GlobalIsSet(GasPriceFlag.Name) { @@ -901,22 +1012,16 @@ func SetupNetwork(ctx *cli.Context) { params.TargetGasLimit = new(big.Int).SetUint64(ctx.GlobalUint64(TargetGasLimitFlag.Name)) } -func ChainDbName(ctx *cli.Context) string { - if ctx.GlobalBool(LightModeFlag.Name) { - return "lightchaindata" - } else { - return "chaindata" - } -} - // MakeChainDatabase open an LevelDB using the flags passed to the client and will hard crash if it fails. func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database { var ( cache = ctx.GlobalInt(CacheFlag.Name) handles = makeDatabaseHandles() - name = ChainDbName(ctx) ) - + name := "chaindata" + if ctx.GlobalBool(LightModeFlag.Name) { + name = "lightchaindata" + } chainDb, err := stack.OpenDatabase(name, cache, handles) if err != nil { Fatalf("Could not open database: %v", err) @@ -927,8 +1032,10 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database { func MakeGenesis(ctx *cli.Context) *core.Genesis { var genesis *core.Genesis switch { - case ctx.GlobalBool(TestNetFlag.Name): + case ctx.GlobalBool(TestnetFlag.Name): genesis = core.DefaultTestnetGenesisBlock() + case ctx.GlobalBool(RinkebyFlag.Name): + genesis = core.DefaultRinkebyGenesisBlock() case ctx.GlobalBool(DevModeFlag.Name): genesis = core.DevGenesisBlock() } @@ -972,3 +1079,27 @@ func MakeConsolePreloads(ctx *cli.Context) []string { } return preloads } + +// MigrateFlags sets the global flag from a local flag when it's set. +// This is a temporary function used for migrating old command/flags to the +// new format. +// +// e.g. geth account new --keystore /tmp/mykeystore --lightkdf +// +// is equivalent after calling this method with: +// +// geth --keystore /tmp/mykeystore --lightkdf account new +// +// This allows the use of the existing configuration functionality. +// When all flags are migrated this function can be removed and the existing +// configuration functionality must be changed that is uses local flags +func MigrateFlags(action func(ctx *cli.Context) error) func(*cli.Context) error { + return func(ctx *cli.Context) error { + for _, name := range ctx.FlagNames() { + if ctx.IsSet(name) { + ctx.GlobalSet(name, ctx.String(name)) + } + } + return action(ctx) + } +} diff --git a/cmd/wnode/main.go b/cmd/wnode/main.go index d13932ac22..ae6683aab1 100644 --- a/cmd/wnode/main.go +++ b/cmd/wnode/main.go @@ -65,7 +65,7 @@ var ( pub *ecdsa.PublicKey asymKey *ecdsa.PrivateKey nodeid *ecdsa.PrivateKey - topic []byte + topic whisper.TopicType asymKeyID string filterID string symPass string @@ -84,7 +84,7 @@ var ( testMode = flag.Bool("test", false, "use of predefined parameters for diagnostics") echoMode = flag.Bool("echo", false, "echo mode: prints some arguments for diagnostics") - argVerbosity = flag.Int("verbosity", int(log.LvlWarn), "log verbosity level") + argVerbosity = flag.Int("verbosity", int(log.LvlError), "log verbosity level") argTTL = flag.Uint("ttl", 30, "time-to-live for messages in seconds") argWorkTime = flag.Uint("work", 5, "work time in seconds") argMaxSize = flag.Int("maxsize", whisper.DefaultMaxMessageLength, "max size of message") @@ -129,7 +129,7 @@ func processArgs() { if err != nil { utils.Fatalf("Failed to parse the topic: %s", err) } - topic = x + topic = whisper.BytesToTopic(x) } if *asymmetricMode && len(*argPub) > 0 { @@ -183,7 +183,7 @@ func initialize() { if *testMode { symPass = "wwww" // ascii code: 0x77777777 - msPassword = "mail server test password" + msPassword = "wwww" } if *bootstrapMode { @@ -307,7 +307,11 @@ func configureNode() { if *asymmetricMode { if len(*argPub) == 0 { s := scanLine("Please enter the peer's public key: ") - pub = crypto.ToECDSAPub(common.FromHex(s)) + b := common.FromHex(s) + if b == nil { + utils.Fatalf("Error: can not convert hexadecimal string") + } + pub = crypto.ToECDSAPub(b) if !isKeyValid(pub) { utils.Fatalf("Error: invalid public key") } @@ -326,7 +330,7 @@ func configureNode() { if !*asymmetricMode && !*forwarderMode { if len(symPass) == 0 { - symPass, err = console.Stdin.PromptPassword("Please enter the password: ") + symPass, err = console.Stdin.PromptPassword("Please enter the password for symmetric encryption: ") if err != nil { utils.Fatalf("Failed to read passphrase: %v", err) } @@ -343,6 +347,8 @@ func configureNode() { if len(*argTopic) == 0 { generateTopic([]byte(symPass)) } + + fmt.Printf("Filter is configured for the topic: %x \n", topic) } if *mailServerMode { @@ -354,18 +360,17 @@ func configureNode() { filter := whisper.Filter{ KeySym: symKey, KeyAsym: asymKey, - Topics: [][]byte{topic}, + Topics: [][]byte{topic[:]}, AllowP2P: p2pAccept, } filterID, err = shh.Subscribe(&filter) if err != nil { utils.Fatalf("Failed to install filter: %s", err) } - fmt.Printf("Filter is configured for the topic: %x \n", topic) } func generateTopic(password []byte) { - x := pbkdf2.Key(password, password, 8196, 128, sha512.New) + x := pbkdf2.Key(password, password, 4096, 128, sha512.New) for i := 0; i < len(x); i++ { topic[i%whisper.TopicLength] ^= x[i] } @@ -485,16 +490,15 @@ func sendMsg(payload []byte) common.Hash { Dst: pub, KeySym: symKey, Payload: payload, - Topic: whisper.BytesToTopic(topic), + Topic: topic, TTL: uint32(*argTTL), PoW: *argPoW, WorkTime: uint32(*argWorkTime), } - msg := whisper.NewSentMessage(¶ms) - if msg == nil { - fmt.Printf("failed to create new message (OS level error)") - os.Exit(0) + msg, err := whisper.NewSentMessage(¶ms) + if err != nil { + utils.Fatalf("failed to create new message: %s", err) } envelope, err := msg.Wrap(¶ms) if err != nil { @@ -624,9 +628,9 @@ func requestExpiredMessagesLoop() { params.Src = nodeid params.WorkTime = 5 - msg := whisper.NewSentMessage(¶ms) - if msg == nil { - utils.Fatalf("failed to create new message (OS level error)") + msg, err := whisper.NewSentMessage(¶ms) + if err != nil { + utils.Fatalf("failed to create new message: %s", err) } env, err := msg.Wrap(¶ms) if err != nil { diff --git a/common/bitutil/bitutil.go b/common/bitutil/bitutil.go new file mode 100644 index 0000000000..117616543d --- /dev/null +++ b/common/bitutil/bitutil.go @@ -0,0 +1,188 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Adapted from: https://golang.org/src/crypto/cipher/xor.go + +// Package bitutil implements fast bitwise operations. +package bitutil + +import ( + "runtime" + "unsafe" +) + +const wordSize = int(unsafe.Sizeof(uintptr(0))) +const supportsUnaligned = runtime.GOARCH == "386" || runtime.GOARCH == "amd64" || runtime.GOARCH == "ppc64" || runtime.GOARCH == "ppc64le" || runtime.GOARCH == "s390x" + +// XORBytes xors the bytes in a and b. The destination is assumed to have enough +// space. Returns the number of bytes xor'd. +func XORBytes(dst, a, b []byte) int { + if supportsUnaligned { + return fastXORBytes(dst, a, b) + } + return safeXORBytes(dst, a, b) +} + +// fastXORBytes xors in bulk. It only works on architectures that support +// unaligned read/writes. +func fastXORBytes(dst, a, b []byte) int { + n := len(a) + if len(b) < n { + n = len(b) + } + w := n / wordSize + if w > 0 { + dw := *(*[]uintptr)(unsafe.Pointer(&dst)) + aw := *(*[]uintptr)(unsafe.Pointer(&a)) + bw := *(*[]uintptr)(unsafe.Pointer(&b)) + for i := 0; i < w; i++ { + dw[i] = aw[i] ^ bw[i] + } + } + for i := (n - n%wordSize); i < n; i++ { + dst[i] = a[i] ^ b[i] + } + return n +} + +// safeXORBytes xors one by one. It works on all architectures, independent if +// it supports unaligned read/writes or not. +func safeXORBytes(dst, a, b []byte) int { + n := len(a) + if len(b) < n { + n = len(b) + } + for i := 0; i < n; i++ { + dst[i] = a[i] ^ b[i] + } + return n +} + +// ANDBytes ands the bytes in a and b. The destination is assumed to have enough +// space. Returns the number of bytes and'd. +func ANDBytes(dst, a, b []byte) int { + if supportsUnaligned { + return fastANDBytes(dst, a, b) + } + return safeANDBytes(dst, a, b) +} + +// fastANDBytes ands in bulk. It only works on architectures that support +// unaligned read/writes. +func fastANDBytes(dst, a, b []byte) int { + n := len(a) + if len(b) < n { + n = len(b) + } + w := n / wordSize + if w > 0 { + dw := *(*[]uintptr)(unsafe.Pointer(&dst)) + aw := *(*[]uintptr)(unsafe.Pointer(&a)) + bw := *(*[]uintptr)(unsafe.Pointer(&b)) + for i := 0; i < w; i++ { + dw[i] = aw[i] & bw[i] + } + } + for i := (n - n%wordSize); i < n; i++ { + dst[i] = a[i] & b[i] + } + return n +} + +// safeANDBytes ands one by one. It works on all architectures, independent if +// it supports unaligned read/writes or not. +func safeANDBytes(dst, a, b []byte) int { + n := len(a) + if len(b) < n { + n = len(b) + } + for i := 0; i < n; i++ { + dst[i] = a[i] & b[i] + } + return n +} + +// ORBytes ors the bytes in a and b. The destination is assumed to have enough +// space. Returns the number of bytes or'd. +func ORBytes(dst, a, b []byte) int { + if supportsUnaligned { + return fastORBytes(dst, a, b) + } + return safeORBytes(dst, a, b) +} + +// fastORBytes ors in bulk. It only works on architectures that support +// unaligned read/writes. +func fastORBytes(dst, a, b []byte) int { + n := len(a) + if len(b) < n { + n = len(b) + } + w := n / wordSize + if w > 0 { + dw := *(*[]uintptr)(unsafe.Pointer(&dst)) + aw := *(*[]uintptr)(unsafe.Pointer(&a)) + bw := *(*[]uintptr)(unsafe.Pointer(&b)) + for i := 0; i < w; i++ { + dw[i] = aw[i] | bw[i] + } + } + for i := (n - n%wordSize); i < n; i++ { + dst[i] = a[i] | b[i] + } + return n +} + +// safeORBytes ors one by one. It works on all architectures, independent if +// it supports unaligned read/writes or not. +func safeORBytes(dst, a, b []byte) int { + n := len(a) + if len(b) < n { + n = len(b) + } + for i := 0; i < n; i++ { + dst[i] = a[i] | b[i] + } + return n +} + +// TestBytes tests whether any bit is set in the input byte slice. +func TestBytes(p []byte) bool { + if supportsUnaligned { + return fastTestBytes(p) + } + return safeTestBytes(p) +} + +// fastTestBytes tests for set bits in bulk. It only works on architectures that +// support unaligned read/writes. +func fastTestBytes(p []byte) bool { + n := len(p) + w := n / wordSize + if w > 0 { + pw := *(*[]uintptr)(unsafe.Pointer(&p)) + for i := 0; i < w; i++ { + if pw[i] != 0 { + return true + } + } + } + for i := (n - n%wordSize); i < n; i++ { + if p[i] != 0 { + return true + } + } + return false +} + +// safeTestBytes tests for set bits one byte at a time. It works on all +// architectures, independent if it supports unaligned read/writes or not. +func safeTestBytes(p []byte) bool { + for i := 0; i < len(p); i++ { + if p[i] != 0 { + return true + } + } + return false +} diff --git a/common/bitutil/bitutil_test.go b/common/bitutil/bitutil_test.go new file mode 100644 index 0000000000..93647031ef --- /dev/null +++ b/common/bitutil/bitutil_test.go @@ -0,0 +1,215 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Adapted from: https://golang.org/src/crypto/cipher/xor_test.go + +package bitutil + +import ( + "bytes" + "testing" +) + +// Tests that bitwise XOR works for various alignments. +func TestXOR(t *testing.T) { + for alignP := 0; alignP < 2; alignP++ { + for alignQ := 0; alignQ < 2; alignQ++ { + for alignD := 0; alignD < 2; alignD++ { + p := make([]byte, 1023)[alignP:] + q := make([]byte, 1023)[alignQ:] + + for i := 0; i < len(p); i++ { + p[i] = byte(i) + } + for i := 0; i < len(q); i++ { + q[i] = byte(len(q) - i) + } + d1 := make([]byte, 1023+alignD)[alignD:] + d2 := make([]byte, 1023+alignD)[alignD:] + + XORBytes(d1, p, q) + safeXORBytes(d2, p, q) + if !bytes.Equal(d1, d2) { + t.Error("not equal", d1, d2) + } + } + } + } +} + +// Tests that bitwise AND works for various alignments. +func TestAND(t *testing.T) { + for alignP := 0; alignP < 2; alignP++ { + for alignQ := 0; alignQ < 2; alignQ++ { + for alignD := 0; alignD < 2; alignD++ { + p := make([]byte, 1023)[alignP:] + q := make([]byte, 1023)[alignQ:] + + for i := 0; i < len(p); i++ { + p[i] = byte(i) + } + for i := 0; i < len(q); i++ { + q[i] = byte(len(q) - i) + } + d1 := make([]byte, 1023+alignD)[alignD:] + d2 := make([]byte, 1023+alignD)[alignD:] + + ANDBytes(d1, p, q) + safeANDBytes(d2, p, q) + if !bytes.Equal(d1, d2) { + t.Error("not equal") + } + } + } + } +} + +// Tests that bitwise OR works for various alignments. +func TestOR(t *testing.T) { + for alignP := 0; alignP < 2; alignP++ { + for alignQ := 0; alignQ < 2; alignQ++ { + for alignD := 0; alignD < 2; alignD++ { + p := make([]byte, 1023)[alignP:] + q := make([]byte, 1023)[alignQ:] + + for i := 0; i < len(p); i++ { + p[i] = byte(i) + } + for i := 0; i < len(q); i++ { + q[i] = byte(len(q) - i) + } + d1 := make([]byte, 1023+alignD)[alignD:] + d2 := make([]byte, 1023+alignD)[alignD:] + + ORBytes(d1, p, q) + safeORBytes(d2, p, q) + if !bytes.Equal(d1, d2) { + t.Error("not equal") + } + } + } + } +} + +// Tests that bit testing works for various alignments. +func TestTest(t *testing.T) { + for align := 0; align < 2; align++ { + // Test for bits set in the bulk part + p := make([]byte, 1023)[align:] + p[100] = 1 + + if TestBytes(p) != safeTestBytes(p) { + t.Error("not equal") + } + // Test for bits set in the tail part + q := make([]byte, 1023)[align:] + q[len(q)-1] = 1 + + if TestBytes(q) != safeTestBytes(q) { + t.Error("not equal") + } + } +} + +// Benchmarks the potentially optimized XOR performance. +func BenchmarkFastXOR1KB(b *testing.B) { benchmarkFastXOR(b, 1024) } +func BenchmarkFastXOR2KB(b *testing.B) { benchmarkFastXOR(b, 2048) } +func BenchmarkFastXOR4KB(b *testing.B) { benchmarkFastXOR(b, 4096) } + +func benchmarkFastXOR(b *testing.B, size int) { + p, q := make([]byte, size), make([]byte, size) + + for i := 0; i < b.N; i++ { + XORBytes(p, p, q) + } +} + +// Benchmarks the baseline XOR performance. +func BenchmarkBaseXOR1KB(b *testing.B) { benchmarkBaseXOR(b, 1024) } +func BenchmarkBaseXOR2KB(b *testing.B) { benchmarkBaseXOR(b, 2048) } +func BenchmarkBaseXOR4KB(b *testing.B) { benchmarkBaseXOR(b, 4096) } + +func benchmarkBaseXOR(b *testing.B, size int) { + p, q := make([]byte, size), make([]byte, size) + + for i := 0; i < b.N; i++ { + safeXORBytes(p, p, q) + } +} + +// Benchmarks the potentially optimized AND performance. +func BenchmarkFastAND1KB(b *testing.B) { benchmarkFastAND(b, 1024) } +func BenchmarkFastAND2KB(b *testing.B) { benchmarkFastAND(b, 2048) } +func BenchmarkFastAND4KB(b *testing.B) { benchmarkFastAND(b, 4096) } + +func benchmarkFastAND(b *testing.B, size int) { + p, q := make([]byte, size), make([]byte, size) + + for i := 0; i < b.N; i++ { + ANDBytes(p, p, q) + } +} + +// Benchmarks the baseline AND performance. +func BenchmarkBaseAND1KB(b *testing.B) { benchmarkBaseAND(b, 1024) } +func BenchmarkBaseAND2KB(b *testing.B) { benchmarkBaseAND(b, 2048) } +func BenchmarkBaseAND4KB(b *testing.B) { benchmarkBaseAND(b, 4096) } + +func benchmarkBaseAND(b *testing.B, size int) { + p, q := make([]byte, size), make([]byte, size) + + for i := 0; i < b.N; i++ { + safeANDBytes(p, p, q) + } +} + +// Benchmarks the potentially optimized OR performance. +func BenchmarkFastOR1KB(b *testing.B) { benchmarkFastOR(b, 1024) } +func BenchmarkFastOR2KB(b *testing.B) { benchmarkFastOR(b, 2048) } +func BenchmarkFastOR4KB(b *testing.B) { benchmarkFastOR(b, 4096) } + +func benchmarkFastOR(b *testing.B, size int) { + p, q := make([]byte, size), make([]byte, size) + + for i := 0; i < b.N; i++ { + ORBytes(p, p, q) + } +} + +// Benchmarks the baseline OR performance. +func BenchmarkBaseOR1KB(b *testing.B) { benchmarkBaseOR(b, 1024) } +func BenchmarkBaseOR2KB(b *testing.B) { benchmarkBaseOR(b, 2048) } +func BenchmarkBaseOR4KB(b *testing.B) { benchmarkBaseOR(b, 4096) } + +func benchmarkBaseOR(b *testing.B, size int) { + p, q := make([]byte, size), make([]byte, size) + + for i := 0; i < b.N; i++ { + safeORBytes(p, p, q) + } +} + +// Benchmarks the potentially optimized bit testing performance. +func BenchmarkFastTest1KB(b *testing.B) { benchmarkFastTest(b, 1024) } +func BenchmarkFastTest2KB(b *testing.B) { benchmarkFastTest(b, 2048) } +func BenchmarkFastTest4KB(b *testing.B) { benchmarkFastTest(b, 4096) } + +func benchmarkFastTest(b *testing.B, size int) { + p := make([]byte, size) + for i := 0; i < b.N; i++ { + TestBytes(p) + } +} + +// Benchmarks the baseline bit testing performance. +func BenchmarkBaseTest1KB(b *testing.B) { benchmarkBaseTest(b, 1024) } +func BenchmarkBaseTest2KB(b *testing.B) { benchmarkBaseTest(b, 2048) } +func BenchmarkBaseTest4KB(b *testing.B) { benchmarkBaseTest(b, 4096) } + +func benchmarkBaseTest(b *testing.B, size int) { + p := make([]byte, size) + for i := 0; i < b.N; i++ { + safeTestBytes(p) + } +} diff --git a/common/bitutil/compress.go b/common/bitutil/compress.go new file mode 100644 index 0000000000..c057cee4a6 --- /dev/null +++ b/common/bitutil/compress.go @@ -0,0 +1,170 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package bitutil + +import "errors" + +var ( + // errMissingData is returned from decompression if the byte referenced by + // the bitset header overflows the input data. + errMissingData = errors.New("missing bytes on input") + + // errUnreferencedData is returned from decompression if not all bytes were used + // up from the input data after decompressing it. + errUnreferencedData = errors.New("extra bytes on input") + + // errExceededTarget is returned from decompression if the bitset header has + // more bits defined than the number of target buffer space available. + errExceededTarget = errors.New("target data size exceeded") + + // errZeroContent is returned from decompression if a data byte referenced in + // the bitset header is actually a zero byte. + errZeroContent = errors.New("zero byte in input content") +) + +// The compression algorithm implemented by CompressBytes and DecompressBytes is +// optimized for sparse input data which contains a lot of zero bytes. Decompression +// requires knowledge of the decompressed data length. +// +// Compression works as follows: +// +// if data only contains zeroes, +// CompressBytes(data) == nil +// otherwise if len(data) <= 1, +// CompressBytes(data) == data +// otherwise: +// CompressBytes(data) == append(CompressBytes(nonZeroBitset(data)), nonZeroBytes(data)...) +// where +// nonZeroBitset(data) is a bit vector with len(data) bits (MSB first): +// nonZeroBitset(data)[i/8] && (1 << (7-i%8)) != 0 if data[i] != 0 +// len(nonZeroBitset(data)) == (len(data)+7)/8 +// nonZeroBytes(data) contains the non-zero bytes of data in the same order + +// CompressBytes compresses the input byte slice according to the sparse bitset +// representation algorithm. If the result is bigger than the original input, no +// compression is done. +func CompressBytes(data []byte) []byte { + if out := bitsetEncodeBytes(data); len(out) < len(data) { + return out + } + cpy := make([]byte, len(data)) + copy(cpy, data) + return cpy +} + +// bitsetEncodeBytes compresses the input byte slice according to the sparse +// bitset representation algorithm. +func bitsetEncodeBytes(data []byte) []byte { + // Empty slices get compressed to nil + if len(data) == 0 { + return nil + } + // One byte slices compress to nil or retain the single byte + if len(data) == 1 { + if data[0] == 0 { + return nil + } + return data + } + // Calculate the bitset of set bytes, and gather the non-zero bytes + nonZeroBitset := make([]byte, (len(data)+7)/8) + nonZeroBytes := make([]byte, 0, len(data)) + + for i, b := range data { + if b != 0 { + nonZeroBytes = append(nonZeroBytes, b) + nonZeroBitset[i/8] |= 1 << byte(7-i%8) + } + } + if len(nonZeroBytes) == 0 { + return nil + } + return append(bitsetEncodeBytes(nonZeroBitset), nonZeroBytes...) +} + +// DecompressBytes decompresses data with a known target size. If the input data +// matches the size of the target, it means no compression was done in the first +// place. +func DecompressBytes(data []byte, target int) ([]byte, error) { + if len(data) > target { + return nil, errExceededTarget + } + if len(data) == target { + cpy := make([]byte, len(data)) + copy(cpy, data) + return cpy, nil + } + return bitsetDecodeBytes(data, target) +} + +// bitsetDecodeBytes decompresses data with a known target size. +func bitsetDecodeBytes(data []byte, target int) ([]byte, error) { + out, size, err := bitsetDecodePartialBytes(data, target) + if err != nil { + return nil, err + } + if size != len(data) { + return nil, errUnreferencedData + } + return out, nil +} + +// bitsetDecodePartialBytes decompresses data with a known target size, but does +// not enforce consuming all the input bytes. In addition to the decompressed +// output, the function returns the length of compressed input data corresponding +// to the output as the input slice may be longer. +func bitsetDecodePartialBytes(data []byte, target int) ([]byte, int, error) { + // Sanity check 0 targets to avoid infinite recursion + if target == 0 { + return nil, 0, nil + } + // Handle the zero and single byte corner cases + decomp := make([]byte, target) + if len(data) == 0 { + return decomp, 0, nil + } + if target == 1 { + decomp[0] = data[0] // copy to avoid referencing the input slice + if data[0] != 0 { + return decomp, 1, nil + } + return decomp, 0, nil + } + // Decompress the bitset of set bytes and distribute the non zero bytes + nonZeroBitset, ptr, err := bitsetDecodePartialBytes(data, (target+7)/8) + if err != nil { + return nil, ptr, err + } + for i := 0; i < 8*len(nonZeroBitset); i++ { + if nonZeroBitset[i/8]&(1<= len(data) { + return nil, 0, errMissingData + } + if i >= len(decomp) { + return nil, 0, errExceededTarget + } + // Make sure the data is valid and push into the slot + if data[ptr] == 0 { + return nil, 0, errZeroContent + } + decomp[i] = data[ptr] + ptr++ + } + } + return decomp, ptr, nil +} diff --git a/common/bitutil/compress_fuzz.go b/common/bitutil/compress_fuzz.go new file mode 100644 index 0000000000..1b87f50edc --- /dev/null +++ b/common/bitutil/compress_fuzz.go @@ -0,0 +1,56 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build gofuzz + +package bitutil + +import "bytes" + +// Fuzz implements a go-fuzz fuzzer method to test various encoding method +// invocations. +func Fuzz(data []byte) int { + if len(data) == 0 { + return -1 + } + if data[0]%2 == 0 { + return fuzzEncode(data[1:]) + } + return fuzzDecode(data[1:]) +} + +// fuzzEncode implements a go-fuzz fuzzer method to test the bitset encoding and +// decoding algorithm. +func fuzzEncode(data []byte) int { + proc, _ := bitsetDecodeBytes(bitsetEncodeBytes(data), len(data)) + if !bytes.Equal(data, proc) { + panic("content mismatch") + } + return 0 +} + +// fuzzDecode implements a go-fuzz fuzzer method to test the bit decoding and +// reencoding algorithm. +func fuzzDecode(data []byte) int { + blob, err := bitsetDecodeBytes(data, 1024) + if err != nil { + return 0 + } + if comp := bitsetEncodeBytes(blob); !bytes.Equal(comp, data) { + panic("content mismatch") + } + return 0 +} diff --git a/common/bitutil/compress_test.go b/common/bitutil/compress_test.go new file mode 100644 index 0000000000..67326c2a2b --- /dev/null +++ b/common/bitutil/compress_test.go @@ -0,0 +1,181 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package bitutil + +import ( + "bytes" + "math/rand" + "testing" + + "github.com/expanse-org/go-expanse/common/hexutil" +) + +// Tests that data bitset encoding and decoding works and is bijective. +func TestEncodingCycle(t *testing.T) { + tests := []string{ + // Tests generated by go-fuzz to maximize code coverage + "0x000000000000000000", + "0xef0400", + "0xdf7070533534333636313639343638373532313536346c1bc33339343837313070706336343035336336346c65fefb3930393233383838ac2f65fefb", + "0x7b64000000", + "0x000034000000000000", + "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f0000000000000000000", + "0x4912385c0e7b64000000", + "0x000034000000000000000000000000000000", + "0x00", + "0x000003e834ff7f0000", + "0x0000", + "0x0000000000000000000000000000000000000000000000000000000000ff00", + "0x895f0c6a020f850c6a020f85f88df88d", + "0xdf7070533534333636313639343638373432313536346c1bc3315aac2f65fefb", + "0x0000000000", + "0xdf70706336346c65fefb", + "0x00006d643634000000", + "0xdf7070533534333636313639343638373532313536346c1bc333393438373130707063363430353639343638373532313536346c1bc333393438336336346c65fe", + } + for i, tt := range tests { + data := hexutil.MustDecode(tt) + + proc, err := bitsetDecodeBytes(bitsetEncodeBytes(data), len(data)) + if err != nil { + t.Errorf("test %d: failed to decompress compressed data: %v", i, err) + continue + } + if !bytes.Equal(data, proc) { + t.Errorf("test %d: compress/decompress mismatch: have %x, want %x", i, proc, data) + } + } +} + +// Tests that data bitset decoding and rencoding works and is bijective. +func TestDecodingCycle(t *testing.T) { + tests := []struct { + size int + input string + fail error + }{ + {size: 0, input: "0x"}, + + // Crashers generated by go-fuzz + {size: 0, input: "0x0020", fail: errUnreferencedData}, + {size: 0, input: "0x30", fail: errUnreferencedData}, + {size: 1, input: "0x00", fail: errUnreferencedData}, + {size: 2, input: "0x07", fail: errMissingData}, + {size: 1024, input: "0x8000", fail: errZeroContent}, + + // Tests generated by go-fuzz to maximize code coverage + {size: 29490, input: "0x343137343733323134333839373334323073333930783e3078333930783e70706336346c65303e", fail: errMissingData}, + {size: 59395, input: "0x00", fail: errUnreferencedData}, + {size: 52574, input: "0x70706336346c65c0de", fail: errExceededTarget}, + {size: 42264, input: "0x07", fail: errMissingData}, + {size: 52, input: "0xa5045bad48f4", fail: errExceededTarget}, + {size: 52574, input: "0xc0de", fail: errMissingData}, + {size: 52574, input: "0x"}, + {size: 29490, input: "0x34313734373332313433383937333432307333393078073034333839373334323073333930783e3078333937333432307333393078073061333930783e70706336346c65303e", fail: errMissingData}, + {size: 29491, input: "0x3973333930783e30783e", fail: errMissingData}, + + {size: 1024, input: "0x808080608080"}, + {size: 1024, input: "0x808470705e3632383337363033313434303137393130306c6580ef46806380635a80"}, + {size: 1024, input: "0x8080808070"}, + {size: 1024, input: "0x808070705e36346c6580ef46806380635a80"}, + {size: 1024, input: "0x80808046802680"}, + {size: 1024, input: "0x4040404035"}, + {size: 1024, input: "0x4040bf3ba2b3f684402d353234373438373934409fe5b1e7ada94ebfd7d0505e27be4035"}, + {size: 1024, input: "0x404040bf3ba2b3f6844035"}, + {size: 1024, input: "0x40402d35323437343837393440bfd7d0505e27be4035"}, + } + for i, tt := range tests { + data := hexutil.MustDecode(tt.input) + + orig, err := bitsetDecodeBytes(data, tt.size) + if err != tt.fail { + t.Errorf("test %d: failure mismatch: have %v, want %v", i, err, tt.fail) + } + if err != nil { + continue + } + if comp := bitsetEncodeBytes(orig); !bytes.Equal(comp, data) { + t.Errorf("test %d: decompress/compress mismatch: have %x, want %x", i, comp, data) + } + } +} + +// TestCompression tests that compression works by returning either the bitset +// encoded input, or the actual input if the bitset version is longer. +func TestCompression(t *testing.T) { + // Check the the compression returns the bitset encoding is shorter + in := hexutil.MustDecode("0x4912385c0e7b64000000") + out := hexutil.MustDecode("0x80fe4912385c0e7b64") + + if data := CompressBytes(in); bytes.Compare(data, out) != 0 { + t.Errorf("encoding mismatch for sparse data: have %x, want %x", data, out) + } + if data, err := DecompressBytes(out, len(in)); err != nil || bytes.Compare(data, in) != 0 { + t.Errorf("decoding mismatch for sparse data: have %x, want %x, error %v", data, in, err) + } + // Check the the compression returns the input if the bitset encoding is longer + in = hexutil.MustDecode("0xdf7070533534333636313639343638373532313536346c1bc33339343837313070706336343035336336346c65fefb3930393233383838ac2f65fefb") + out = hexutil.MustDecode("0xdf7070533534333636313639343638373532313536346c1bc33339343837313070706336343035336336346c65fefb3930393233383838ac2f65fefb") + + if data := CompressBytes(in); bytes.Compare(data, out) != 0 { + t.Errorf("encoding mismatch for dense data: have %x, want %x", data, out) + } + if data, err := DecompressBytes(out, len(in)); err != nil || bytes.Compare(data, in) != 0 { + t.Errorf("decoding mismatch for dense data: have %x, want %x, error %v", data, in, err) + } + // Check that decompressing a longer input than the target fails + if _, err := DecompressBytes([]byte{0xc0, 0x01, 0x01}, 2); err != errExceededTarget { + t.Errorf("decoding error mismatch for long data: have %v, want %v", err, errExceededTarget) + } +} + +// Crude benchmark for compressing random slices of bytes. +func BenchmarkEncoding1KBVerySparse(b *testing.B) { benchmarkEncoding(b, 1024, 0.0001) } +func BenchmarkEncoding2KBVerySparse(b *testing.B) { benchmarkEncoding(b, 2048, 0.0001) } +func BenchmarkEncoding4KBVerySparse(b *testing.B) { benchmarkEncoding(b, 4096, 0.0001) } + +func BenchmarkEncoding1KBSparse(b *testing.B) { benchmarkEncoding(b, 1024, 0.001) } +func BenchmarkEncoding2KBSparse(b *testing.B) { benchmarkEncoding(b, 2048, 0.001) } +func BenchmarkEncoding4KBSparse(b *testing.B) { benchmarkEncoding(b, 4096, 0.001) } + +func BenchmarkEncoding1KBDense(b *testing.B) { benchmarkEncoding(b, 1024, 0.1) } +func BenchmarkEncoding2KBDense(b *testing.B) { benchmarkEncoding(b, 2048, 0.1) } +func BenchmarkEncoding4KBDense(b *testing.B) { benchmarkEncoding(b, 4096, 0.1) } + +func BenchmarkEncoding1KBSaturated(b *testing.B) { benchmarkEncoding(b, 1024, 0.5) } +func BenchmarkEncoding2KBSaturated(b *testing.B) { benchmarkEncoding(b, 2048, 0.5) } +func BenchmarkEncoding4KBSaturated(b *testing.B) { benchmarkEncoding(b, 4096, 0.5) } + +func benchmarkEncoding(b *testing.B, bytes int, fill float64) { + // Generate a random slice of bytes to compress + random := rand.NewSource(0) // reproducible and comparable + + data := make([]byte, bytes) + bits := int(float64(bytes) * 8 * fill) + + for i := 0; i < bits; i++ { + idx := random.Int63() % int64(len(data)) + bit := uint(random.Int63() % 8) + data[idx] |= 1 << bit + } + // Reset the benchmark and measure encoding/decoding + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + bitsetDecodeBytes(bitsetEncodeBytes(data), len(data)) + } +} diff --git a/common/bytes.go b/common/bytes.go index 0342083a1e..c445968f22 100644 --- a/common/bytes.go +++ b/common/bytes.go @@ -89,18 +89,18 @@ func Hex2BytesFixed(str string, flen int) []byte { } func RightPadBytes(slice []byte, l int) []byte { - if l < len(slice) { + if l <= len(slice) { return slice } padded := make([]byte, l) - copy(padded[0:len(slice)], slice) + copy(padded, slice) return padded } func LeftPadBytes(slice []byte, l int) []byte { - if l < len(slice) { + if l <= len(slice) { return slice } diff --git a/common/math/big.go b/common/math/big.go index 5255a88e9f..fd0174b366 100644 --- a/common/math/big.go +++ b/common/math/big.go @@ -27,6 +27,8 @@ var ( tt256 = BigPow(2, 256) tt256m1 = new(big.Int).Sub(tt256, big.NewInt(1)) MaxBig256 = new(big.Int).Set(tt256m1) + tt63 = BigPow(2, 63) + MaxBig63 = new(big.Int).Sub(tt63, big.NewInt(1)) ) const ( diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index 780e2a0018..a576a0f41f 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -44,7 +44,7 @@ import ( const ( checkpointInterval = 1024 // Number of blocks after which to save the vote snapshot to the database inmemorySnapshots = 128 // Number of recent vote snapshots to keep in memory - inmemorySignatures = 1024 // Number of recent blocks to keep in memory + inmemorySignatures = 4096 // Number of recent block signatures to keep in memory wiggleTime = 500 * time.Millisecond // Random delay (per signer) to allow concurrent signers ) @@ -162,7 +162,12 @@ func sigHash(header *types.Header) (hash common.Hash) { } // ecrecover extracts the Ethereum account address from a signed header. -func ecrecover(header *types.Header) (common.Address, error) { +func ecrecover(header *types.Header, sigcache *lru.ARCCache) (common.Address, error) { + // If the signature's already cached, return that + hash := header.Hash() + if address, known := sigcache.Get(hash); known { + return address.(common.Address), nil + } // Retrieve the signature from the header extra-data if len(header.Extra) < extraSeal { return common.Address{}, errMissingSignature @@ -177,6 +182,7 @@ func ecrecover(header *types.Header) (common.Address, error) { var signer common.Address copy(signer[:], crypto.Keccak256(pubkey[1:])[12:]) + sigcache.Add(hash, signer) return signer, nil } @@ -223,7 +229,7 @@ func New(config *params.CliqueConfig, db ethdb.Database) *Clique { // Author implements consensus.Engine, returning the Ethereum address recovered // from the signature in the header's extra-data section. func (c *Clique) Author(header *types.Header) (common.Address, error) { - return ecrecover(header) + return ecrecover(header, c.signatures) } // VerifyHeader checks whether a header conforms to the consensus rules. @@ -369,7 +375,7 @@ func (c *Clique) snapshot(chain consensus.ChainReader, number uint64, hash commo } // If an on-disk checkpoint snapshot can be found, use that if number%checkpointInterval == 0 { - if s, err := loadSnapshot(c.config, c.db, hash); err == nil { + if s, err := loadSnapshot(c.config, c.signatures, c.db, hash); err == nil { log.Trace("Loaded voting snapshot form disk", "number", number, "hash", hash) snap = s break @@ -385,7 +391,7 @@ func (c *Clique) snapshot(chain consensus.ChainReader, number uint64, hash commo for i := 0; i < len(signers); i++ { copy(signers[i][:], genesis.Extra[extraVanity+i*common.AddressLength:]) } - snap = newSnapshot(c.config, 0, genesis.Hash(), signers) + snap = newSnapshot(c.config, c.signatures, 0, genesis.Hash(), signers) if err := snap.store(c.db); err != nil { return nil, err } @@ -464,7 +470,7 @@ func (c *Clique) verifySeal(chain consensus.ChainReader, header *types.Header, p c.recents.Add(snap.Hash, snap) // Resolve the authorization key and check against signers - signer, err := ecrecover(header) + signer, err := ecrecover(header, c.signatures) if err != nil { return err } @@ -599,7 +605,7 @@ func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, stop <-ch for seen, recent := range snap.Recents { if recent == signer { // Signer is among recents, only wait if the current block doens't shift it out - if limit := uint64(len(snap.Signers)/2 + 1); seen > number-limit { + if limit := uint64(len(snap.Signers)/2 + 1); number < limit || seen > number-limit { log.Info("Signed recently, must wait for others") <-stop return nil, nil diff --git a/consensus/clique/snapshot.go b/consensus/clique/snapshot.go index 05bae80bc4..99fdf09a79 100644 --- a/consensus/clique/snapshot.go +++ b/consensus/clique/snapshot.go @@ -24,6 +24,7 @@ import ( "github.com/expanse-org/go-expanse/core/types" "github.com/expanse-org/go-expanse/ethdb" "github.com/expanse-org/go-expanse/params" + lru "github.com/hashicorp/golang-lru" ) // Vote represents a single vote that an authorized signer made to modify the @@ -44,7 +45,8 @@ type Tally struct { // Snapshot is the state of the authorization voting at a given point in time. type Snapshot struct { - config *params.CliqueConfig // Consensus engine parameters to fine tune behavior + config *params.CliqueConfig // Consensus engine parameters to fine tune behavior + sigcache *lru.ARCCache // Cache of recent block signatures to speed up ecrecover Number uint64 `json:"number"` // Block number where the snapshot was created Hash common.Hash `json:"hash"` // Block hash where the snapshot was created @@ -57,14 +59,15 @@ type Snapshot struct { // newSnapshot create a new snapshot with the specified startup parameters. This // method does not initialize the set of recent signers, so only ever use if for // the genesis block. -func newSnapshot(config *params.CliqueConfig, number uint64, hash common.Hash, signers []common.Address) *Snapshot { +func newSnapshot(config *params.CliqueConfig, sigcache *lru.ARCCache, number uint64, hash common.Hash, signers []common.Address) *Snapshot { snap := &Snapshot{ - config: config, - Number: number, - Hash: hash, - Signers: make(map[common.Address]struct{}), - Recents: make(map[uint64]common.Address), - Tally: make(map[common.Address]Tally), + config: config, + sigcache: sigcache, + Number: number, + Hash: hash, + Signers: make(map[common.Address]struct{}), + Recents: make(map[uint64]common.Address), + Tally: make(map[common.Address]Tally), } for _, signer := range signers { snap.Signers[signer] = struct{}{} @@ -73,7 +76,7 @@ func newSnapshot(config *params.CliqueConfig, number uint64, hash common.Hash, s } // loadSnapshot loads an existing snapshot from the database. -func loadSnapshot(config *params.CliqueConfig, db ethdb.Database, hash common.Hash) (*Snapshot, error) { +func loadSnapshot(config *params.CliqueConfig, sigcache *lru.ARCCache, db ethdb.Database, hash common.Hash) (*Snapshot, error) { blob, err := db.Get(append([]byte("clique-"), hash[:]...)) if err != nil { return nil, err @@ -83,6 +86,7 @@ func loadSnapshot(config *params.CliqueConfig, db ethdb.Database, hash common.Ha return nil, err } snap.config = config + snap.sigcache = sigcache return snap, nil } @@ -99,13 +103,14 @@ func (s *Snapshot) store(db ethdb.Database) error { // copy creates a deep copy of the snapshot, though not the individual votes. func (s *Snapshot) copy() *Snapshot { cpy := &Snapshot{ - config: s.config, - Number: s.Number, - Hash: s.Hash, - Signers: make(map[common.Address]struct{}), - Recents: make(map[uint64]common.Address), - Votes: make([]*Vote, len(s.Votes)), - Tally: make(map[common.Address]Tally), + config: s.config, + sigcache: s.sigcache, + Number: s.Number, + Hash: s.Hash, + Signers: make(map[common.Address]struct{}), + Recents: make(map[uint64]common.Address), + Votes: make([]*Vote, len(s.Votes)), + Tally: make(map[common.Address]Tally), } for signer := range s.Signers { cpy.Signers[signer] = struct{}{} @@ -190,7 +195,7 @@ func (s *Snapshot) apply(headers []*types.Header) (*Snapshot, error) { delete(snap.Recents, number-limit) } // Resolve the authorization key and check against signers - signer, err := ecrecover(header) + signer, err := ecrecover(header, s.sigcache) if err != nil { return nil, err } diff --git a/consensus/ethash/algorithm.go b/consensus/ethash/algorithm.go index ecbc18d108..a40ce52d5b 100644 --- a/consensus/ethash/algorithm.go +++ b/consensus/ethash/algorithm.go @@ -27,6 +27,7 @@ import ( "unsafe" "github.com/expanse-org/go-expanse/common" + "github.com/expanse-org/go-expanse/common/bitutil" "github.com/expanse-org/go-expanse/crypto" "github.com/expanse-org/go-expanse/crypto/sha3" "github.com/expanse-org/go-expanse/log" @@ -142,7 +143,7 @@ func generateCache(dest []uint32, epoch uint64, seed []byte) { dstOff = j * hashBytes xorOff = (binary.LittleEndian.Uint32(cache[dstOff:]) % uint32(rows)) * hashBytes ) - xorBytes(temp, cache[srcOff:srcOff+hashBytes], cache[xorOff:xorOff+hashBytes]) + bitutil.XORBytes(temp, cache[srcOff:srcOff+hashBytes], cache[xorOff:xorOff+hashBytes]) keccak512(cache[dstOff:], temp) atomic.AddUint32(&progress, 1) diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 9e3853006c..987944bf94 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -239,10 +239,19 @@ func (ethash *Ethash) verifyHeader(chain consensus.ChainReader, header, parent * return errZeroBlockTime } // Verify the block's difficulty based in it's timestamp and parent's difficulty - expected := CalcDifficulty(chain.Config(), header.Time.Uint64(), parent.Time.Uint64(), parent.Number, parent.Difficulty) + expected := CalcDifficulty(chain.Config(), header.Time.Uint64(), parent) if expected.Cmp(header.Difficulty) != 0 { return fmt.Errorf("invalid difficulty: have %v, want %v", header.Difficulty, expected) } + // Verify that the gas limit is <= 2^63-1 + if header.GasLimit.Cmp(math.MaxBig63) > 0 { + return fmt.Errorf("invalid gasLimit: have %v, max %v", header.GasLimit, math.MaxBig63) + } + // Verify that the gasUsed is <= gasLimit + if header.GasUsed.Cmp(header.GasLimit) > 0 { + return fmt.Errorf("invalid gasUsed: have %v, gasLimit %v", header.GasUsed, header.GasLimit) + } + // Verify that the gas limit remains within allowed bounds diff := new(big.Int).Set(parent.GasLimit) diff = diff.Sub(diff, header.GasLimit) @@ -274,16 +283,19 @@ func (ethash *Ethash) verifyHeader(chain consensus.ChainReader, header, parent * return nil } -// CalcDifficulty is the difficulty adjustment algorithm. It returns the difficulty -// that a new block should have when created at time given the parent block's time -// and difficulty. +// CalcDifficulty is the difficulty adjustment algorithm. It returns +// the difficulty that a new block should have when created at time +// given the parent block's time and difficulty. // // TODO (karalabe): Move the chain maker into this package and make this private! -func CalcDifficulty(config *params.ChainConfig, time, parentTime uint64, parentNumber, parentDiff *big.Int) *big.Int { - if config.IsHomestead(new(big.Int).Add(parentNumber, common.Big1)) { - return calcDifficultyHomestead(time, parentTime, parentNumber, parentDiff) +func CalcDifficulty(config *params.ChainConfig, time uint64, parent *types.Header) *big.Int { + next := new(big.Int).Add(parent.Number, common.Big1) + switch { + case config.IsHomestead(next): + return calcDifficultyHomestead(time, parent) + default: + return calcDifficultyFrontier(time, parent) } - return calcDifficultyFrontier(time, parentTime, parentNumber, parentDiff) } // Some weird constants to avoid constant memory allocs for them. @@ -297,7 +309,7 @@ var ( // calcDifficultyHomestead is the difficulty adjustment algorithm. It returns // the difficulty that a new block should have when created at time given the // parent block's time and difficulty. The calculation uses the Homestead rules. -func calcDifficultyHomestead(time, parentTime uint64, parentNumber, parentDiff *big.Int) *big.Int { +func calcDifficultyHomestead(time uint64, parent *types.Header) *big.Int { // https://github.com/ethereum/EIPs/blob/master/EIPS/eip-2.mediawiki // algorithm: // diff = (parent_diff + @@ -305,7 +317,7 @@ func calcDifficultyHomestead(time, parentTime uint64, parentNumber, parentDiff * // ) + 2^(periodCount - 2) bigTime := new(big.Int).SetUint64(time) - bigParentTime := new(big.Int).SetUint64(parentTime) + bigParentTime := new(big.Int).Set(parent.Time) // holds intermediate values to make the algo easier to read & audit x := new(big.Int) @@ -321,10 +333,10 @@ func calcDifficultyHomestead(time, parentTime uint64, parentNumber, parentDiff * x.Set(bigMinus99) } - // (parent_diff + parent_diff // 512 * max(1 - (block_timestamp - parent_timestamp) // 10, -99)) - y.Div(parentDiff, params.DifficultyBoundDivisor2) + // (parent_diff + parent_diff // 2048 * max(1 - (block_timestamp - parent_timestamp) // 10, -99)) + y.Div(parent.Difficulty, params.DifficultyBoundDivisor2) x.Mul(y, x) - x.Add(parentDiff, x) + x.Add(parent.Difficulty, x) // minimum difficulty can ever be (before exponential factor) if x.Cmp(params.MinimumDifficulty) < 0 { @@ -337,42 +349,40 @@ func calcDifficultyHomestead(time, parentTime uint64, parentNumber, parentDiff * // calcDifficultyFrontier is the difficulty adjustment algorithm. It returns the // difficulty that a new block should have when created at time given the parent // block's time and difficulty. The calculation uses the Frontier rules. -func calcDifficultyFrontier(time, parentTime uint64, parentNumber, parentDiff *big.Int) *big.Int { - diff := new(big.Int) - adjust := new(big.Int) +func calcDifficultyFrontier(time uint64, parent *types.Header) *big.Int { + diff := new(big.Int) + adjust := new(big.Int) + bigTime := new(big.Int) + bigParentTime := new(big.Int) - if parentNumber.Cmp(params.HardFork1) < 0 { - adjust = new(big.Int).Div(parentDiff, params.DifficultyBoundDivisor) - } else { - adjust = new(big.Int).Div(parentDiff, params.DifficultyBoundDivisor2) - } + if parent.Number.Cmp(params.HardFork1) < 0 { + adjust = new(big.Int).Div(parent.Difficulty, params.DifficultyBoundDivisor) + } else { + adjust = new(big.Int).Div(parent.Difficulty, params.DifficultyBoundDivisor2) + } - bigTime := new(big.Int) - bigParentTime := new(big.Int) + bigTime.SetUint64(time) + bigParentTime.Set(parent.Time) - bigTime.SetUint64(time) - bigParentTime.SetUint64(parentTime) + if bigTime.Sub(bigTime, bigParentTime).Cmp(params.DurationLimit) < 0 { + diff.Add(parent.Difficulty, adjust) + } else { + diff.Sub(parent.Difficulty, adjust) + } + if diff.Cmp(params.MinimumDifficulty) < 0 { + diff.Set(params.MinimumDifficulty) + } - if bigTime.Sub(bigTime, bigParentTime).Cmp(params.DurationLimit) < 0 { - diff.Add(parentDiff, adjust) - } else { - diff.Sub(parentDiff, adjust) - } - if diff.Cmp(params.MinimumDifficulty) < 0 { - diff.Set(params.MinimumDifficulty) - } - - periodCount := new(big.Int).Add(parentNumber, common.Big1) - periodCount.Div(periodCount, expDiffPeriod) - if periodCount.Cmp(common.Big1) > 0 { - // diff = diff + 2^(periodCount - 2) - expDiff := periodCount.Sub(periodCount, common.Big2) - expDiff.Exp(common.Big2, expDiff, nil) - diff.Add(diff, expDiff) - diff = math.BigMax(diff, params.MinimumDifficulty) - } - - return diff + periodCount := new(big.Int).Add(parent.Number, common.Big1) + periodCount.Div(periodCount, expDiffPeriod) + if periodCount.Cmp(common.Big1) > 0 { + // diff = diff + 2^(periodCount - 2) + expDiff := periodCount.Sub(periodCount, common.Big2) + expDiff.Exp(common.Big2, expDiff, nil) + diff.Add(diff, expDiff) + diff = math.BigMax(diff, params.MinimumDifficulty) + } + return diff } // VerifySeal implements consensus.Engine, checking whether the given block satisfies @@ -425,8 +435,7 @@ func (ethash *Ethash) Prepare(chain consensus.ChainReader, header *types.Header) if parent == nil { return consensus.ErrUnknownAncestor } - header.Difficulty = CalcDifficulty(chain.Config(), header.Time.Uint64(), - parent.Time.Uint64(), parent.Number, parent.Difficulty) + header.Difficulty = CalcDifficulty(chain.Config(), header.Time.Uint64(), parent) return nil } diff --git a/consensus/ethash/consensus_test.go b/consensus/ethash/consensus_test.go index f4a579ef76..4b80ff5c3a 100644 --- a/consensus/ethash/consensus_test.go +++ b/consensus/ethash/consensus_test.go @@ -23,6 +23,7 @@ import ( "testing" "github.com/expanse-org/go-expanse/common/math" + "github.com/expanse-org/go-expanse/core/types" "github.com/expanse-org/go-expanse/params" ) @@ -71,7 +72,11 @@ func TestCalcDifficulty(t *testing.T) { config := ¶ms.ChainConfig{HomesteadBlock: big.NewInt(1150000)} for name, test := range tests { number := new(big.Int).Sub(test.CurrentBlocknumber, big.NewInt(1)) - diff := CalcDifficulty(config, test.CurrentTimestamp, test.ParentTimestamp, number, test.ParentDifficulty) + diff := CalcDifficulty(config, test.CurrentTimestamp, &types.Header{ + Number: number, + Time: new(big.Int).SetUint64(test.ParentTimestamp), + Difficulty: test.ParentDifficulty, + }) if diff.Cmp(test.CurrentDifficulty) != 0 { t.Error(name, "failed. Expected", test.CurrentDifficulty, "and calculated", diff) } diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index a69cfbedaf..6f61ff7952 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -468,8 +468,9 @@ func (ethash *Ethash) cache(block uint64) []uint32 { future = &cache{epoch: epoch + 1} ethash.fcache = future } + // New current cache, set its initial timestamp + current.used = time.Now() } - current.used = time.Now() ethash.lock.Unlock() // Wait for generation finish, bump the timestamp and finalize the cache @@ -530,8 +531,9 @@ func (ethash *Ethash) dataset(block uint64) []uint32 { future = &dataset{epoch: epoch + 1} ethash.fdataset = future } + // New current dataset, set its initial timestamp + current.used = time.Now() } - current.used = time.Now() ethash.lock.Unlock() // Wait for generation finish, bump the timestamp and finalize the cache diff --git a/consensus/ethash/xor.go b/consensus/ethash/xor.go deleted file mode 100644 index 90e2327466..0000000000 --- a/consensus/ethash/xor.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Source: https://golang.org/src/crypto/cipher/xor.go - -package ethash - -import ( - "runtime" - "unsafe" -) - -const wordSize = int(unsafe.Sizeof(uintptr(0))) -const supportsUnaligned = runtime.GOARCH == "386" || runtime.GOARCH == "amd64" || runtime.GOARCH == "ppc64" || runtime.GOARCH == "ppc64le" || runtime.GOARCH == "s390x" - -// fastXORBytes xors in bulk. It only works on architectures that -// support unaligned read/writes. -func fastXORBytes(dst, a, b []byte) int { - n := len(a) - if len(b) < n { - n = len(b) - } - - w := n / wordSize - if w > 0 { - dw := *(*[]uintptr)(unsafe.Pointer(&dst)) - aw := *(*[]uintptr)(unsafe.Pointer(&a)) - bw := *(*[]uintptr)(unsafe.Pointer(&b)) - for i := 0; i < w; i++ { - dw[i] = aw[i] ^ bw[i] - } - } - - for i := (n - n%wordSize); i < n; i++ { - dst[i] = a[i] ^ b[i] - } - - return n -} - -func safeXORBytes(dst, a, b []byte) int { - n := len(a) - if len(b) < n { - n = len(b) - } - for i := 0; i < n; i++ { - dst[i] = a[i] ^ b[i] - } - return n -} - -// xorBytes xors the bytes in a and b. The destination is assumed to have enough -// space. Returns the number of bytes xor'd. -func xorBytes(dst, a, b []byte) int { - if supportsUnaligned { - return fastXORBytes(dst, a, b) - } - // TODO(hanwen): if (dst, a, b) have common alignment - // we could still try fastXORBytes. It is not clear - // how often this happens, and it's only worth it if - // the block encryption itself is hardware - // accelerated. - return safeXORBytes(dst, a, b) -} - -// fastXORWords XORs multiples of 4 or 8 bytes (depending on architecture.) -// The arguments are assumed to be of equal length. -func fastXORWords(dst, a, b []byte) { - dw := *(*[]uintptr)(unsafe.Pointer(&dst)) - aw := *(*[]uintptr)(unsafe.Pointer(&a)) - bw := *(*[]uintptr)(unsafe.Pointer(&b)) - n := len(b) / wordSize - for i := 0; i < n; i++ { - dw[i] = aw[i] ^ bw[i] - } -} - -func xorWords(dst, a, b []byte) { - if supportsUnaligned { - fastXORWords(dst, a, b) - } else { - safeXORBytes(dst, a, b) - } -} diff --git a/console/bridge.go b/console/bridge.go index fd371aeb4a..22a4e6a2ec 100644 --- a/console/bridge.go +++ b/console/bridge.go @@ -20,6 +20,7 @@ import ( "encoding/json" "fmt" "io" + "strings" "time" "github.com/expanse-org/go-expanse/log" @@ -240,17 +241,19 @@ func (b *bridge) Send(call otto.FunctionCall) (response otto.Value) { throwJSException(err.Error()) } var ( - rawReq = []byte(reqVal.String()) + rawReq = reqVal.String() + dec = json.NewDecoder(strings.NewReader(rawReq)) reqs []jsonrpcCall batch bool ) + dec.UseNumber() // avoid float64s if rawReq[0] == '[' { batch = true - json.Unmarshal(rawReq, &reqs) + dec.Decode(&reqs) } else { batch = false reqs = make([]jsonrpcCall, 1) - json.Unmarshal(rawReq, &reqs[0]) + dec.Decode(&reqs[0]) } // Execute the requests. diff --git a/containers/docker/master-alpine/Dockerfile b/containers/docker/master-alpine/Dockerfile index 8c5d17afa7..f7a868323a 100644 --- a/containers/docker/master-alpine/Dockerfile +++ b/containers/docker/master-alpine/Dockerfile @@ -2,7 +2,7 @@ FROM alpine:3.5 RUN \ apk add --update go git make gcc musl-dev linux-headers ca-certificates && \ - git clone --depth 1 --branch release/1.5 https://github.com/expanse-org/go-expanse && \ + git clone --depth 1 --branch release/1.6 https://github.com/expanse-org/go-expanse && \ (cd go-expanse && make gexp) && \ cp go-expanse/build/bin/gexp /gexp && \ apk del go git make gcc musl-dev linux-headers && \ diff --git a/containers/vagrant/.gitignore b/containers/vagrant/.gitignore new file mode 100644 index 0000000000..8000dd9db4 --- /dev/null +++ b/containers/vagrant/.gitignore @@ -0,0 +1 @@ +.vagrant diff --git a/containers/vagrant/Vagrantfile b/containers/vagrant/Vagrantfile index 9452ac443f..c31f09912d 100644 --- a/containers/vagrant/Vagrantfile +++ b/containers/vagrant/Vagrantfile @@ -1,29 +1,38 @@ # -*- mode: ruby -*- # vi: set ft=ruby : -Vagrant.configure(2) do |config| - config.vm.box = "ubuntu/trusty64" +require 'yaml' - config.vm.provider "virtualbox" do |vb| - vb.memory = "2048" +VAGRANTFILE_API_VERSION = 2 +VM_RAM = 2048 + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + + config.vm.define "ubuntu", :primary => true do |ubuntu| + ubuntu.vm.box = "ubuntu/trusty64" + ubuntu.vm.provision "shell", :path => "provisioners/shell/ubuntu.sh" end + config.vm.define "debian", :primary => true do |debian| + debian.vm.box = "debian/jessie64" + debian.vm.provision "shell", :path => "provisioners/shell/debian.sh" + end + + config.vm.define "centos", :autostart => false do |centos| + centos.vm.box = "centos/7" + centos.vm.provision "shell", :path => "provisioners/shell/centos.sh" + end + + config.vm.provider "virtualbox" do |vb| + vb.memory = VM_RAM + end + + config.vm.provider "libvirt" do |lv| + lv.memory = VM_RAM + + config.vm.synced_folder ".", "/home/vagrant/sync", :disabled => true + end + + config.vm.synced_folder ".", "/vagrant", :disabled => true config.vm.synced_folder "../../", "/home/vagrant/go/src/github.com/expanse-org/go-expanse" - config.vm.synced_folder ".", "/vagrant", disabled: true - - config.vm.provision "shell", inline: <<-SHELL - sudo apt-get install software-properties-common - sudo add-apt-repository -y ppa:expanse/expanse - sudo add-apt-repository -y ppa:expanse/expanse-dev - sudo apt-get update - - sudo apt-get install -y build-essential golang git-all - - GOPATH=/home/vagrant/go go get github.com/tools/godep - - sudo chown -R vagrant:vagrant ~vagrant/go - - echo "export GOPATH=/home/vagrant/go" >> ~vagrant/.bashrc - echo "export PATH=\\\$PATH:\\\$GOPATH/bin:/usr/local/go/bin" >> ~vagrant/.bashrc - SHELL end diff --git a/containers/vagrant/provisioners/shell/centos.sh b/containers/vagrant/provisioners/shell/centos.sh new file mode 100755 index 0000000000..df04bb22e0 --- /dev/null +++ b/containers/vagrant/provisioners/shell/centos.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +sudo yum install -y git wget +sudo yum update -y + +wget --continue https://storage.googleapis.com/golang/go1.8.1.linux-amd64.tar.gz +sudo tar -C /usr/local -xzf go1.8.1.linux-amd64.tar.gz + +GETH_PATH="~vagrant/go/src/github.com/expanse-org/go-expanse/build/bin/" + +echo "export PATH=$PATH:/usr/local/go/bin:$GETH_PATH" >> ~vagrant/.bashrc diff --git a/containers/vagrant/provisioners/shell/debian.sh b/containers/vagrant/provisioners/shell/debian.sh new file mode 100755 index 0000000000..462f43df56 --- /dev/null +++ b/containers/vagrant/provisioners/shell/debian.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +sudo apt-get install -y build-essential git-all wget +sudo apt-get update + +wget --continue https://storage.googleapis.com/golang/go1.8.1.linux-amd64.tar.gz +sudo tar -C /usr/local -xzf go1.8.1.linux-amd64.tar.gz + +GETH_PATH="~vagrant/go/src/github.com/expanse-org/go-expanse/build/bin/" + +echo "export PATH=$PATH:/usr/local/go/bin:$GETH_PATH" >> ~vagrant/.bashrc diff --git a/containers/vagrant/provisioners/shell/ubuntu.sh b/containers/vagrant/provisioners/shell/ubuntu.sh new file mode 100755 index 0000000000..462f43df56 --- /dev/null +++ b/containers/vagrant/provisioners/shell/ubuntu.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +sudo apt-get install -y build-essential git-all wget +sudo apt-get update + +wget --continue https://storage.googleapis.com/golang/go1.8.1.linux-amd64.tar.gz +sudo tar -C /usr/local -xzf go1.8.1.linux-amd64.tar.gz + +GETH_PATH="~vagrant/go/src/github.com/expanse-org/go-expanse/build/bin/" + +echo "export PATH=$PATH:/usr/local/go/bin:$GETH_PATH" >> ~vagrant/.bashrc diff --git a/core/blockchain.go b/core/blockchain.go index 8fe530318d..1841b7fd3f 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -56,10 +56,10 @@ const ( blockCacheLimit = 256 maxFutureBlocks = 256 maxTimeFutureBlocks = 30 - // must be bumped when consensus algorithm is changed, this forces the upgradedb - // command to be run (forces the blocks to be imported again using the new algorithm) + badBlockLimit = 10 + + // BlockChainVersion ensures that an incompatible database forces a resync from scratch. BlockChainVersion = 3 - badBlockLimit = 10 ) // BlockChain represents the canonical chain given a database with a genesis @@ -168,67 +168,67 @@ func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, engine co return bc, nil } -func (self *BlockChain) getProcInterrupt() bool { - return atomic.LoadInt32(&self.procInterrupt) == 1 +func (bc *BlockChain) getProcInterrupt() bool { + return atomic.LoadInt32(&bc.procInterrupt) == 1 } // loadLastState loads the last known chain state from the database. This method // assumes that the chain manager mutex is held. -func (self *BlockChain) loadLastState() error { +func (bc *BlockChain) loadLastState() error { // Restore the last known head block - head := GetHeadBlockHash(self.chainDb) + head := GetHeadBlockHash(bc.chainDb) if head == (common.Hash{}) { // Corrupt or empty database, init from scratch log.Warn("Empty database, resetting chain") - return self.Reset() + return bc.Reset() } // Make sure the entire head block is available - currentBlock := self.GetBlockByHash(head) + currentBlock := bc.GetBlockByHash(head) if currentBlock == nil { // Corrupt or empty database, init from scratch log.Warn("Head block missing, resetting chain", "hash", head) - return self.Reset() + return bc.Reset() } // Make sure the state associated with the block is available - if _, err := state.New(currentBlock.Root(), self.chainDb); err != nil { + if _, err := state.New(currentBlock.Root(), bc.chainDb); err != nil { // Dangling block without a state associated, init from scratch log.Warn("Head state missing, resetting chain", "number", currentBlock.Number(), "hash", currentBlock.Hash()) - return self.Reset() + return bc.Reset() } // Everything seems to be fine, set as the head block - self.currentBlock = currentBlock + bc.currentBlock = currentBlock // Restore the last known head header - currentHeader := self.currentBlock.Header() - if head := GetHeadHeaderHash(self.chainDb); head != (common.Hash{}) { - if header := self.GetHeaderByHash(head); header != nil { + currentHeader := bc.currentBlock.Header() + if head := GetHeadHeaderHash(bc.chainDb); head != (common.Hash{}) { + if header := bc.GetHeaderByHash(head); header != nil { currentHeader = header } } - self.hc.SetCurrentHeader(currentHeader) + bc.hc.SetCurrentHeader(currentHeader) // Restore the last known head fast block - self.currentFastBlock = self.currentBlock - if head := GetHeadFastBlockHash(self.chainDb); head != (common.Hash{}) { - if block := self.GetBlockByHash(head); block != nil { - self.currentFastBlock = block + bc.currentFastBlock = bc.currentBlock + if head := GetHeadFastBlockHash(bc.chainDb); head != (common.Hash{}) { + if block := bc.GetBlockByHash(head); block != nil { + bc.currentFastBlock = block } } // Initialize a statedb cache to ensure singleton account bloom filter generation - statedb, err := state.New(self.currentBlock.Root(), self.chainDb) + statedb, err := state.New(bc.currentBlock.Root(), bc.chainDb) if err != nil { return err } - self.stateCache = statedb + bc.stateCache = statedb // Issue a status log for the user - headerTd := self.GetTd(currentHeader.Hash(), currentHeader.Number.Uint64()) - blockTd := self.GetTd(self.currentBlock.Hash(), self.currentBlock.NumberU64()) - fastTd := self.GetTd(self.currentFastBlock.Hash(), self.currentFastBlock.NumberU64()) + headerTd := bc.GetTd(currentHeader.Hash(), currentHeader.Number.Uint64()) + blockTd := bc.GetTd(bc.currentBlock.Hash(), bc.currentBlock.NumberU64()) + fastTd := bc.GetTd(bc.currentFastBlock.Hash(), bc.currentFastBlock.NumberU64()) log.Info("Loaded most recent local header", "number", currentHeader.Number, "hash", currentHeader.Hash(), "td", headerTd) - log.Info("Loaded most recent local full block", "number", self.currentBlock.Number(), "hash", self.currentBlock.Hash(), "td", blockTd) - log.Info("Loaded most recent local fast block", "number", self.currentFastBlock.Number(), "hash", self.currentFastBlock.Hash(), "td", fastTd) + log.Info("Loaded most recent local full block", "number", bc.currentBlock.Number(), "hash", bc.currentBlock.Hash(), "td", blockTd) + log.Info("Loaded most recent local fast block", "number", bc.currentFastBlock.Number(), "hash", bc.currentFastBlock.Hash(), "td", fastTd) return nil } @@ -288,103 +288,103 @@ func (bc *BlockChain) SetHead(head uint64) error { // FastSyncCommitHead sets the current head block to the one defined by the hash // irrelevant what the chain contents were prior. -func (self *BlockChain) FastSyncCommitHead(hash common.Hash) error { +func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error { // Make sure that both the block as well at its state trie exists - block := self.GetBlockByHash(hash) + block := bc.GetBlockByHash(hash) if block == nil { return fmt.Errorf("non existent block [%x…]", hash[:4]) } - if _, err := trie.NewSecure(block.Root(), self.chainDb, 0); err != nil { + if _, err := trie.NewSecure(block.Root(), bc.chainDb, 0); err != nil { return err } // If all checks out, manually set the head block - self.mu.Lock() - self.currentBlock = block - self.mu.Unlock() + bc.mu.Lock() + bc.currentBlock = block + bc.mu.Unlock() log.Info("Committed new head block", "number", block.Number(), "hash", hash) return nil } // GasLimit returns the gas limit of the current HEAD block. -func (self *BlockChain) GasLimit() *big.Int { - self.mu.RLock() - defer self.mu.RUnlock() +func (bc *BlockChain) GasLimit() *big.Int { + bc.mu.RLock() + defer bc.mu.RUnlock() - return self.currentBlock.GasLimit() + return bc.currentBlock.GasLimit() } // LastBlockHash return the hash of the HEAD block. -func (self *BlockChain) LastBlockHash() common.Hash { - self.mu.RLock() - defer self.mu.RUnlock() +func (bc *BlockChain) LastBlockHash() common.Hash { + bc.mu.RLock() + defer bc.mu.RUnlock() - return self.currentBlock.Hash() + return bc.currentBlock.Hash() } // CurrentBlock retrieves the current head block of the canonical chain. The // block is retrieved from the blockchain's internal cache. -func (self *BlockChain) CurrentBlock() *types.Block { - self.mu.RLock() - defer self.mu.RUnlock() +func (bc *BlockChain) CurrentBlock() *types.Block { + bc.mu.RLock() + defer bc.mu.RUnlock() - return self.currentBlock + return bc.currentBlock } // CurrentFastBlock retrieves the current fast-sync head block of the canonical // chain. The block is retrieved from the blockchain's internal cache. -func (self *BlockChain) CurrentFastBlock() *types.Block { - self.mu.RLock() - defer self.mu.RUnlock() +func (bc *BlockChain) CurrentFastBlock() *types.Block { + bc.mu.RLock() + defer bc.mu.RUnlock() - return self.currentFastBlock + return bc.currentFastBlock } // Status returns status information about the current chain such as the HEAD Td, // the HEAD hash and the hash of the genesis block. -func (self *BlockChain) Status() (td *big.Int, currentBlock common.Hash, genesisBlock common.Hash) { - self.mu.RLock() - defer self.mu.RUnlock() +func (bc *BlockChain) Status() (td *big.Int, currentBlock common.Hash, genesisBlock common.Hash) { + bc.mu.RLock() + defer bc.mu.RUnlock() - return self.GetTd(self.currentBlock.Hash(), self.currentBlock.NumberU64()), self.currentBlock.Hash(), self.genesisBlock.Hash() + return bc.GetTd(bc.currentBlock.Hash(), bc.currentBlock.NumberU64()), bc.currentBlock.Hash(), bc.genesisBlock.Hash() } // SetProcessor sets the processor required for making state modifications. -func (self *BlockChain) SetProcessor(processor Processor) { - self.procmu.Lock() - defer self.procmu.Unlock() - self.processor = processor +func (bc *BlockChain) SetProcessor(processor Processor) { + bc.procmu.Lock() + defer bc.procmu.Unlock() + bc.processor = processor } // SetValidator sets the validator which is used to validate incoming blocks. -func (self *BlockChain) SetValidator(validator Validator) { - self.procmu.Lock() - defer self.procmu.Unlock() - self.validator = validator +func (bc *BlockChain) SetValidator(validator Validator) { + bc.procmu.Lock() + defer bc.procmu.Unlock() + bc.validator = validator } // Validator returns the current validator. -func (self *BlockChain) Validator() Validator { - self.procmu.RLock() - defer self.procmu.RUnlock() - return self.validator +func (bc *BlockChain) Validator() Validator { + bc.procmu.RLock() + defer bc.procmu.RUnlock() + return bc.validator } // Processor returns the current processor. -func (self *BlockChain) Processor() Processor { - self.procmu.RLock() - defer self.procmu.RUnlock() - return self.processor +func (bc *BlockChain) Processor() Processor { + bc.procmu.RLock() + defer bc.procmu.RUnlock() + return bc.processor } // State returns a new mutable state based on the current HEAD block. -func (self *BlockChain) State() (*state.StateDB, error) { - return self.StateAt(self.CurrentBlock().Root()) +func (bc *BlockChain) State() (*state.StateDB, error) { + return bc.StateAt(bc.CurrentBlock().Root()) } // StateAt returns a new mutable state based on a particular point in time. -func (self *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) { - return self.stateCache.New(root) +func (bc *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) { + return bc.stateCache.New(root) } // Reset purges the entire blockchain, restoring it to its genesis state. @@ -420,14 +420,14 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error { } // Export writes the active chain to the given writer. -func (self *BlockChain) Export(w io.Writer) error { - return self.ExportN(w, uint64(0), self.currentBlock.NumberU64()) +func (bc *BlockChain) Export(w io.Writer) error { + return bc.ExportN(w, uint64(0), bc.currentBlock.NumberU64()) } // ExportN writes a subset of the active chain to the given writer. -func (self *BlockChain) ExportN(w io.Writer, first uint64, last uint64) error { - self.mu.RLock() - defer self.mu.RUnlock() +func (bc *BlockChain) ExportN(w io.Writer, first uint64, last uint64) error { + bc.mu.RLock() + defer bc.mu.RUnlock() if first > last { return fmt.Errorf("export failed: first (%d) is greater than last (%d)", first, last) @@ -435,7 +435,7 @@ func (self *BlockChain) ExportN(w io.Writer, first uint64, last uint64) error { log.Info("Exporting batch of blocks", "count", last-first+1) for nr := first; nr <= last; nr++ { - block := self.GetBlockByNumber(nr) + block := bc.GetBlockByNumber(nr) if block == nil { return fmt.Errorf("export failed on #%d: not found", nr) } @@ -478,41 +478,41 @@ func (bc *BlockChain) insert(block *types.Block) { } } -// Accessors +// Genesis retrieves the chain's genesis block. func (bc *BlockChain) Genesis() *types.Block { return bc.genesisBlock } // GetBody retrieves a block body (transactions and uncles) from the database by // hash, caching it if found. -func (self *BlockChain) GetBody(hash common.Hash) *types.Body { +func (bc *BlockChain) GetBody(hash common.Hash) *types.Body { // Short circuit if the body's already in the cache, retrieve otherwise - if cached, ok := self.bodyCache.Get(hash); ok { + if cached, ok := bc.bodyCache.Get(hash); ok { body := cached.(*types.Body) return body } - body := GetBody(self.chainDb, hash, self.hc.GetBlockNumber(hash)) + body := GetBody(bc.chainDb, hash, bc.hc.GetBlockNumber(hash)) if body == nil { return nil } // Cache the found body for next time and return - self.bodyCache.Add(hash, body) + bc.bodyCache.Add(hash, body) return body } // GetBodyRLP retrieves a block body in RLP encoding from the database by hash, // caching it if found. -func (self *BlockChain) GetBodyRLP(hash common.Hash) rlp.RawValue { +func (bc *BlockChain) GetBodyRLP(hash common.Hash) rlp.RawValue { // Short circuit if the body's already in the cache, retrieve otherwise - if cached, ok := self.bodyRLPCache.Get(hash); ok { + if cached, ok := bc.bodyRLPCache.Get(hash); ok { return cached.(rlp.RawValue) } - body := GetBodyRLP(self.chainDb, hash, self.hc.GetBlockNumber(hash)) + body := GetBodyRLP(bc.chainDb, hash, bc.hc.GetBlockNumber(hash)) if len(body) == 0 { return nil } // Cache the found body for next time and return - self.bodyRLPCache.Add(hash, body) + bc.bodyRLPCache.Add(hash, body) return body } @@ -537,41 +537,41 @@ func (bc *BlockChain) HasBlockAndState(hash common.Hash) bool { // GetBlock retrieves a block from the database by hash and number, // caching it if found. -func (self *BlockChain) GetBlock(hash common.Hash, number uint64) *types.Block { +func (bc *BlockChain) GetBlock(hash common.Hash, number uint64) *types.Block { // Short circuit if the block's already in the cache, retrieve otherwise - if block, ok := self.blockCache.Get(hash); ok { + if block, ok := bc.blockCache.Get(hash); ok { return block.(*types.Block) } - block := GetBlock(self.chainDb, hash, number) + block := GetBlock(bc.chainDb, hash, number) if block == nil { return nil } // Cache the found block for next time and return - self.blockCache.Add(block.Hash(), block) + bc.blockCache.Add(block.Hash(), block) return block } // GetBlockByHash retrieves a block from the database by hash, caching it if found. -func (self *BlockChain) GetBlockByHash(hash common.Hash) *types.Block { - return self.GetBlock(hash, self.hc.GetBlockNumber(hash)) +func (bc *BlockChain) GetBlockByHash(hash common.Hash) *types.Block { + return bc.GetBlock(hash, bc.hc.GetBlockNumber(hash)) } // GetBlockByNumber retrieves a block from the database by number, caching it // (associated with its hash) if found. -func (self *BlockChain) GetBlockByNumber(number uint64) *types.Block { - hash := GetCanonicalHash(self.chainDb, number) +func (bc *BlockChain) GetBlockByNumber(number uint64) *types.Block { + hash := GetCanonicalHash(bc.chainDb, number) if hash == (common.Hash{}) { return nil } - return self.GetBlock(hash, number) + return bc.GetBlock(hash, number) } -// [deprecated by eth/62] // GetBlocksFromHash returns the block corresponding to hash and up to n-1 ancestors. -func (self *BlockChain) GetBlocksFromHash(hash common.Hash, n int) (blocks []*types.Block) { - number := self.hc.GetBlockNumber(hash) +// [deprecated by eth/62] +func (bc *BlockChain) GetBlocksFromHash(hash common.Hash, n int) (blocks []*types.Block) { + number := bc.hc.GetBlockNumber(hash) for i := 0; i < n; i++ { - block := self.GetBlock(hash, number) + block := bc.GetBlock(hash, number) if block == nil { break } @@ -584,11 +584,11 @@ func (self *BlockChain) GetBlocksFromHash(hash common.Hash, n int) (blocks []*ty // GetUnclesInChain retrieves all the uncles from a given block backwards until // a specific distance is reached. -func (self *BlockChain) GetUnclesInChain(block *types.Block, length int) []*types.Header { +func (bc *BlockChain) GetUnclesInChain(block *types.Block, length int) []*types.Header { uncles := []*types.Header{} for i := 0; block != nil && i < length; i++ { uncles = append(uncles, block.Uncles()...) - block = self.GetBlock(block.ParentHash(), block.NumberU64()-1) + block = bc.GetBlock(block.ParentHash(), block.NumberU64()-1) } return uncles } @@ -606,10 +606,10 @@ func (bc *BlockChain) Stop() { log.Info("Blockchain manager stopped") } -func (self *BlockChain) procFutureBlocks() { - blocks := make([]*types.Block, 0, self.futureBlocks.Len()) - for _, hash := range self.futureBlocks.Keys() { - if block, exist := self.futureBlocks.Peek(hash); exist { +func (bc *BlockChain) procFutureBlocks() { + blocks := make([]*types.Block, 0, bc.futureBlocks.Len()) + for _, hash := range bc.futureBlocks.Keys() { + if block, exist := bc.futureBlocks.Peek(hash); exist { blocks = append(blocks, block.(*types.Block)) } } @@ -618,40 +618,40 @@ func (self *BlockChain) procFutureBlocks() { // Insert one by one as chain insertion needs contiguous ancestry between blocks for i := range blocks { - self.InsertChain(blocks[i : i+1]) + bc.InsertChain(blocks[i : i+1]) } } } +// WriteStatus status of write type WriteStatus byte const ( NonStatTy WriteStatus = iota CanonStatTy - SplitStatTy SideStatTy ) // Rollback is designed to remove a chain of links from the database that aren't // certain enough to be valid. -func (self *BlockChain) Rollback(chain []common.Hash) { - self.mu.Lock() - defer self.mu.Unlock() +func (bc *BlockChain) Rollback(chain []common.Hash) { + bc.mu.Lock() + defer bc.mu.Unlock() for i := len(chain) - 1; i >= 0; i-- { hash := chain[i] - currentHeader := self.hc.CurrentHeader() + currentHeader := bc.hc.CurrentHeader() if currentHeader.Hash() == hash { - self.hc.SetCurrentHeader(self.GetHeader(currentHeader.ParentHash, currentHeader.Number.Uint64()-1)) + bc.hc.SetCurrentHeader(bc.GetHeader(currentHeader.ParentHash, currentHeader.Number.Uint64()-1)) } - if self.currentFastBlock.Hash() == hash { - self.currentFastBlock = self.GetBlock(self.currentFastBlock.ParentHash(), self.currentFastBlock.NumberU64()-1) - WriteHeadFastBlockHash(self.chainDb, self.currentFastBlock.Hash()) + if bc.currentFastBlock.Hash() == hash { + bc.currentFastBlock = bc.GetBlock(bc.currentFastBlock.ParentHash(), bc.currentFastBlock.NumberU64()-1) + WriteHeadFastBlockHash(bc.chainDb, bc.currentFastBlock.Hash()) } - if self.currentBlock.Hash() == hash { - self.currentBlock = self.GetBlock(self.currentBlock.ParentHash(), self.currentBlock.NumberU64()-1) - WriteHeadBlockHash(self.chainDb, self.currentBlock.Hash()) + if bc.currentBlock.Hash() == hash { + bc.currentBlock = bc.GetBlock(bc.currentBlock.ParentHash(), bc.currentBlock.NumberU64()-1) + WriteHeadBlockHash(bc.chainDb, bc.currentBlock.Hash()) } } } @@ -666,10 +666,11 @@ func SetReceiptsData(config *params.ChainConfig, block *types.Block, receipts ty // The transaction hash can be retrieved from the transaction itself receipts[j].TxHash = transactions[j].Hash() - tx, _ := transactions[j].AsMessage(signer) // The contract address can be derived from the transaction itself - if MessageCreatesContract(tx) { - receipts[j].ContractAddress = crypto.CreateAddress(tx.From(), tx.Nonce()) + if transactions[j].To() == nil { + // Deriving the signer is expensive, only do if it's actually needed + from, _ := types.Sender(signer, transactions[j]) + receipts[j].ContractAddress = crypto.CreateAddress(from, transactions[j].Nonce()) } // The used gas can be calculated based on previous receipts if j == 0 { @@ -692,7 +693,7 @@ func SetReceiptsData(config *params.ChainConfig, block *types.Block, receipts ty // InsertReceiptChain attempts to complete an already existing header chain with // transaction and receipt data. // XXX should this be moved to the test? -func (self *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain []types.Receipts) (int, error) { +func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain []types.Receipts) (int, error) { // Do a sanity check that the provided chain is actually ordered and linked for i := 1; i < len(blockChain); i++ { if blockChain[i].NumberU64() != blockChain[i-1].NumberU64()+1 || blockChain[i].ParentHash() != blockChain[i-1].Hash() { @@ -705,8 +706,8 @@ func (self *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain } } // Pre-checks passed, start the block body and receipt imports - self.wg.Add(1) - defer self.wg.Done() + bc.wg.Add(1) + defer bc.wg.Done() // Collect some import statistics to report on stats := struct{ processed, ignored int32 }{} @@ -725,51 +726,51 @@ func (self *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain block, receipts := blockChain[index], receiptChain[index] // Short circuit insertion if shutting down or processing failed - if atomic.LoadInt32(&self.procInterrupt) == 1 { + if atomic.LoadInt32(&bc.procInterrupt) == 1 { return } if atomic.LoadInt32(&failed) > 0 { return } // Short circuit if the owner header is unknown - if !self.HasHeader(block.Hash()) { + if !bc.HasHeader(block.Hash()) { errs[index] = fmt.Errorf("containing header #%d [%x…] unknown", block.Number(), block.Hash().Bytes()[:4]) atomic.AddInt32(&failed, 1) return } // Skip if the entire data is already known - if self.HasBlock(block.Hash()) { + if bc.HasBlock(block.Hash()) { atomic.AddInt32(&stats.ignored, 1) continue } // Compute all the non-consensus fields of the receipts - SetReceiptsData(self.config, block, receipts) + SetReceiptsData(bc.config, block, receipts) // Write all the data out into the database - if err := WriteBody(self.chainDb, block.Hash(), block.NumberU64(), block.Body()); err != nil { + if err := WriteBody(bc.chainDb, block.Hash(), block.NumberU64(), block.Body()); err != nil { errs[index] = fmt.Errorf("failed to write block body: %v", err) atomic.AddInt32(&failed, 1) log.Crit("Failed to write block body", "err", err) return } - if err := WriteBlockReceipts(self.chainDb, block.Hash(), block.NumberU64(), receipts); err != nil { + if err := WriteBlockReceipts(bc.chainDb, block.Hash(), block.NumberU64(), receipts); err != nil { errs[index] = fmt.Errorf("failed to write block receipts: %v", err) atomic.AddInt32(&failed, 1) log.Crit("Failed to write block receipts", "err", err) return } - if err := WriteMipmapBloom(self.chainDb, block.NumberU64(), receipts); err != nil { + if err := WriteMipmapBloom(bc.chainDb, block.NumberU64(), receipts); err != nil { errs[index] = fmt.Errorf("failed to write log blooms: %v", err) atomic.AddInt32(&failed, 1) log.Crit("Failed to write log blooms", "err", err) return } - if err := WriteTransactions(self.chainDb, block); err != nil { + if err := WriteTransactions(bc.chainDb, block); err != nil { errs[index] = fmt.Errorf("failed to write individual transactions: %v", err) atomic.AddInt32(&failed, 1) log.Crit("Failed to write individual transactions", "err", err) return } - if err := WriteReceipts(self.chainDb, receipts); err != nil { + if err := WriteReceipts(bc.chainDb, receipts); err != nil { errs[index] = fmt.Errorf("failed to write individual receipts: %v", err) atomic.AddInt32(&failed, 1) log.Crit("Failed to write individual receipts", "err", err) @@ -797,23 +798,23 @@ func (self *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain } } } - if atomic.LoadInt32(&self.procInterrupt) == 1 { + if atomic.LoadInt32(&bc.procInterrupt) == 1 { log.Debug("Premature abort during receipts processing") return 0, nil } // Update the head fast sync block if better - self.mu.Lock() + bc.mu.Lock() head := blockChain[len(errs)-1] - if td := self.GetTd(head.Hash(), head.NumberU64()); td != nil { // Rewind may have occurred, skip in that case - if self.GetTd(self.currentFastBlock.Hash(), self.currentFastBlock.NumberU64()).Cmp(td) < 0 { - if err := WriteHeadFastBlockHash(self.chainDb, head.Hash()); err != nil { + if td := bc.GetTd(head.Hash(), head.NumberU64()); td != nil { // Rewind may have occurred, skip in that case + if bc.GetTd(bc.currentFastBlock.Hash(), bc.currentFastBlock.NumberU64()).Cmp(td) < 0 { + if err := WriteHeadFastBlockHash(bc.chainDb, head.Hash()); err != nil { log.Crit("Failed to update head fast block hash", "err", err) } - self.currentFastBlock = head + bc.currentFastBlock = head } } - self.mu.Unlock() + bc.mu.Unlock() // Report some public statistics so the user has a clue what's going on last := blockChain[len(blockChain)-1] @@ -824,27 +825,27 @@ func (self *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain } // WriteBlock writes the block to the chain. -func (self *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err error) { - self.wg.Add(1) - defer self.wg.Done() +func (bc *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err error) { + bc.wg.Add(1) + defer bc.wg.Done() // Calculate the total difficulty of the block - ptd := self.GetTd(block.ParentHash(), block.NumberU64()-1) + ptd := bc.GetTd(block.ParentHash(), block.NumberU64()-1) if ptd == nil { return NonStatTy, consensus.ErrUnknownAncestor } // Make sure no inconsistent state is leaked during insertion - self.mu.Lock() - defer self.mu.Unlock() + bc.mu.Lock() + defer bc.mu.Unlock() - localTd := self.GetTd(self.currentBlock.Hash(), self.currentBlock.NumberU64()) + localTd := bc.GetTd(bc.currentBlock.Hash(), bc.currentBlock.NumberU64()) externTd := new(big.Int).Add(block.Difficulty(), ptd) // Irrelevant of the canonical status, write the block itself to the database - if err := self.hc.WriteTd(block.Hash(), block.NumberU64(), externTd); err != nil { + if err := bc.hc.WriteTd(block.Hash(), block.NumberU64(), externTd); err != nil { log.Crit("Failed to write block total difficulty", "err", err) } - if err := WriteBlock(self.chainDb, block); err != nil { + if err := WriteBlock(bc.chainDb, block); err != nil { log.Crit("Failed to write block contents", "err", err) } @@ -853,25 +854,25 @@ func (self *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err // Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf if externTd.Cmp(localTd) > 0 || (externTd.Cmp(localTd) == 0 && mrand.Float64() < 0.5) { // Reorganise the chain if the parent is not the head block - if block.ParentHash() != self.currentBlock.Hash() { - if err := self.reorg(self.currentBlock, block); err != nil { + if block.ParentHash() != bc.currentBlock.Hash() { + if err := bc.reorg(bc.currentBlock, block); err != nil { return NonStatTy, err } } - self.insert(block) // Insert the block as the new head of the chain + bc.insert(block) // Insert the block as the new head of the chain status = CanonStatTy } else { status = SideStatTy } - self.futureBlocks.Remove(block.Hash()) + bc.futureBlocks.Remove(block.Hash()) return } // InsertChain will attempt to insert the given chain in to the canonical chain or, otherwise, create a fork. If an error is returned // it will return the index number of the failing block as well an error describing what went wrong (for possible errors see core/errors.go). -func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) { +func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { // Do a sanity check that the provided chain is actually ordered and linked for i := 1; i < len(chain); i++ { if chain[i].NumberU64() != chain[i-1].NumberU64()+1 || chain[i].ParentHash() != chain[i-1].Hash() { @@ -884,11 +885,11 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) { } } // Pre-checks passed, start the full block imports - self.wg.Add(1) - defer self.wg.Done() + bc.wg.Add(1) + defer bc.wg.Done() - self.chainmu.Lock() - defer self.chainmu.Unlock() + bc.chainmu.Lock() + defer bc.chainmu.Unlock() // A queued approach to delivering events. This is generally // faster than direct delivery and requires much less mutex @@ -906,19 +907,19 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) { headers[i] = block.Header() seals[i] = true } - abort, results := self.engine.VerifyHeaders(self, headers, seals) + abort, results := bc.engine.VerifyHeaders(bc, headers, seals) defer close(abort) // Iterate over the blocks and insert when the verifier permits for i, block := range chain { // If the chain is terminating, stop processing blocks - if atomic.LoadInt32(&self.procInterrupt) == 1 { + if atomic.LoadInt32(&bc.procInterrupt) == 1 { log.Debug("Premature abort during blocks processing") break } // If the header is a banned one, straight out abort if BadHashes[block.Hash()] { - self.reportBlock(block, nil, ErrBlacklistedHash) + bc.reportBlock(block, nil, ErrBlacklistedHash) return i, ErrBlacklistedHash } // Wait for the block's verification to complete @@ -926,7 +927,7 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) { err := <-results if err == nil { - err = self.Validator().ValidateBody(block) + err = bc.Validator().ValidateBody(block) } if err != nil { if err == ErrKnownBlock { @@ -942,46 +943,46 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) { if block.Time().Cmp(max) > 0 { return i, fmt.Errorf("future block: %v > %v", block.Time(), max) } - self.futureBlocks.Add(block.Hash(), block) + bc.futureBlocks.Add(block.Hash(), block) stats.queued++ continue } - if err == consensus.ErrUnknownAncestor && self.futureBlocks.Contains(block.ParentHash()) { - self.futureBlocks.Add(block.Hash(), block) + if err == consensus.ErrUnknownAncestor && bc.futureBlocks.Contains(block.ParentHash()) { + bc.futureBlocks.Add(block.Hash(), block) stats.queued++ continue } - self.reportBlock(block, nil, err) + bc.reportBlock(block, nil, err) return i, err } // Create a new statedb using the parent block and report an // error if it fails. switch { case i == 0: - err = self.stateCache.Reset(self.GetBlock(block.ParentHash(), block.NumberU64()-1).Root()) + err = bc.stateCache.Reset(bc.GetBlock(block.ParentHash(), block.NumberU64()-1).Root()) default: - err = self.stateCache.Reset(chain[i-1].Root()) + err = bc.stateCache.Reset(chain[i-1].Root()) } if err != nil { - self.reportBlock(block, nil, err) + bc.reportBlock(block, nil, err) return i, err } // Process block using the parent state as reference point. - receipts, logs, usedGas, err := self.processor.Process(block, self.stateCache, self.vmConfig) + receipts, logs, usedGas, err := bc.processor.Process(block, bc.stateCache, bc.vmConfig) if err != nil { - self.reportBlock(block, receipts, err) + bc.reportBlock(block, receipts, err) return i, err } // Validate the state using the default validator - err = self.Validator().ValidateState(block, self.GetBlock(block.ParentHash(), block.NumberU64()-1), self.stateCache, receipts, usedGas) + err = bc.Validator().ValidateState(block, bc.GetBlock(block.ParentHash(), block.NumberU64()-1), bc.stateCache, receipts, usedGas) if err != nil { - self.reportBlock(block, receipts, err) + bc.reportBlock(block, receipts, err) return i, err } // Write state changes to database - _, err = self.stateCache.Commit(self.config.IsEIP158(block.Number())) + _, err = bc.stateCache.Commit(bc.config.IsEIP158(block.Number())) if err != nil { return i, err } @@ -989,12 +990,12 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) { // coalesce logs for later processing coalescedLogs = append(coalescedLogs, logs...) - if err = WriteBlockReceipts(self.chainDb, block.Hash(), block.NumberU64(), receipts); err != nil { + if err = WriteBlockReceipts(bc.chainDb, block.Hash(), block.NumberU64(), receipts); err != nil { return i, err } // write the block to the chain and get the status - status, err := self.WriteBlock(block) + status, err := bc.WriteBlock(block) if err != nil { return i, err } @@ -1008,19 +1009,19 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) { events = append(events, ChainEvent{block, block.Hash(), logs}) // This puts transactions in a extra db for rpc - if err := WriteTransactions(self.chainDb, block); err != nil { + if err := WriteTransactions(bc.chainDb, block); err != nil { return i, err } // store the receipts - if err := WriteReceipts(self.chainDb, receipts); err != nil { + if err := WriteReceipts(bc.chainDb, receipts); err != nil { return i, err } // Write map map bloom filters - if err := WriteMipmapBloom(self.chainDb, block.NumberU64(), receipts); err != nil { + if err := WriteMipmapBloom(bc.chainDb, block.NumberU64(), receipts); err != nil { return i, err } // Write hash preimages - if err := WritePreimages(self.chainDb, block.NumberU64(), self.stateCache.Preimages()); err != nil { + if err := WritePreimages(bc.chainDb, block.NumberU64(), bc.stateCache.Preimages()); err != nil { return i, err } case SideStatTy: @@ -1029,15 +1030,12 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) { blockInsertTimer.UpdateSince(bstart) events = append(events, ChainSideEvent{block}) - - case SplitStatTy: - events = append(events, ChainSplitEvent{block, logs}) } stats.processed++ stats.usedGas += usedGas.Uint64() stats.report(chain, i) } - go self.postChainEvents(events, coalescedLogs) + go bc.postChainEvents(events, coalescedLogs) return 0, nil } @@ -1095,7 +1093,7 @@ func countTransactions(chain []*types.Block) (c int) { // reorgs takes two blocks, an old chain and a new chain and will reconstruct the blocks and inserts them // to be part of the new canonical chain and accumulates potential missing transactions and post an // event about them -func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error { +func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { var ( newChain types.Blocks oldChain types.Blocks @@ -1107,7 +1105,7 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // These logs are later announced as deleted. collectLogs = func(h common.Hash) { // Coalesce logs and set 'Removed'. - receipts := GetBlockReceipts(self.chainDb, h, self.hc.GetBlockNumber(h)) + receipts := GetBlockReceipts(bc.chainDb, h, bc.hc.GetBlockNumber(h)) for _, receipt := range receipts { for _, log := range receipt.Logs { del := *log @@ -1121,7 +1119,7 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // first reduce whoever is higher bound if oldBlock.NumberU64() > newBlock.NumberU64() { // reduce old chain - for ; oldBlock != nil && oldBlock.NumberU64() != newBlock.NumberU64(); oldBlock = self.GetBlock(oldBlock.ParentHash(), oldBlock.NumberU64()-1) { + for ; oldBlock != nil && oldBlock.NumberU64() != newBlock.NumberU64(); oldBlock = bc.GetBlock(oldBlock.ParentHash(), oldBlock.NumberU64()-1) { oldChain = append(oldChain, oldBlock) deletedTxs = append(deletedTxs, oldBlock.Transactions()...) @@ -1129,7 +1127,7 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error { } } else { // reduce new chain and append new chain blocks for inserting later on - for ; newBlock != nil && newBlock.NumberU64() != oldBlock.NumberU64(); newBlock = self.GetBlock(newBlock.ParentHash(), newBlock.NumberU64()-1) { + for ; newBlock != nil && newBlock.NumberU64() != oldBlock.NumberU64(); newBlock = bc.GetBlock(newBlock.ParentHash(), newBlock.NumberU64()-1) { newChain = append(newChain, newBlock) } } @@ -1151,7 +1149,7 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error { deletedTxs = append(deletedTxs, oldBlock.Transactions()...) collectLogs(oldBlock.Hash()) - oldBlock, newBlock = self.GetBlock(oldBlock.ParentHash(), oldBlock.NumberU64()-1), self.GetBlock(newBlock.ParentHash(), newBlock.NumberU64()-1) + oldBlock, newBlock = bc.GetBlock(oldBlock.ParentHash(), oldBlock.NumberU64()-1), bc.GetBlock(newBlock.ParentHash(), newBlock.NumberU64()-1) if oldBlock == nil { return fmt.Errorf("Invalid old chain") } @@ -1174,18 +1172,18 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // insert blocks. Order does not matter. Last block will be written in ImportChain itself which creates the new head properly for _, block := range newChain { // insert the block in the canonical way, re-writing history - self.insert(block) + bc.insert(block) // write canonical receipts and transactions - if err := WriteTransactions(self.chainDb, block); err != nil { + if err := WriteTransactions(bc.chainDb, block); err != nil { return err } - receipts := GetBlockReceipts(self.chainDb, block.Hash(), block.NumberU64()) + receipts := GetBlockReceipts(bc.chainDb, block.Hash(), block.NumberU64()) // write receipts - if err := WriteReceipts(self.chainDb, receipts); err != nil { + if err := WriteReceipts(bc.chainDb, receipts); err != nil { return err } // Write map map bloom filters - if err := WriteMipmapBloom(self.chainDb, block.NumberU64(), receipts); err != nil { + if err := WriteMipmapBloom(bc.chainDb, block.NumberU64(), receipts); err != nil { return err } addedTxs = append(addedTxs, block.Transactions()...) @@ -1196,22 +1194,22 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // When transactions get deleted from the database that means the // receipts that were created in the fork must also be deleted for _, tx := range diff { - DeleteReceipt(self.chainDb, tx.Hash()) - DeleteTransaction(self.chainDb, tx.Hash()) + DeleteReceipt(bc.chainDb, tx.Hash()) + DeleteTransaction(bc.chainDb, tx.Hash()) } // Must be posted in a goroutine because of the transaction pool trying // to acquire the chain manager lock if len(diff) > 0 { - go self.eventMux.Post(RemovedTransactionEvent{diff}) + go bc.eventMux.Post(RemovedTransactionEvent{diff}) } if len(deletedLogs) > 0 { - go self.eventMux.Post(RemovedLogsEvent{deletedLogs}) + go bc.eventMux.Post(RemovedLogsEvent{deletedLogs}) } if len(oldChain) > 0 { go func() { for _, block := range oldChain { - self.eventMux.Post(ChainSideEvent{Block: block}) + bc.eventMux.Post(ChainSideEvent{Block: block}) } }() } @@ -1221,29 +1219,30 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // postChainEvents iterates over the events generated by a chain insertion and // posts them into the event mux. -func (self *BlockChain) postChainEvents(events []interface{}, logs []*types.Log) { +func (bc *BlockChain) postChainEvents(events []interface{}, logs []*types.Log) { // post event logs for further processing - self.eventMux.Post(logs) + bc.eventMux.Post(logs) for _, event := range events { if event, ok := event.(ChainEvent); ok { - // We need some control over the mining operation. Acquiring locks and waiting for the miner to create new block takes too long - // and in most cases isn't even necessary. - if self.LastBlockHash() == event.Hash { - self.eventMux.Post(ChainHeadEvent{event.Block}) + // We need some control over the mining operation. Acquiring locks and waiting + // for the miner to create new block takes too long and in most cases isn't + // even necessary. + if bc.LastBlockHash() == event.Hash { + bc.eventMux.Post(ChainHeadEvent{event.Block}) } } // Fire the insertion events individually too - self.eventMux.Post(event) + bc.eventMux.Post(event) } } -func (self *BlockChain) update() { +func (bc *BlockChain) update() { futureTimer := time.Tick(5 * time.Second) for { select { case <-futureTimer: - self.procFutureBlocks() - case <-self.quit: + bc.procFutureBlocks() + case <-bc.quit: return } } @@ -1301,28 +1300,28 @@ Error: %v // should be done or not. The reason behind the optional check is because some // of the header retrieval mechanisms already need to verify nonces, as well as // because nonces can be verified sparsely, not needing to check each. -func (self *BlockChain) InsertHeaderChain(chain []*types.Header, checkFreq int) (int, error) { +func (bc *BlockChain) InsertHeaderChain(chain []*types.Header, checkFreq int) (int, error) { start := time.Now() - if i, err := self.hc.ValidateHeaderChain(chain, checkFreq); err != nil { + if i, err := bc.hc.ValidateHeaderChain(chain, checkFreq); err != nil { return i, err } // Make sure only one thread manipulates the chain at once - self.chainmu.Lock() - defer self.chainmu.Unlock() + bc.chainmu.Lock() + defer bc.chainmu.Unlock() - self.wg.Add(1) - defer self.wg.Done() + bc.wg.Add(1) + defer bc.wg.Done() whFunc := func(header *types.Header) error { - self.mu.Lock() - defer self.mu.Unlock() + bc.mu.Lock() + defer bc.mu.Unlock() - _, err := self.hc.WriteHeader(header) + _, err := bc.hc.WriteHeader(header) return err } - return self.hc.InsertHeaderChain(chain, whFunc, start) + return bc.hc.InsertHeaderChain(chain, whFunc, start) } // writeHeader writes a header into the local chain, given that its parent is @@ -1334,48 +1333,48 @@ func (self *BlockChain) InsertHeaderChain(chain []*types.Header, checkFreq int) // without the real blocks. Hence, writing headers directly should only be done // in two scenarios: pure-header mode of operation (light clients), or properly // separated header/block phases (non-archive clients). -func (self *BlockChain) writeHeader(header *types.Header) error { - self.wg.Add(1) - defer self.wg.Done() +func (bc *BlockChain) writeHeader(header *types.Header) error { + bc.wg.Add(1) + defer bc.wg.Done() - self.mu.Lock() - defer self.mu.Unlock() + bc.mu.Lock() + defer bc.mu.Unlock() - _, err := self.hc.WriteHeader(header) + _, err := bc.hc.WriteHeader(header) return err } // CurrentHeader retrieves the current head header of the canonical chain. The // header is retrieved from the HeaderChain's internal cache. -func (self *BlockChain) CurrentHeader() *types.Header { - self.mu.RLock() - defer self.mu.RUnlock() +func (bc *BlockChain) CurrentHeader() *types.Header { + bc.mu.RLock() + defer bc.mu.RUnlock() - return self.hc.CurrentHeader() + return bc.hc.CurrentHeader() } // GetTd retrieves a block's total difficulty in the canonical chain from the // database by hash and number, caching it if found. -func (self *BlockChain) GetTd(hash common.Hash, number uint64) *big.Int { - return self.hc.GetTd(hash, number) +func (bc *BlockChain) GetTd(hash common.Hash, number uint64) *big.Int { + return bc.hc.GetTd(hash, number) } // GetTdByHash retrieves a block's total difficulty in the canonical chain from the // database by hash, caching it if found. -func (self *BlockChain) GetTdByHash(hash common.Hash) *big.Int { - return self.hc.GetTdByHash(hash) +func (bc *BlockChain) GetTdByHash(hash common.Hash) *big.Int { + return bc.hc.GetTdByHash(hash) } // GetHeader retrieves a block header from the database by hash and number, // caching it if found. -func (self *BlockChain) GetHeader(hash common.Hash, number uint64) *types.Header { - return self.hc.GetHeader(hash, number) +func (bc *BlockChain) GetHeader(hash common.Hash, number uint64) *types.Header { + return bc.hc.GetHeader(hash, number) } // GetHeaderByHash retrieves a block header from the database by hash, caching it if // found. -func (self *BlockChain) GetHeaderByHash(hash common.Hash) *types.Header { - return self.hc.GetHeaderByHash(hash) +func (bc *BlockChain) GetHeaderByHash(hash common.Hash) *types.Header { + return bc.hc.GetHeaderByHash(hash) } // HasHeader checks if a block header is present in the database or not, caching @@ -1386,18 +1385,18 @@ func (bc *BlockChain) HasHeader(hash common.Hash) bool { // GetBlockHashesFromHash retrieves a number of block hashes starting at a given // hash, fetching towards the genesis block. -func (self *BlockChain) GetBlockHashesFromHash(hash common.Hash, max uint64) []common.Hash { - return self.hc.GetBlockHashesFromHash(hash, max) +func (bc *BlockChain) GetBlockHashesFromHash(hash common.Hash, max uint64) []common.Hash { + return bc.hc.GetBlockHashesFromHash(hash, max) } // GetHeaderByNumber retrieves a block header from the database by number, // caching it (associated with its hash) if found. -func (self *BlockChain) GetHeaderByNumber(number uint64) *types.Header { - return self.hc.GetHeaderByNumber(number) +func (bc *BlockChain) GetHeaderByNumber(number uint64) *types.Header { + return bc.hc.GetHeaderByNumber(number) } // Config retrieves the blockchain's chain configuration. -func (self *BlockChain) Config() *params.ChainConfig { return self.config } +func (bc *BlockChain) Config() *params.ChainConfig { return bc.config } // Engine retrieves the blockchain's consensus engine. -func (self *BlockChain) Engine() consensus.Engine { return self.engine } +func (bc *BlockChain) Engine() consensus.Engine { return bc.engine } diff --git a/core/blocks.go b/core/blocks.go index fa5fac2684..74c997cdc1 100644 --- a/core/blocks.go +++ b/core/blocks.go @@ -18,7 +18,7 @@ package core import "github.com/expanse-org/go-expanse/common" -// Set of manually tracked bad hashes (usually hard forks) +// BadHashes represent a set of manually tracked bad hashes (usually hard forks) var BadHashes = map[common.Hash]bool{ common.HexToHash("05bef30ef572270f654746da22639a7a0c97dd97a7050b9e252391996aaeb689"): true, common.HexToHash("7d05d08cbc596a2e5e4f13b80a743e53e09221b5323c3a61946b20873e58583f"): true, diff --git a/core/chain_makers.go b/core/chain_makers.go index 2c6aa04ed7..15db2127b9 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -84,7 +84,7 @@ func (b *BlockGen) AddTx(tx *types.Transaction) { if b.gasPool == nil { b.SetCoinbase(common.Address{}) } - b.statedb.StartRecord(tx.Hash(), common.Hash{}, len(b.txs)) + b.statedb.Prepare(tx.Hash(), common.Hash{}, len(b.txs)) receipt, _, err := ApplyTransaction(b.config, nil, &b.header.Coinbase, b.gasPool, b.statedb, b.header, tx, b.header.GasUsed, vm.Config{}) if err != nil { panic(err) @@ -98,10 +98,10 @@ func (b *BlockGen) Number() *big.Int { return new(big.Int).Set(b.header.Number) } -// AddUncheckedReceipts forcefully adds a receipts to the block without a +// AddUncheckedReceipt forcefully adds a receipts to the block without a // backing transaction. // -// AddUncheckedReceipts will cause consensus failures when used during real +// AddUncheckedReceipt will cause consensus failures when used during real // chain processing. This is best used in conjunction with raw block insertion. func (b *BlockGen) AddUncheckedReceipt(receipt *types.Receipt) { b.receipts = append(b.receipts, receipt) @@ -142,7 +142,7 @@ func (b *BlockGen) OffsetTime(seconds int64) { if b.header.Time.Cmp(b.parent.Header().Time) <= 0 { panic("block time out of range") } - b.header.Difficulty = ethash.CalcDifficulty(b.config, b.header.Time.Uint64(), b.parent.Time().Uint64(), b.parent.Number(), b.parent.Difficulty()) + b.header.Difficulty = ethash.CalcDifficulty(b.config, b.header.Time.Uint64(), b.parent.Header()) } // GenerateChain creates a chain of n blocks. The first block's @@ -209,15 +209,20 @@ func makeHeader(config *params.ChainConfig, parent *types.Block, state *state.St } else { time = new(big.Int).Add(parent.Time(), big.NewInt(10)) // block time is fixed at 10 seconds } + return &types.Header{ Root: state.IntermediateRoot(config.IsEIP158(parent.Number())), ParentHash: parent.Hash(), Coinbase: parent.Coinbase(), - Difficulty: ethash.CalcDifficulty(config, time.Uint64(), new(big.Int).Sub(time, big.NewInt(10)).Uint64(), parent.Number(), parent.Difficulty()), - GasLimit: CalcGasLimit(parent), - GasUsed: new(big.Int), - Number: new(big.Int).Add(parent.Number(), common.Big1), - Time: time, + Difficulty: ethash.CalcDifficulty(config, time.Uint64(), &types.Header{ + Number: parent.Number(), + Time: new(big.Int).Sub(time, big.NewInt(10)), + Difficulty: parent.Difficulty(), + }), + GasLimit: CalcGasLimit(parent), + GasUsed: new(big.Int), + Number: new(big.Int).Add(parent.Number(), common.Big1), + Time: time, } } diff --git a/core/database_util.go b/core/database_util.go index 25c9bc7106..ce8293a3a7 100644 --- a/core/database_util.go +++ b/core/database_util.go @@ -64,7 +64,7 @@ var ( oldBlockReceiptsPrefix = []byte("receipts-block-") oldBlockHashPrefix = []byte("block-hash-") // [deprecated by the header/block split, remove eventually] - ChainConfigNotFoundErr = errors.New("ChainConfig not found") // general config not found error + ErrChainConfigNotFound = errors.New("ChainConfig not found") // general config not found error mipmapBloomMu sync.Mutex // protect against race condition when updating mipmap blooms @@ -546,7 +546,7 @@ func mipmapKey(num, level uint64) []byte { return append(mipmapPre, append(lkey, key.Bytes()...)...) } -// WriteMapmapBloom writes each address included in the receipts' logs to the +// WriteMipmapBloom writes each address included in the receipts' logs to the // MIP bloom bin. func WriteMipmapBloom(db ethdb.Database, number uint64, receipts types.Receipts) error { mipmapBloomMu.Lock() @@ -638,7 +638,7 @@ func WriteChainConfig(db ethdb.Database, hash common.Hash, cfg *params.ChainConf func GetChainConfig(db ethdb.Database, hash common.Hash) (*params.ChainConfig, error) { jsonChainConfig, _ := db.Get(append(configPrefix, hash[:]...)) if len(jsonChainConfig) == 0 { - return nil, ChainConfigNotFoundErr + return nil, ErrChainConfigNotFound } var config params.ChainConfig diff --git a/core/events.go b/core/events.go index 443e644d5c..3818049764 100644 --- a/core/events.go +++ b/core/events.go @@ -17,8 +17,6 @@ package core import ( - "math/big" - "github.com/expanse-org/go-expanse/common" "github.com/expanse-org/go-expanse/core/types" ) @@ -43,15 +41,9 @@ type NewMinedBlockEvent struct{ Block *types.Block } // RemovedTransactionEvent is posted when a reorg happens type RemovedTransactionEvent struct{ Txs types.Transactions } -// RemovedLogEvent is posted when a reorg happens +// RemovedLogsEvent is posted when a reorg happens type RemovedLogsEvent struct{ Logs []*types.Log } -// ChainSplit is posted when a new head is detected -type ChainSplitEvent struct { - Block *types.Block - Logs []*types.Log -} - type ChainEvent struct { Block *types.Block Hash common.Hash @@ -73,8 +65,6 @@ type ChainUncleEvent struct { type ChainHeadEvent struct{ Block *types.Block } -type GasPriceChanged struct{ Price *big.Int } - // Mining operation events type StartMining struct{} type TopMining struct{} diff --git a/core/fees.go b/core/fees.go index de5ecc9d61..ad395b2840 100644 --- a/core/fees.go +++ b/core/fees.go @@ -20,4 +20,4 @@ import ( "math/big" ) -var BlockReward *big.Int = big.NewInt(8e+18) +var BlockReward = big.NewInt(8e+18) diff --git a/core/genesis.go b/core/genesis.go index dafb8f518a..7c64d1d195 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -86,7 +86,7 @@ type GenesisMismatchError struct { } func (e *GenesisMismatchError) Error() string { - return fmt.Sprintf("wrong genesis block in database (have %x, new %x)", e.Stored[:8], e.New[:8]) + return fmt.Sprintf("database already contains an incompatible genesis block (have %x, new %x)", e.Stored[:8], e.New[:8]) } // SetupGenesisBlock writes or updates the genesis block in db. @@ -133,7 +133,7 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig newcfg := genesis.configOrDefault(stored) storedcfg, err := GetChainConfig(db, stored) if err != nil { - if err == ChainConfigNotFoundErr { + if err == ErrChainConfigNotFound { // This case happens if a genesis write was interrupted. log.Warn("Found genesis block without chain config") err = WriteChainConfig(db, stored, newcfg) @@ -279,6 +279,18 @@ func DefaultTestnetGenesisBlock() *Genesis { } } +// DefaultRinkebyGenesisBlock returns the Rinkeby network genesis block. +func DefaultRinkebyGenesisBlock() *Genesis { + return &Genesis{ + Config: params.RinkebyChainConfig, + Timestamp: 1492009146, + ExtraData: hexutil.MustDecode("0x52657370656374206d7920617574686f7269746168207e452e436172746d616e42eb768f2244c8811c63729a21a3569731535f067ffc57839b00206d1ad20c69a1981b489f772031b279182d99e65703f0076e4812653aab85fca0f00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), + GasLimit: 4700000, + Difficulty: big.NewInt(1), + Alloc: decodePrealloc(rinkebyAllocData), + } +} + // DevGenesisBlock returns the 'geth --dev' genesis block. func DevGenesisBlock() *Genesis { return &Genesis{ diff --git a/core/genesis_alloc.go b/core/genesis_alloc.go index c3c02d497f..56e5b90986 100644 --- a/core/genesis_alloc.go +++ b/core/genesis_alloc.go @@ -22,4 +22,5 @@ package core const mainnetAllocData = "\xf8\x85\xe0\x94\x15eg\x15\x06\x8a\xb0\xdb\xdf\n\xb0\aH\xa8\xa1\x9e@\U0008148a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\x93\xde\u02b0\xcdtU\x98\x86\x0fx*\xc1\xe8\xf0F\u02d9\u860a\x02\x1e\x19\xe0\u027a\xb2@\x00\x00\u153b\x94\xf0\u03b3\"W'[*z\x9c\tL\x13\xe4i\xb4V>\x8b\bE\x95\x16\x14\x01HJ\x00\x00\x00\xe0\x94\xc0u\xfa\x11\xf8[\xda:\xab\xa6q\x06\"j\xaf\bj\xc1oN\x8a\x15-\x02\xc7\xe1J\xf6\x80\x00\x00" const testnetAllocData = "\xf8\x85\xe0\x94\x15eg\x15\x06\x8a\xb0\xdb\xdf\n\xb0\aH\xa8\xa1\x9e@\U0008148a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\x93\xde\u02b0\xcdtU\x98\x86\x0fx*\xc1\xe8\xf0F\u02d9\u860a\x02\x1e\x19\xe0\u027a\xb2@\x00\x00\u153b\x94\xf0\u03b3\"W'[*z\x9c\tL\x13\xe4i\xb4V>\x8b\bE\x95\x16\x14\x01HJ\x00\x00\x00\xe0\x94\xc0u\xfa\x11\xf8[\xda:\xab\xa6q\x06\"j\xaf\bj\xc1oN\x8a\x15-\x02\xc7\xe1J\xf6\x80\x00\x00" +const rinkebyAllocData = "\xf8\x85\xe0\x94\x15eg\x15\x06\x8a\xb0\xdb\xdf\n\xb0\aH\xa8\xa1\x9e@\U0008148a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\x93\xde\u02b0\xcdtU\x98\x86\x0fx*\xc1\xe8\xf0F\u02d9\u860a\x02\x1e\x19\xe0\u027a\xb2@\x00\x00\u153b\x94\xf0\u03b3\"W'[*z\x9c\tL\x13\xe4i\xb4V>\x8b\bE\x95\x16\x14\x01HJ\x00\x00\x00\xe0\x94\xc0u\xfa\x11\xf8[\xda:\xab\xa6q\x06\"j\xaf\bj\xc1oN\x8a\x15-\x02\xc7\xe1J\xf6\x80\x00\x00" const devAllocData = "\xf8\x85\xe0\x94\x15eg\x15\x06\x8a\xb0\xdb\xdf\n\xb0\aH\xa8\xa1\x9e@\U0008148a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\x93\xde\u02b0\xcdtU\x98\x86\x0fx*\xc1\xe8\xf0F\u02d9\u860a\x02\x1e\x19\xe0\u027a\xb2@\x00\x00\u153b\x94\xf0\u03b3\"W'[*z\x9c\tL\x13\xe4i\xb4V>\x8b\bE\x95\x16\x14\x01HJ\x00\x00\x00\xe0\x94\xc0u\xfa\x11\xf8[\xda:\xab\xa6q\x06\"j\xaf\bj\xc1oN\x8a\x15-\x02\xc7\xe1J\xf6\x80\x00\x00" diff --git a/core/headerchain.go b/core/headerchain.go index e31ff79d3b..b9b2a5e9f5 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -201,15 +201,6 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er // header writes should be protected by the parent chain mutex individually. type WhCallback func(*types.Header) error -// InsertHeaderChain attempts to insert the given header chain in to the local -// chain, possibly creating a reorg. If an error is returned, it will return the -// index number of the failing header as well an error describing what went wrong. -// -// The verify parameter can be used to fine tune whether nonce verification -// should be done or not. The reason behind the optional check is because some -// of the header retrieval mechanisms already need to verfy nonces, as well as -// because nonces can be verified sparsely, not needing to check each. - func (hc *HeaderChain) ValidateHeaderChain(chain []*types.Header, checkFreq int) (int, error) { // Do a sanity check that the provided chain is actually ordered and linked for i := 1; i < len(chain); i++ { @@ -257,6 +248,14 @@ func (hc *HeaderChain) ValidateHeaderChain(chain []*types.Header, checkFreq int) return 0, nil } +// InsertHeaderChain attempts to insert the given header chain in to the local +// chain, possibly creating a reorg. If an error is returned, it will return the +// index number of the failing header as well an error describing what went wrong. +// +// The verify parameter can be used to fine tune whether nonce verification +// should be done or not. The reason behind the optional check is because some +// of the header retrieval mechanisms already need to verfy nonces, as well as +// because nonces can be verified sparsely, not needing to check each. func (hc *HeaderChain) InsertHeaderChain(chain []*types.Header, writeHeader WhCallback, start time.Time) (int, error) { // Collect some import statistics to report on stats := struct{ processed, ignored int }{} diff --git a/core/helper_test.go b/core/helper_test.go index c904d138cd..d57352ec97 100644 --- a/core/helper_test.go +++ b/core/helper_test.go @@ -21,8 +21,6 @@ import ( "fmt" "github.com/expanse-org/go-expanse/core/types" - // "github.com/expanse-org/go-expanse/crypto" - "github.com/expanse-org/go-expanse/ethdb" "github.com/expanse-org/go-expanse/event" ) @@ -38,24 +36,24 @@ type TestManager struct { Blocks []*types.Block } -func (s *TestManager) IsListening() bool { +func (tm *TestManager) IsListening() bool { return false } -func (s *TestManager) IsMining() bool { +func (tm *TestManager) IsMining() bool { return false } -func (s *TestManager) PeerCount() int { +func (tm *TestManager) PeerCount() int { return 0 } -func (s *TestManager) Peers() *list.List { +func (tm *TestManager) Peers() *list.List { return list.New() } -func (s *TestManager) BlockChain() *BlockChain { - return s.blockChain +func (tm *TestManager) BlockChain() *BlockChain { + return tm.blockChain } func (tm *TestManager) TxPool() *TxPool { diff --git a/core/state/dump.go b/core/state/dump.go index 7f61a6e536..4642898391 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -22,6 +22,7 @@ import ( "github.com/expanse-org/go-expanse/common" "github.com/expanse-org/go-expanse/rlp" + "github.com/expanse-org/go-expanse/trie" ) type DumpAccount struct { @@ -44,7 +45,7 @@ func (self *StateDB) RawDump() Dump { Accounts: make(map[string]DumpAccount), } - it := self.trie.Iterator() + it := trie.NewIterator(self.trie.NodeIterator(nil)) for it.Next() { addr := self.trie.GetKey(it.Key) var data Account @@ -61,7 +62,7 @@ func (self *StateDB) RawDump() Dump { Code: common.Bytes2Hex(obj.Code(self.db)), Storage: make(map[string]string), } - storageIt := obj.getTrie(self.db).Iterator() + storageIt := trie.NewIterator(obj.getTrie(self.db).NodeIterator(nil)) for storageIt.Next() { account.Storage[common.Bytes2Hex(self.trie.GetKey(storageIt.Key))] = common.Bytes2Hex(storageIt.Value) } diff --git a/core/state/iterator.go b/core/state/iterator.go index 00b7e1432e..f9e546376e 100644 --- a/core/state/iterator.go +++ b/core/state/iterator.go @@ -75,7 +75,7 @@ func (it *NodeIterator) step() error { } // Initialize the iterator if we've just started if it.stateIt == nil { - it.stateIt = it.state.trie.NodeIterator() + it.stateIt = it.state.trie.NodeIterator(nil) } // If we had data nodes previously, we surely have at least state nodes if it.dataIt != nil { @@ -118,7 +118,7 @@ func (it *NodeIterator) step() error { if err != nil { return err } - it.dataIt = trie.NewNodeIterator(dataTrie) + it.dataIt = dataTrie.NodeIterator(nil) if !it.dataIt.Next(true) { it.dataIt = nil } diff --git a/core/state/journal.go b/core/state/journal.go index 64ad7ceff4..c98b4d8096 100644 --- a/core/state/journal.go +++ b/core/state/journal.go @@ -71,8 +71,8 @@ type ( hash common.Hash } touchChange struct { - account *common.Address - prev bool + account *common.Address + prev bool prevDirty bool } ) @@ -91,6 +91,11 @@ func (ch suicideChange) undo(s *StateDB) { if obj != nil { obj.suicided = ch.prev obj.setBalance(ch.prevbalance) + // if the object wasn't suicided before, remove + // it from the list of destructed objects as well. + if !obj.suicided { + delete(s.stateObjectsDestructed, *ch.account) + } } } diff --git a/core/state/state_object.go b/core/state/state_object.go index b5a96c7633..1401a32187 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -201,7 +201,7 @@ func (self *stateObject) setState(key, value common.Hash) { } // updateTrie writes cached storage modifications into the object's storage trie. -func (self *stateObject) updateTrie(db trie.Database) { +func (self *stateObject) updateTrie(db trie.Database) *trie.SecureTrie { tr := self.getTrie(db) for key, value := range self.dirtyStorage { delete(self.dirtyStorage, key) @@ -213,6 +213,7 @@ func (self *stateObject) updateTrie(db trie.Database) { v, _ := rlp.EncodeToBytes(bytes.TrimLeft(value[:], "\x00")) tr.Update(key[:], v) } + return tr } // UpdateRoot sets the trie root to the current root hash of @@ -280,7 +281,11 @@ func (c *stateObject) ReturnGas(gas *big.Int) {} func (self *stateObject) deepCopy(db *StateDB, onDirty func(addr common.Address)) *stateObject { stateObject := newObject(db, self.address, self.data, onDirty) - stateObject.trie = self.trie + if self.trie != nil { + // A shallow copy makes the two tries independent. + cpy := *self.trie + stateObject.trie = &cpy + } stateObject.code = self.code stateObject.dirtyStorage = self.dirtyStorage.Copy() stateObject.cachedStorage = self.dirtyStorage.Copy() diff --git a/core/state/statedb.go b/core/state/statedb.go index 0310743159..4c7033f1e0 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -62,8 +62,9 @@ type StateDB struct { codeSizeCache *lru.Cache // This map holds 'live' objects, which will get modified while processing a state transition. - stateObjects map[common.Address]*stateObject - stateObjectsDirty map[common.Address]struct{} + stateObjects map[common.Address]*stateObject + stateObjectsDirty map[common.Address]struct{} + stateObjectsDestructed map[common.Address]struct{} // The refund counter, also used by state transitioning. refund *big.Int @@ -92,14 +93,15 @@ func New(root common.Hash, db ethdb.Database) (*StateDB, error) { } csc, _ := lru.New(codeSizeCacheSize) return &StateDB{ - db: db, - trie: tr, - codeSizeCache: csc, - stateObjects: make(map[common.Address]*stateObject), - stateObjectsDirty: make(map[common.Address]struct{}), - refund: new(big.Int), - logs: make(map[common.Hash][]*types.Log), - preimages: make(map[common.Hash][]byte), + db: db, + trie: tr, + codeSizeCache: csc, + stateObjects: make(map[common.Address]*stateObject), + stateObjectsDirty: make(map[common.Address]struct{}), + stateObjectsDestructed: make(map[common.Address]struct{}), + refund: new(big.Int), + logs: make(map[common.Hash][]*types.Log), + preimages: make(map[common.Hash][]byte), }, nil } @@ -114,14 +116,15 @@ func (self *StateDB) New(root common.Hash) (*StateDB, error) { return nil, err } return &StateDB{ - db: self.db, - trie: tr, - codeSizeCache: self.codeSizeCache, - stateObjects: make(map[common.Address]*stateObject), - stateObjectsDirty: make(map[common.Address]struct{}), - refund: new(big.Int), - logs: make(map[common.Hash][]*types.Log), - preimages: make(map[common.Hash][]byte), + db: self.db, + trie: tr, + codeSizeCache: self.codeSizeCache, + stateObjects: make(map[common.Address]*stateObject), + stateObjectsDirty: make(map[common.Address]struct{}), + stateObjectsDestructed: make(map[common.Address]struct{}), + refund: new(big.Int), + logs: make(map[common.Hash][]*types.Log), + preimages: make(map[common.Hash][]byte), }, nil } @@ -138,6 +141,7 @@ func (self *StateDB) Reset(root common.Hash) error { self.trie = tr self.stateObjects = make(map[common.Address]*stateObject) self.stateObjectsDirty = make(map[common.Address]struct{}) + self.stateObjectsDestructed = make(map[common.Address]struct{}) self.thash = common.Hash{} self.bhash = common.Hash{} self.txIndex = 0 @@ -173,12 +177,6 @@ func (self *StateDB) pushTrie(t *trie.SecureTrie) { } } -func (self *StateDB) StartRecord(thash, bhash common.Hash, ti int) { - self.thash = thash - self.bhash = bhash - self.txIndex = ti -} - func (self *StateDB) AddLog(log *types.Log) { self.journal = append(self.journal, addLogChange{txhash: self.thash}) @@ -296,6 +294,17 @@ func (self *StateDB) GetState(a common.Address, b common.Hash) common.Hash { return common.Hash{} } +// StorageTrie returns the storage trie of an account. +// The return value is a copy and is nil for non-existent accounts. +func (self *StateDB) StorageTrie(a common.Address) *trie.SecureTrie { + stateObject := self.getStateObject(a) + if stateObject == nil { + return nil + } + cpy := stateObject.deepCopy(self, nil) + return cpy.updateTrie(self.db) +} + func (self *StateDB) HasSuicided(addr common.Address) bool { stateObject := self.getStateObject(addr) if stateObject != nil { @@ -369,6 +378,8 @@ func (self *StateDB) Suicide(addr common.Address) bool { }) stateObject.markSuicided() stateObject.data.Balance = new(big.Int) + self.stateObjectsDestructed[addr] = struct{}{} + return true } @@ -481,7 +492,7 @@ func (db *StateDB) ForEachStorage(addr common.Address, cb func(key, value common cb(h, value) } - it := so.getTrie(db.db).Iterator() + it := trie.NewIterator(so.getTrie(db.db).NodeIterator(nil)) for it.Next() { // ignore cached values key := common.BytesToHash(db.trie.GetKey(it.Key)) @@ -499,21 +510,25 @@ func (self *StateDB) Copy() *StateDB { // Copy all the basic fields, initialize the memory ones state := &StateDB{ - db: self.db, - trie: self.trie, - pastTries: self.pastTries, - codeSizeCache: self.codeSizeCache, - stateObjects: make(map[common.Address]*stateObject, len(self.stateObjectsDirty)), - stateObjectsDirty: make(map[common.Address]struct{}, len(self.stateObjectsDirty)), - refund: new(big.Int).Set(self.refund), - logs: make(map[common.Hash][]*types.Log, len(self.logs)), - logSize: self.logSize, - preimages: make(map[common.Hash][]byte), + db: self.db, + trie: self.trie, + pastTries: self.pastTries, + codeSizeCache: self.codeSizeCache, + stateObjects: make(map[common.Address]*stateObject, len(self.stateObjectsDirty)), + stateObjectsDirty: make(map[common.Address]struct{}, len(self.stateObjectsDirty)), + stateObjectsDestructed: make(map[common.Address]struct{}, len(self.stateObjectsDestructed)), + refund: new(big.Int).Set(self.refund), + logs: make(map[common.Hash][]*types.Log, len(self.logs)), + logSize: self.logSize, + preimages: make(map[common.Hash][]byte), } // Copy the dirty states, logs, and preimages for addr := range self.stateObjectsDirty { state.stateObjects[addr] = self.stateObjects[addr].deepCopy(state, state.MarkStateObjectDirty) state.stateObjectsDirty[addr] = struct{}{} + if self.stateObjects[addr].suicided { + state.stateObjectsDestructed[addr] = struct{}{} + } } for hash, logs := range self.logs { state.logs[hash] = make([]*types.Log, len(logs)) @@ -579,6 +594,27 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { return s.trie.Hash() } +// Prepare sets the current transaction hash and index and block hash which is +// used when the EVM emits new state logs. +func (self *StateDB) Prepare(thash, bhash common.Hash, ti int) { + self.thash = thash + self.bhash = bhash + self.txIndex = ti +} + +// Finalise finalises the state by removing the self destructed objects +// in the current stateObjectsDestructed buffer and clears the journal +// as well as the refunds. +// +// Please note that Finalise is used by EIP#98 and is used instead of +// IntermediateRoot. +func (s *StateDB) Finalise() { + for addr := range s.stateObjectsDestructed { + s.deleteStateObject(s.stateObjects[addr]) + } + s.clearJournalAndRefund() +} + // DeleteSuicides flags the suicided objects for deletion so that it // won't be referenced again when called / queried up on. // diff --git a/core/state_processor.go b/core/state_processor.go index 4b94351b1c..498276887e 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -69,7 +69,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg } // Iterate over and process the individual transactions for i, tx := range block.Transactions() { - statedb.StartRecord(tx.Hash(), block.Hash(), i) + statedb.Prepare(tx.Hash(), block.Hash(), i) receipt, _, err := ApplyTransaction(p.config, p.bc, nil, gp, statedb, header, tx, totalUsedGas, cfg) if err != nil { return nil, nil, nil, err @@ -107,7 +107,8 @@ func ApplyTransaction(config *params.ChainConfig, bc *BlockChain, author *common usedGas.Add(usedGas, gas) // Create a new receipt for the transaction, storing the intermediate root and gas used by the tx // based on the eip phase, we're passing wether the root touch-delete accounts. - receipt := types.NewReceipt(statedb.IntermediateRoot(config.IsEIP158(header.Number)).Bytes(), usedGas) + root := statedb.IntermediateRoot(config.IsEIP158(header.Number)) + receipt := types.NewReceipt(root.Bytes(), usedGas) receipt.TxHash = tx.Hash() receipt.GasUsed = new(big.Int).Set(gas) // if the transaction created a contract, store the creation address in the receipt. diff --git a/core/state_transition.go b/core/state_transition.go index 0c112f61fb..b60e19e16d 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -78,10 +78,6 @@ type Message interface { Data() []byte } -func MessageCreatesContract(msg Message) bool { - return msg.To() == nil -} - // IntrinsicGas computes the 'intrinsic gas' for a message // with the given data. // @@ -138,112 +134,113 @@ func ApplyMessage(evm *vm.EVM, msg Message, gp *GasPool) ([]byte, *big.Int, erro return ret, gasUsed, err } -func (self *StateTransition) from() vm.AccountRef { - f := self.msg.From() - if !self.state.Exist(f) { - self.state.CreateAccount(f) +func (st *StateTransition) from() vm.AccountRef { + f := st.msg.From() + if !st.state.Exist(f) { + st.state.CreateAccount(f) } return vm.AccountRef(f) } -func (self *StateTransition) to() vm.AccountRef { - if self.msg == nil { +func (st *StateTransition) to() vm.AccountRef { + if st.msg == nil { return vm.AccountRef{} } - to := self.msg.To() + to := st.msg.To() if to == nil { return vm.AccountRef{} // contract creation } reference := vm.AccountRef(*to) - if !self.state.Exist(*to) { - self.state.CreateAccount(*to) + if !st.state.Exist(*to) { + st.state.CreateAccount(*to) } return reference } -func (self *StateTransition) useGas(amount uint64) error { - if self.gas < amount { +func (st *StateTransition) useGas(amount uint64) error { + if st.gas < amount { return vm.ErrOutOfGas } - self.gas -= amount + st.gas -= amount return nil } -func (self *StateTransition) buyGas() error { - mgas := self.msg.Gas() +func (st *StateTransition) buyGas() error { + mgas := st.msg.Gas() if mgas.BitLen() > 64 { return vm.ErrOutOfGas } - mgval := new(big.Int).Mul(mgas, self.gasPrice) + mgval := new(big.Int).Mul(mgas, st.gasPrice) var ( - state = self.state - sender = self.from() + state = st.state + sender = st.from() ) if state.GetBalance(sender.Address()).Cmp(mgval) < 0 { return errInsufficientBalanceForGas } - if err := self.gp.SubGas(mgas); err != nil { + if err := st.gp.SubGas(mgas); err != nil { return err } - self.gas += mgas.Uint64() + st.gas += mgas.Uint64() - self.initialGas.Set(mgas) + st.initialGas.Set(mgas) state.SubBalance(sender.Address(), mgval) return nil } -func (self *StateTransition) preCheck() error { - msg := self.msg - sender := self.from() +func (st *StateTransition) preCheck() error { + msg := st.msg + sender := st.from() // Make sure this transaction's nonce is correct if msg.CheckNonce() { - if n := self.state.GetNonce(sender.Address()); n != msg.Nonce() { + if n := st.state.GetNonce(sender.Address()); n != msg.Nonce() { return fmt.Errorf("invalid nonce: have %d, expected %d", msg.Nonce(), n) } } - return self.buyGas() + return st.buyGas() } // TransitionDb will transition the state by applying the current message and returning the result // including the required gas for the operation as well as the used gas. It returns an error if it // failed. An error indicates a consensus issue. -func (self *StateTransition) TransitionDb() (ret []byte, requiredGas, usedGas *big.Int, err error) { - if err = self.preCheck(); err != nil { +func (st *StateTransition) TransitionDb() (ret []byte, requiredGas, usedGas *big.Int, err error) { + if err = st.preCheck(); err != nil { return } - msg := self.msg - sender := self.from() // err checked in preCheck + msg := st.msg + sender := st.from() // err checked in preCheck + + homestead := st.evm.ChainConfig().IsHomestead(st.evm.BlockNumber) + contractCreation := msg.To() == nil - homestead := self.evm.ChainConfig().IsHomestead(self.evm.BlockNumber) - contractCreation := MessageCreatesContract(msg) // Pay intrinsic gas // TODO convert to uint64 - intrinsicGas := IntrinsicGas(self.data, contractCreation, homestead) + intrinsicGas := IntrinsicGas(st.data, contractCreation, homestead) if intrinsicGas.BitLen() > 64 { return nil, nil, nil, vm.ErrOutOfGas } - if err = self.useGas(intrinsicGas.Uint64()); err != nil { + if err = st.useGas(intrinsicGas.Uint64()); err != nil { return nil, nil, nil, err } var ( - evm = self.evm + evm = st.evm // vm errors do not effect consensus and are therefor // not assigned to err, except for insufficient balance // error. vmerr error ) if contractCreation { - ret, _, self.gas, vmerr = evm.Create(sender, self.data, self.gas, self.value) + ret, _, st.gas, vmerr = evm.Create(sender, st.data, st.gas, st.value) } else { // Increment the nonce for the next transaction - self.state.SetNonce(sender.Address(), self.state.GetNonce(sender.Address())+1) - ret, self.gas, vmerr = evm.Call(sender, self.to().Address(), self.data, self.gas, self.value) + st.state.SetNonce(sender.Address(), st.state.GetNonce(sender.Address())+1) + ret, st.gas, vmerr = evm.Call(sender, st.to().Address(), st.data, st.gas, st.value) } if vmerr != nil { log.Debug("VM returned with error", "err", err) @@ -254,33 +251,33 @@ func (self *StateTransition) TransitionDb() (ret []byte, requiredGas, usedGas *b return nil, nil, nil, vmerr } } - requiredGas = new(big.Int).Set(self.gasUsed()) + requiredGas = new(big.Int).Set(st.gasUsed()) - self.refundGas() - self.state.AddBalance(self.evm.Coinbase, new(big.Int).Mul(self.gasUsed(), self.gasPrice)) + st.refundGas() + st.state.AddBalance(st.evm.Coinbase, new(big.Int).Mul(st.gasUsed(), st.gasPrice)) - return ret, requiredGas, self.gasUsed(), err + return ret, requiredGas, st.gasUsed(), err } -func (self *StateTransition) refundGas() { +func (st *StateTransition) refundGas() { // Return eth for remaining gas to the sender account, // exchanged at the original rate. - sender := self.from() // err already checked - remaining := new(big.Int).Mul(new(big.Int).SetUint64(self.gas), self.gasPrice) - self.state.AddBalance(sender.Address(), remaining) + sender := st.from() // err already checked + remaining := new(big.Int).Mul(new(big.Int).SetUint64(st.gas), st.gasPrice) + st.state.AddBalance(sender.Address(), remaining) // Apply refund counter, capped to half of the used gas. - uhalf := remaining.Div(self.gasUsed(), common.Big2) - refund := math.BigMin(uhalf, self.state.GetRefund()) - self.gas += refund.Uint64() + uhalf := remaining.Div(st.gasUsed(), common.Big2) + refund := math.BigMin(uhalf, st.state.GetRefund()) + st.gas += refund.Uint64() - self.state.AddBalance(sender.Address(), refund.Mul(refund, self.gasPrice)) + st.state.AddBalance(sender.Address(), refund.Mul(refund, st.gasPrice)) // Also return remaining gas to the block gas counter so it is // available for the next transaction. - self.gp.AddGas(new(big.Int).SetUint64(self.gas)) + st.gp.AddGas(new(big.Int).SetUint64(st.gas)) } -func (self *StateTransition) gasUsed() *big.Int { - return new(big.Int).Sub(self.initialGas, new(big.Int).SetUint64(self.gas)) +func (st *StateTransition) gasUsed() *big.Int { + return new(big.Int).Sub(st.initialGas, new(big.Int).SetUint64(st.gas)) } diff --git a/core/tx_list.go b/core/tx_list.go index d456f2a456..e48a8fb34c 100644 --- a/core/tx_list.go +++ b/core/tx_list.go @@ -22,7 +22,9 @@ import ( "math/big" "sort" + "github.com/expanse-org/go-expanse/common" "github.com/expanse-org/go-expanse/core/types" + "github.com/expanse-org/go-expanse/log" ) // nonceHeap is a heap.Interface implementation over 64bit unsigned integers for @@ -53,11 +55,11 @@ type txSortedMap struct { cache types.Transactions // Cache of the transactions already sorted } -// newTxSortedMap creates a new sorted transaction map. +// newTxSortedMap creates a new nonce-sorted transaction map. func newTxSortedMap() *txSortedMap { return &txSortedMap{ items: make(map[uint64]*types.Transaction), - index: &nonceHeap{}, + index: new(nonceHeap), } } @@ -218,9 +220,11 @@ func (m *txSortedMap) Flatten() types.Transactions { // the executable/pending queue; and for storing gapped transactions for the non- // executable/future queue, with minor behavioral changes. type txList struct { - strict bool // Whether nonces are strictly continuous or not - txs *txSortedMap // Heap indexed sorted hash map of the transactions - costcap *big.Int // Price of the highest costing transaction (reset only if exceeds balance) + strict bool // Whether nonces are strictly continuous or not + txs *txSortedMap // Heap indexed sorted hash map of the transactions + + costcap *big.Int // Price of the highest costing transaction (reset only if exceeds balance) + gascap *big.Int // Gas limit of the highest spending transaction (reset only if exceeds block limit) } // newTxList create a new transaction list for maintaining nonce-indexable fast, @@ -230,25 +234,38 @@ func newTxList(strict bool) *txList { strict: strict, txs: newTxSortedMap(), costcap: new(big.Int), + gascap: new(big.Int), } } +// Overlaps returns whether the transaction specified has the same nonce as one +// already contained within the list. +func (l *txList) Overlaps(tx *types.Transaction) bool { + return l.txs.Get(tx.Nonce()) != nil +} + // Add tries to insert a new transaction into the list, returning whether the // transaction was accepted, and if yes, any previous transaction it replaced. // -// If the new transaction is accepted into the list, the lists' cost threshold -// is also potentially updated. -func (l *txList) Add(tx *types.Transaction) (bool, *types.Transaction) { +// If the new transaction is accepted into the list, the lists' cost and gas +// thresholds are also potentially updated. +func (l *txList) Add(tx *types.Transaction, priceBump uint64) (bool, *types.Transaction) { // If there's an older better transaction, abort old := l.txs.Get(tx.Nonce()) - if old != nil && old.GasPrice().Cmp(tx.GasPrice()) >= 0 { - return false, nil + if old != nil { + threshold := new(big.Int).Div(new(big.Int).Mul(old.GasPrice(), big.NewInt(100+int64(priceBump))), big.NewInt(100)) + if threshold.Cmp(tx.GasPrice()) >= 0 { + return false, nil + } } // Otherwise overwrite the old transaction with the current one l.txs.Put(tx) if cost := tx.Cost(); l.costcap.Cmp(cost) < 0 { l.costcap = cost } + if gas := tx.Gas(); l.gascap.Cmp(gas) < 0 { + l.gascap = gas + } return true, old } @@ -259,23 +276,25 @@ func (l *txList) Forward(threshold uint64) types.Transactions { return l.txs.Forward(threshold) } -// Filter removes all transactions from the list with a cost higher than the -// provided threshold. Every removed transaction is returned for any post-removal -// maintenance. Strict-mode invalidated transactions are also returned. +// Filter removes all transactions from the list with a cost or gas limit higher +// than the provided thresholds. Every removed transaction is returned for any +// post-removal maintenance. Strict-mode invalidated transactions are also +// returned. // -// This method uses the cached costcap to quickly decide if there's even a point -// in calculating all the costs or if the balance covers all. If the threshold is -// lower than the costcap, the costcap will be reset to a new high after removing -// expensive the too transactions. -func (l *txList) Filter(threshold *big.Int) (types.Transactions, types.Transactions) { +// This method uses the cached costcap and gascap to quickly decide if there's even +// a point in calculating all the costs or if the balance covers all. If the threshold +// is lower than the costgas cap, the caps will be reset to a new high after removing +// the newly invalidated transactions. +func (l *txList) Filter(costLimit, gasLimit *big.Int) (types.Transactions, types.Transactions) { // If all transactions are below the threshold, short circuit - if l.costcap.Cmp(threshold) <= 0 { + if l.costcap.Cmp(costLimit) <= 0 && l.gascap.Cmp(gasLimit) <= 0 { return nil, nil } - l.costcap = new(big.Int).Set(threshold) // Lower the cap to the threshold + l.costcap = new(big.Int).Set(costLimit) // Lower the caps to the thresholds + l.gascap = new(big.Int).Set(gasLimit) // Filter out all the transactions above the account's funds - removed := l.txs.Filter(func(tx *types.Transaction) bool { return tx.Cost().Cmp(threshold) > 0 }) + removed := l.txs.Filter(func(tx *types.Transaction) bool { return tx.Cost().Cmp(costLimit) > 0 || tx.Gas().Cmp(gasLimit) > 0 }) // If the list was strict, filter anything above the lowest nonce var invalids types.Transactions @@ -340,3 +359,150 @@ func (l *txList) Empty() bool { func (l *txList) Flatten() types.Transactions { return l.txs.Flatten() } + +// priceHeap is a heap.Interface implementation over transactions for retrieving +// price-sorted transactions to discard when the pool fills up. +type priceHeap []*types.Transaction + +func (h priceHeap) Len() int { return len(h) } +func (h priceHeap) Less(i, j int) bool { return h[i].GasPrice().Cmp(h[j].GasPrice()) < 0 } +func (h priceHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } + +func (h *priceHeap) Push(x interface{}) { + *h = append(*h, x.(*types.Transaction)) +} + +func (h *priceHeap) Pop() interface{} { + old := *h + n := len(old) + x := old[n-1] + *h = old[0 : n-1] + return x +} + +// txPricedList is a price-sorted heap to allow operating on transactions pool +// contents in a price-incrementing way. +type txPricedList struct { + all *map[common.Hash]*types.Transaction // Pointer to the map of all transactions + items *priceHeap // Heap of prices of all the stored transactions + stales int // Number of stale price points to (re-heap trigger) +} + +// newTxPricedList creates a new price-sorted transaction heap. +func newTxPricedList(all *map[common.Hash]*types.Transaction) *txPricedList { + return &txPricedList{ + all: all, + items: new(priceHeap), + } +} + +// Put inserts a new transaction into the heap. +func (l *txPricedList) Put(tx *types.Transaction) { + heap.Push(l.items, tx) +} + +// Removed notifies the prices transaction list that an old transaction dropped +// from the pool. The list will just keep a counter of stale objects and update +// the heap if a large enough ratio of transactions go stale. +func (l *txPricedList) Removed() { + // Bump the stale counter, but exit if still too low (< 25%) + l.stales++ + if l.stales <= len(*l.items)/4 { + return + } + // Seems we've reached a critical number of stale transactions, reheap + reheap := make(priceHeap, 0, len(*l.all)) + + l.stales, l.items = 0, &reheap + for _, tx := range *l.all { + *l.items = append(*l.items, tx) + } + heap.Init(l.items) +} + +// Discard finds all the transactions below the given price threshold, drops them +// from the priced list and returs them for further removal from the entire pool. +func (l *txPricedList) Cap(threshold *big.Int, local *txSet) types.Transactions { + drop := make(types.Transactions, 0, 128) // Remote underpriced transactions to drop + save := make(types.Transactions, 0, 64) // Local underpriced transactions to keep + + for len(*l.items) > 0 { + // Discard stale transactions if found during cleanup + tx := heap.Pop(l.items).(*types.Transaction) + + hash := tx.Hash() + if _, ok := (*l.all)[hash]; !ok { + l.stales-- + continue + } + // Stop the discards if we've reached the threshold + if tx.GasPrice().Cmp(threshold) >= 0 { + break + } + // Non stale transaction found, discard unless local + if local.contains(hash) { + save = append(save, tx) + } else { + drop = append(drop, tx) + } + } + for _, tx := range save { + heap.Push(l.items, tx) + } + return drop +} + +// Underpriced checks whether a transaction is cheaper than (or as cheap as) the +// lowest priced transaction currently being tracked. +func (l *txPricedList) Underpriced(tx *types.Transaction, local *txSet) bool { + // Local transactions cannot be underpriced + if local.contains(tx.Hash()) { + return false + } + // Discard stale price points if found at the heap start + for len(*l.items) > 0 { + head := []*types.Transaction(*l.items)[0] + if _, ok := (*l.all)[head.Hash()]; !ok { + l.stales-- + heap.Pop(l.items) + continue + } + break + } + // Check if the transaction is underpriced or not + if len(*l.items) == 0 { + log.Error("Pricing query for empty pool") // This cannot happen, print to catch programming errors + return false + } + cheapest := []*types.Transaction(*l.items)[0] + return cheapest.GasPrice().Cmp(tx.GasPrice()) >= 0 +} + +// Discard finds a number of most underpriced transactions, removes them from the +// priced list and returs them for further removal from the entire pool. +func (l *txPricedList) Discard(count int, local *txSet) types.Transactions { + drop := make(types.Transactions, 0, count) // Remote underpriced transactions to drop + save := make(types.Transactions, 0, 64) // Local underpriced transactions to keep + + for len(*l.items) > 0 && count > 0 { + // Discard stale transactions if found during cleanup + tx := heap.Pop(l.items).(*types.Transaction) + + hash := tx.Hash() + if _, ok := (*l.all)[hash]; !ok { + l.stales-- + continue + } + // Non stale transaction found, discard unless local + if local.contains(hash) { + save = append(save, tx) + } else { + drop = append(drop, tx) + count-- + } + } + for _, tx := range save { + heap.Push(l.items, tx) + } + return drop +} diff --git a/core/tx_list_test.go b/core/tx_list_test.go index 6ba804d79e..6739811981 100644 --- a/core/tx_list_test.go +++ b/core/tx_list_test.go @@ -38,7 +38,7 @@ func TestStrictTxListAdd(t *testing.T) { // Insert the transactions in a random order list := newTxList(true) for _, v := range rand.Perm(len(txs)) { - list.Add(txs[v]) + list.Add(txs[v], DefaultTxPoolConfig.PriceBump) } // Verify internal state if len(list.txs.items) != len(txs) { diff --git a/core/tx_pool.go b/core/tx_pool.go index 71c5cf3ccf..afe0df7d7d 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -36,23 +36,20 @@ import ( var ( // Transaction Pool Errors - ErrInvalidSender = errors.New("Invalid sender") - ErrNonce = errors.New("Nonce too low") - ErrCheap = errors.New("Gas price too low for acceptance") - ErrBalance = errors.New("Insufficient balance") - ErrInsufficientFunds = errors.New("Insufficient funds for gas * price + value") - ErrIntrinsicGas = errors.New("Intrinsic gas too low") - ErrGasLimit = errors.New("Exceeds block gas limit") - ErrNegativeValue = errors.New("Negative value") + ErrInvalidSender = errors.New("invalid sender") + ErrNonce = errors.New("nonce too low") + ErrUnderpriced = errors.New("transaction underpriced") + ErrReplaceUnderpriced = errors.New("replacement transaction underpriced") + ErrBalance = errors.New("insufficient balance") + ErrInsufficientFunds = errors.New("insufficient funds for gas * price + value") + ErrIntrinsicGas = errors.New("intrinsic gas too low") + ErrGasLimit = errors.New("exceeds block gas limit") + ErrNegativeValue = errors.New("negative value") ) var ( - minPendingPerAccount = uint64(16) // Min number of guaranteed transaction slots per address - maxPendingTotal = uint64(4096) // Max limit of pending transactions from all accounts (soft) - maxQueuedPerAccount = uint64(64) // Max limit of queued transactions per address - maxQueuedInTotal = uint64(1024) // Max limit of queued transactions from all accounts - maxQueuedLifetime = 3 * time.Hour // Max amount of time transactions from idle accounts are queued - evictionInterval = time.Minute // Time interval to check for evictable transactions + evictionInterval = time.Minute // Time interval to check for evictable transactions + statsReportInterval = 8 * time.Second // Time interval to report transaction pool stats ) var ( @@ -69,11 +66,54 @@ var ( queuedNofundsCounter = metrics.NewCounter("txpool/queued/nofunds") // Dropped due to out-of-funds // General tx metrics - invalidTxCounter = metrics.NewCounter("txpool/invalid") + invalidTxCounter = metrics.NewCounter("txpool/invalid") + underpricedTxCounter = metrics.NewCounter("txpool/underpriced") ) type stateFn func() (*state.StateDB, error) +// TxPoolConfig are the configuration parameters of the transaction pool. +type TxPoolConfig struct { + PriceLimit uint64 // Minimum gas price to enforce for acceptance into the pool + PriceBump uint64 // Minimum price bump percentage to replace an already existing transaction (nonce) + + AccountSlots uint64 // Minimum number of executable transaction slots guaranteed per account + GlobalSlots uint64 // Maximum number of executable transaction slots for all accounts + AccountQueue uint64 // Maximum number of non-executable transaction slots permitted per account + GlobalQueue uint64 // Maximum number of non-executable transaction slots for all accounts + + Lifetime time.Duration // Maximum amount of time non-executable transaction are queued +} + +// DefaultTxPoolConfig contains the default configurations for the transaction +// pool. +var DefaultTxPoolConfig = TxPoolConfig{ + PriceLimit: 1, + PriceBump: 10, + + AccountSlots: 16, + GlobalSlots: 4096, + AccountQueue: 64, + GlobalQueue: 1024, + + Lifetime: 3 * time.Hour, +} + +// sanitize checks the provided user configurations and changes anything that's +// unreasonable or unworkable. +func (config *TxPoolConfig) sanitize() TxPoolConfig { + conf := *config + if conf.PriceLimit < 1 { + log.Warn("Sanitizing invalid txpool price limit", "provided", conf.PriceLimit, "updated", DefaultTxPoolConfig.PriceLimit) + conf.PriceLimit = DefaultTxPoolConfig.PriceLimit + } + if conf.PriceBump < 1 { + log.Warn("Sanitizing invalid txpool price bump", "provided", conf.PriceBump, "updated", DefaultTxPoolConfig.PriceBump) + conf.PriceBump = DefaultTxPoolConfig.PriceBump + } + return conf +} + // TxPool contains all currently known transactions. Transactions // enter the pool when they are received from the network or submitted // locally. They exit the pool when they are included in the blockchain. @@ -82,21 +122,23 @@ type stateFn func() (*state.StateDB, error) // current state) and future transactions. Transactions move between those // two states over time as they are received and processed. type TxPool struct { - config *params.ChainConfig + config TxPoolConfig + chainconfig *params.ChainConfig currentState stateFn // The state function which will allow us to do some pre checks pendingState *state.ManagedState gasLimit func() *big.Int // The current gas limit function callback - minGasPrice *big.Int + gasPrice *big.Int eventMux *event.TypeMux events *event.TypeMuxSubscription - localTx *txSet + locals *txSet signer types.Signer mu sync.RWMutex pending map[common.Address]*txList // All currently processable transactions queue map[common.Address]*txList // Queued but non-processable transactions - all map[common.Hash]*types.Transaction // All transactions to allow lookups beats map[common.Address]time.Time // Last heartbeat from each known account + all map[common.Hash]*types.Transaction // All transactions to allow lookups + priced *txPricedList // All transactions sorted by price wg sync.WaitGroup // for shutdown sync quit chan struct{} @@ -104,26 +146,34 @@ type TxPool struct { homestead bool } -func NewTxPool(config *params.ChainConfig, eventMux *event.TypeMux, currentStateFn stateFn, gasLimitFn func() *big.Int) *TxPool { +// NewTxPool creates a new transaction pool to gather, sort and filter inbound +// trnsactions from the network. +func NewTxPool(config TxPoolConfig, chainconfig *params.ChainConfig, eventMux *event.TypeMux, currentStateFn stateFn, gasLimitFn func() *big.Int) *TxPool { + // Sanitize the input to ensure no vulnerable gas prices are set + config = (&config).sanitize() + + // Create the transaction pool with its initial settings pool := &TxPool{ config: config, - signer: types.NewEIP155Signer(config.ChainId), + chainconfig: chainconfig, + signer: types.NewEIP155Signer(chainconfig.ChainId), pending: make(map[common.Address]*txList), queue: make(map[common.Address]*txList), - all: make(map[common.Hash]*types.Transaction), beats: make(map[common.Address]time.Time), + all: make(map[common.Hash]*types.Transaction), eventMux: eventMux, currentState: currentStateFn, gasLimit: gasLimitFn, - minGasPrice: new(big.Int), + gasPrice: new(big.Int).SetUint64(config.PriceLimit), pendingState: nil, - localTx: newTxSet(), - events: eventMux.Subscribe(ChainHeadEvent{}, GasPriceChanged{}, RemovedTransactionEvent{}), + locals: newTxSet(), + events: eventMux.Subscribe(ChainHeadEvent{}, RemovedTransactionEvent{}), quit: make(chan struct{}), } - + pool.priced = newTxPricedList(&pool.all) pool.resetState() + // Start the various events loops and return pool.wg.Add(2) go pool.eventLoop() go pool.expirationLoop() @@ -134,27 +184,48 @@ func NewTxPool(config *params.ChainConfig, eventMux *event.TypeMux, currentState func (pool *TxPool) eventLoop() { defer pool.wg.Done() + // Start a ticker and keep track of interesting pool stats to report + var prevPending, prevQueued, prevStales int + + report := time.NewTicker(statsReportInterval) + defer report.Stop() + // Track chain events. When a chain events occurs (new chain canon block) // we need to know the new state. The new state will help us determine // the nonces in the managed state - for ev := range pool.events.Chan() { - switch ev := ev.Data.(type) { - case ChainHeadEvent: - pool.mu.Lock() - if ev.Block != nil { - if pool.config.IsHomestead(ev.Block.Number()) { - pool.homestead = true + for { + select { + // Handle any events fired by the system + case ev, ok := <-pool.events.Chan(): + if !ok { + return + } + switch ev := ev.Data.(type) { + case ChainHeadEvent: + pool.mu.Lock() + if ev.Block != nil { + if pool.chainconfig.IsHomestead(ev.Block.Number()) { + pool.homestead = true + } } + pool.resetState() + pool.mu.Unlock() + + case RemovedTransactionEvent: + pool.AddBatch(ev.Txs) } - pool.resetState() - pool.mu.Unlock() - case GasPriceChanged: - pool.mu.Lock() - pool.minGasPrice = ev.Price - pool.mu.Unlock() - case RemovedTransactionEvent: - pool.AddBatch(ev.Txs) + // Handle stats reporting ticks + case <-report.C: + pool.mu.RLock() + pending, queued := pool.stats() + stales := pool.priced.stales + pool.mu.RUnlock() + + if pending != prevPending || queued != prevQueued || stales != prevStales { + log.Debug("Transaction pool status report", "executable", pending, "queued", queued, "stales", stales) + prevPending, prevQueued, prevStales = pending, queued, stales + } } } } @@ -180,9 +251,10 @@ func (pool *TxPool) resetState() { } // Check the queue and move transactions over to the pending if possible // or remove those that have become invalid - pool.promoteExecutables(currentState) + pool.promoteExecutables(currentState, nil) } +// Stop terminates the transaction pool. func (pool *TxPool) Stop() { pool.events.Unsubscribe() close(pool.quit) @@ -191,6 +263,28 @@ func (pool *TxPool) Stop() { log.Info("Transaction pool stopped") } +// GasPrice returns the current gas price enforced by the transaction pool. +func (pool *TxPool) GasPrice() *big.Int { + pool.mu.RLock() + defer pool.mu.RUnlock() + + return new(big.Int).Set(pool.gasPrice) +} + +// SetGasPrice updates the minimum price required by the transaction pool for a +// new transaction, and drops all transactions below this threshold. +func (pool *TxPool) SetGasPrice(price *big.Int) { + pool.mu.Lock() + defer pool.mu.Unlock() + + pool.gasPrice = price + for _, tx := range pool.priced.Cap(price, pool.locals) { + pool.removeTx(tx.Hash()) + } + log.Info("Transaction pool price threshold updated", "price", price) +} + +// State returns the virtual managed state of the transaction pool. func (pool *TxPool) State() *state.ManagedState { pool.mu.RLock() defer pool.mu.RUnlock() @@ -200,17 +294,25 @@ func (pool *TxPool) State() *state.ManagedState { // Stats retrieves the current pool stats, namely the number of pending and the // number of queued (non-executable) transactions. -func (pool *TxPool) Stats() (pending int, queued int) { +func (pool *TxPool) Stats() (int, int) { pool.mu.RLock() defer pool.mu.RUnlock() + return pool.stats() +} + +// stats retrieves the current pool stats, namely the number of pending and the +// number of queued (non-executable) transactions. +func (pool *TxPool) stats() (int, int) { + pending := 0 for _, list := range pool.pending { pending += list.Len() } + queued := 0 for _, list := range pool.queue { queued += list.Len() } - return + return pending, queued } // Content retrieves the data content of the transaction pool, returning all the @@ -237,17 +339,6 @@ func (pool *TxPool) Pending() (map[common.Address]types.Transactions, error) { pool.mu.Lock() defer pool.mu.Unlock() - state, err := pool.currentState() - if err != nil { - return nil, err - } - - // check queue first - pool.promoteExecutables(state) - - // invalidate any txs - pool.demoteUnexecutables(state) - pending := make(map[common.Address]types.Transactions) for addr, list := range pool.pending { pending[addr] = list.Flatten() @@ -260,16 +351,16 @@ func (pool *TxPool) Pending() (map[common.Address]types.Transactions, error) { func (pool *TxPool) SetLocal(tx *types.Transaction) { pool.mu.Lock() defer pool.mu.Unlock() - pool.localTx.add(tx.Hash()) + pool.locals.add(tx.Hash()) } // validateTx checks whether a transaction is valid according // to the consensus rules. func (pool *TxPool) validateTx(tx *types.Transaction) error { - local := pool.localTx.contains(tx.Hash()) + local := pool.locals.contains(tx.Hash()) // Drop transactions under our own minimal accepted gas price - if !local && pool.minGasPrice.Cmp(tx.GasPrice()) > 0 { - return ErrCheap + if !local && pool.gasPrice.Cmp(tx.GasPrice()) > 0 { + return ErrUnderpriced } currentState, err := pool.currentState() @@ -314,47 +405,92 @@ func (pool *TxPool) validateTx(tx *types.Transaction) error { } // add validates a transaction and inserts it into the non-executable queue for -// later pending promotion and execution. -func (pool *TxPool) add(tx *types.Transaction) error { +// later pending promotion and execution. If the transaction is a replacement for +// an already pending or queued one, it overwrites the previous and returns this +// so outer code doesn't uselessly call promote. +func (pool *TxPool) add(tx *types.Transaction) (bool, error) { // If the transaction is already known, discard it hash := tx.Hash() if pool.all[hash] != nil { log.Trace("Discarding already known transaction", "hash", hash) - return fmt.Errorf("known transaction: %x", hash) + return false, fmt.Errorf("known transaction: %x", hash) } - // Otherwise ensure basic validation passes and queue it up + // If the transaction fails basic validation, discard it if err := pool.validateTx(tx); err != nil { log.Trace("Discarding invalid transaction", "hash", hash, "err", err) invalidTxCounter.Inc(1) - return err + return false, err } - pool.enqueueTx(hash, tx) + // If the transaction pool is full, discard underpriced transactions + if uint64(len(pool.all)) >= pool.config.GlobalSlots+pool.config.GlobalQueue { + // If the new transaction is underpriced, don't accept it + if pool.priced.Underpriced(tx, pool.locals) { + log.Trace("Discarding underpriced transaction", "hash", hash, "price", tx.GasPrice()) + underpricedTxCounter.Inc(1) + return false, ErrUnderpriced + } + // New transaction is better than our worse ones, make room for it + drop := pool.priced.Discard(len(pool.all)-int(pool.config.GlobalSlots+pool.config.GlobalQueue-1), pool.locals) + for _, tx := range drop { + log.Trace("Discarding freshly underpriced transaction", "hash", tx.Hash(), "price", tx.GasPrice()) + underpricedTxCounter.Inc(1) + pool.removeTx(tx.Hash()) + } + } + // If the transaction is replacing an already pending one, do directly + from, _ := types.Sender(pool.signer, tx) // already validated + if list := pool.pending[from]; list != nil && list.Overlaps(tx) { + // Nonce already pending, check if required price bump is met + inserted, old := list.Add(tx, pool.config.PriceBump) + if !inserted { + pendingDiscardCounter.Inc(1) + return false, ErrReplaceUnderpriced + } + // New transaction is better, replace old one + if old != nil { + delete(pool.all, old.Hash()) + pool.priced.Removed() + pendingReplaceCounter.Inc(1) + } + pool.all[tx.Hash()] = tx + pool.priced.Put(tx) - // Print a log message if low enough level is set - log.Debug("Pooled new transaction", "hash", hash, "from", log.Lazy{Fn: func() common.Address { from, _ := types.Sender(pool.signer, tx); return from }}, "to", tx.To()) - return nil + log.Trace("Pooled new executable transaction", "hash", hash, "from", from, "to", tx.To()) + return old != nil, nil + } + // New transaction isn't replacing a pending one, push into queue + replace, err := pool.enqueueTx(hash, tx) + if err != nil { + return false, err + } + log.Trace("Pooled new future transaction", "hash", hash, "from", from, "to", tx.To()) + return replace, nil } // enqueueTx inserts a new transaction into the non-executable transaction queue. // // Note, this method assumes the pool lock is held! -func (pool *TxPool) enqueueTx(hash common.Hash, tx *types.Transaction) { +func (pool *TxPool) enqueueTx(hash common.Hash, tx *types.Transaction) (bool, error) { // Try to insert the transaction into the future queue from, _ := types.Sender(pool.signer, tx) // already validated if pool.queue[from] == nil { pool.queue[from] = newTxList(false) } - inserted, old := pool.queue[from].Add(tx) + inserted, old := pool.queue[from].Add(tx, pool.config.PriceBump) if !inserted { + // An older transaction was better, discard this queuedDiscardCounter.Inc(1) - return // An older transaction was better, discard this + return false, ErrReplaceUnderpriced } // Discard any previous transaction and mark this if old != nil { delete(pool.all, old.Hash()) + pool.priced.Removed() queuedReplaceCounter.Inc(1) } pool.all[hash] = tx + pool.priced.Put(tx) + return old != nil, nil } // promoteTx adds a transaction to the pending (processable) list of transactions. @@ -367,20 +503,27 @@ func (pool *TxPool) promoteTx(addr common.Address, hash common.Hash, tx *types.T } list := pool.pending[addr] - inserted, old := list.Add(tx) + inserted, old := list.Add(tx, pool.config.PriceBump) if !inserted { // An older transaction was better, discard this delete(pool.all, hash) + pool.priced.Removed() + pendingDiscardCounter.Inc(1) return } // Otherwise discard any previous transaction and mark this if old != nil { delete(pool.all, old.Hash()) + pool.priced.Removed() + pendingReplaceCounter.Inc(1) } - pool.all[hash] = tx // Failsafe to work around direct pending inserts (tests) - + // Failsafe to work around direct pending inserts (tests) + if pool.all[hash] == nil { + pool.all[hash] = tx + pool.priced.Put(tx) + } // Set the potentially new pending nonce and notify any subsystems of the new tx pool.beats[addr] = time.Now() pool.pendingState.SetNonce(addr, tx.Nonce()+1) @@ -392,16 +535,20 @@ func (pool *TxPool) Add(tx *types.Transaction) error { pool.mu.Lock() defer pool.mu.Unlock() - if err := pool.add(tx); err != nil { - return err - } - - state, err := pool.currentState() + // Try to inject the transaction and update any state + replace, err := pool.add(tx) if err != nil { return err } - pool.promoteExecutables(state) - + // If we added a new transaction, run promotion checks and return + if !replace { + state, err := pool.currentState() + if err != nil { + return err + } + from, _ := types.Sender(pool.signer, tx) // already validated + pool.promoteExecutables(state, []common.Address{from}) + } return nil } @@ -411,19 +558,26 @@ func (pool *TxPool) AddBatch(txs []*types.Transaction) error { defer pool.mu.Unlock() // Add the batch of transaction, tracking the accepted ones - added := 0 + dirty := make(map[common.Address]struct{}) for _, tx := range txs { - if err := pool.add(tx); err == nil { - added++ + if replace, err := pool.add(tx); err == nil { + if !replace { + from, _ := types.Sender(pool.signer, tx) // already validated + dirty[from] = struct{}{} + } } } // Only reprocess the internal state if something was actually added - if added > 0 { + if len(dirty) > 0 { state, err := pool.currentState() if err != nil { return err } - pool.promoteExecutables(state) + addrs := make([]common.Address, 0, len(dirty)) + for addr, _ := range dirty { + addrs = append(addrs, addr) + } + pool.promoteExecutables(state, addrs) } return nil } @@ -467,6 +621,7 @@ func (pool *TxPool) removeTx(hash common.Hash) { // Remove it from the list of known transactions delete(pool.all, hash) + pool.priced.Removed() // Remove the transaction from the pending lists and reset the account nonce if pending := pool.pending[addr]; pending != nil { @@ -499,35 +654,51 @@ func (pool *TxPool) removeTx(hash common.Hash) { // promoteExecutables moves transactions that have become processable from the // future queue to the set of pending transactions. During this process, all // invalidated transactions (low nonce, low balance) are deleted. -func (pool *TxPool) promoteExecutables(state *state.StateDB) { +func (pool *TxPool) promoteExecutables(state *state.StateDB, accounts []common.Address) { + gaslimit := pool.gasLimit() + + // Gather all the accounts potentially needing updates + if accounts == nil { + accounts = make([]common.Address, 0, len(pool.queue)) + for addr, _ := range pool.queue { + accounts = append(accounts, addr) + } + } // Iterate over all accounts and promote any executable transactions queued := uint64(0) - for addr, list := range pool.queue { + for _, addr := range accounts { + list := pool.queue[addr] + if list == nil { + continue // Just in case someone calls with a non existing account + } // Drop all transactions that are deemed too old (low nonce) for _, tx := range list.Forward(state.GetNonce(addr)) { hash := tx.Hash() - log.Debug("Removed old queued transaction", "hash", hash) + log.Trace("Removed old queued transaction", "hash", hash) delete(pool.all, hash) + pool.priced.Removed() } - // Drop all transactions that are too costly (low balance) - drops, _ := list.Filter(state.GetBalance(addr)) + // Drop all transactions that are too costly (low balance or out of gas) + drops, _ := list.Filter(state.GetBalance(addr), gaslimit) for _, tx := range drops { hash := tx.Hash() - log.Debug("Removed unpayable queued transaction", "hash", hash) + log.Trace("Removed unpayable queued transaction", "hash", hash) delete(pool.all, hash) + pool.priced.Removed() queuedNofundsCounter.Inc(1) } // Gather all executable transactions and promote them for _, tx := range list.Ready(pool.pendingState.GetNonce(addr)) { hash := tx.Hash() - log.Debug("Promoting queued transaction", "hash", hash) + log.Trace("Promoting queued transaction", "hash", hash) pool.promoteTx(addr, hash, tx) } // Drop all transactions over the allowed limit - for _, tx := range list.Cap(int(maxQueuedPerAccount)) { + for _, tx := range list.Cap(int(pool.config.AccountQueue)) { hash := tx.Hash() - log.Debug("Removed cap-exceeding queued transaction", "hash", hash) + log.Trace("Removed cap-exceeding queued transaction", "hash", hash) delete(pool.all, hash) + pool.priced.Removed() queuedRLCounter.Inc(1) } queued += uint64(list.Len()) @@ -542,16 +713,16 @@ func (pool *TxPool) promoteExecutables(state *state.StateDB) { for _, list := range pool.pending { pending += uint64(list.Len()) } - if pending > maxPendingTotal { + if pending > pool.config.GlobalSlots { pendingBeforeCap := pending // Assemble a spam order to penalize large transactors first spammers := prque.New() for addr, list := range pool.pending { // Only evict transactions from high rollers - if uint64(list.Len()) > minPendingPerAccount { + if uint64(list.Len()) > pool.config.AccountSlots { // Skip local accounts as pools should maintain backlogs for themselves for _, tx := range list.txs.items { - if !pool.localTx.contains(tx.Hash()) { + if !pool.locals.contains(tx.Hash()) { spammers.Push(addr, float32(list.Len())) } break // Checking on transaction for locality is enough @@ -560,7 +731,7 @@ func (pool *TxPool) promoteExecutables(state *state.StateDB) { } // Gradually drop transactions from offenders offenders := []common.Address{} - for pending > maxPendingTotal && !spammers.Empty() { + for pending > pool.config.GlobalSlots && !spammers.Empty() { // Retrieve the next offender if not local address offender, _ := spammers.Pop() offenders = append(offenders, offender.(common.Address)) @@ -571,7 +742,7 @@ func (pool *TxPool) promoteExecutables(state *state.StateDB) { threshold := pool.pending[offender.(common.Address)].Len() // Iteratively reduce all offenders until below limit or threshold reached - for pending > maxPendingTotal && pool.pending[offenders[len(offenders)-2]].Len() > threshold { + for pending > pool.config.GlobalSlots && pool.pending[offenders[len(offenders)-2]].Len() > threshold { for i := 0; i < len(offenders)-1; i++ { list := pool.pending[offenders[i]] list.Cap(list.Len() - 1) @@ -581,8 +752,8 @@ func (pool *TxPool) promoteExecutables(state *state.StateDB) { } } // If still above threshold, reduce to limit or min allowance - if pending > maxPendingTotal && len(offenders) > 0 { - for pending > maxPendingTotal && uint64(pool.pending[offenders[len(offenders)-1]].Len()) > minPendingPerAccount { + if pending > pool.config.GlobalSlots && len(offenders) > 0 { + for pending > pool.config.GlobalSlots && uint64(pool.pending[offenders[len(offenders)-1]].Len()) > pool.config.AccountSlots { for _, addr := range offenders { list := pool.pending[addr] list.Cap(list.Len() - 1) @@ -593,7 +764,7 @@ func (pool *TxPool) promoteExecutables(state *state.StateDB) { pendingRLCounter.Inc(int64(pendingBeforeCap - pending)) } // If we've queued more transactions than the hard limit, drop oldest ones - if queued > maxQueuedInTotal { + if queued > pool.config.GlobalQueue { // Sort all accounts with queued transactions by heartbeat addresses := make(addresssByHeartbeat, 0, len(pool.queue)) for addr := range pool.queue { @@ -602,7 +773,7 @@ func (pool *TxPool) promoteExecutables(state *state.StateDB) { sort.Sort(addresses) // Drop transactions until the total is below the limit - for drop := queued - maxQueuedInTotal; drop > 0; { + for drop := queued - pool.config.GlobalQueue; drop > 0; { addr := addresses[len(addresses)-1] list := pool.queue[addr.address] @@ -632,6 +803,8 @@ func (pool *TxPool) promoteExecutables(state *state.StateDB) { // executable/pending queue and any subsequent transactions that become unexecutable // are moved back into the future queue. func (pool *TxPool) demoteUnexecutables(state *state.StateDB) { + gaslimit := pool.gasLimit() + // Iterate over all accounts and demote any non-executable transactions for addr, list := range pool.pending { nonce := state.GetNonce(addr) @@ -639,20 +812,22 @@ func (pool *TxPool) demoteUnexecutables(state *state.StateDB) { // Drop all transactions that are deemed too old (low nonce) for _, tx := range list.Forward(nonce) { hash := tx.Hash() - log.Debug("Removed old pending transaction", "hash", hash) + log.Trace("Removed old pending transaction", "hash", hash) delete(pool.all, hash) + pool.priced.Removed() } - // Drop all transactions that are too costly (low balance), and queue any invalids back for later - drops, invalids := list.Filter(state.GetBalance(addr)) + // Drop all transactions that are too costly (low balance or out of gas), and queue any invalids back for later + drops, invalids := list.Filter(state.GetBalance(addr), gaslimit) for _, tx := range drops { hash := tx.Hash() - log.Debug("Removed unpayable pending transaction", "hash", hash) + log.Trace("Removed unpayable pending transaction", "hash", hash) delete(pool.all, hash) + pool.priced.Removed() pendingNofundsCounter.Inc(1) } for _, tx := range invalids { hash := tx.Hash() - log.Debug("Demoting pending transaction", "hash", hash) + log.Trace("Demoting pending transaction", "hash", hash) pool.enqueueTx(hash, tx) } // Delete the entire queue entry if it became empty. @@ -677,7 +852,7 @@ func (pool *TxPool) expirationLoop() { case <-evict.C: pool.mu.Lock() for addr := range pool.queue { - if time.Since(pool.beats[addr]) > maxQueuedLifetime { + if time.Since(pool.beats[addr]) > pool.config.Lifetime { for _, tx := range pool.queue[addr].Flatten() { pool.removeTx(tx.Hash()) } @@ -729,22 +904,22 @@ func newTxSet() *txSet { // contains returns true if the set contains the given transaction hash // (not thread safe, should be called from a locked environment) -func (self *txSet) contains(hash common.Hash) bool { - _, ok := self.txMap[hash] +func (ts *txSet) contains(hash common.Hash) bool { + _, ok := ts.txMap[hash] return ok } // add adds a transaction hash to the set, then removes entries older than txSetDuration // (not thread safe, should be called from a locked environment) -func (self *txSet) add(hash common.Hash) { - self.txMap[hash] = struct{}{} +func (ts *txSet) add(hash common.Hash) { + ts.txMap[hash] = struct{}{} now := time.Now() - self.txOrd[self.addPtr] = txOrdType{hash: hash, time: now} - self.addPtr++ + ts.txOrd[ts.addPtr] = txOrdType{hash: hash, time: now} + ts.addPtr++ delBefore := now.Add(-txSetDuration) - for self.delPtr < self.addPtr && self.txOrd[self.delPtr].time.Before(delBefore) { - delete(self.txMap, self.txOrd[self.delPtr].hash) - delete(self.txOrd, self.delPtr) - self.delPtr++ + for ts.delPtr < ts.addPtr && ts.txOrd[ts.delPtr].time.Before(delBefore) { + delete(ts.txMap, ts.txOrd[ts.delPtr].hash) + delete(ts.txOrd, ts.delPtr) + ts.delPtr++ } } diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index 852fe9a8ed..f40fa92a9c 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -33,7 +33,11 @@ import ( ) func transaction(nonce uint64, gaslimit *big.Int, key *ecdsa.PrivateKey) *types.Transaction { - tx, _ := types.SignTx(types.NewTransaction(nonce, common.Address{}, big.NewInt(100), gaslimit, big.NewInt(1), nil), types.HomesteadSigner{}, key) + return pricedTransaction(nonce, gaslimit, big.NewInt(1), key) +} + +func pricedTransaction(nonce uint64, gaslimit, gasprice *big.Int, key *ecdsa.PrivateKey) *types.Transaction { + tx, _ := types.SignTx(types.NewTransaction(nonce, common.Address{}, big.NewInt(100), gaslimit, gasprice, nil), types.HomesteadSigner{}, key) return tx } @@ -42,7 +46,7 @@ func setupTxPool() (*TxPool, *ecdsa.PrivateKey) { statedb, _ := state.New(common.Hash{}, db) key, _ := crypto.GenerateKey() - newPool := NewTxPool(params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) + newPool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) newPool.resetState() return newPool, key @@ -91,7 +95,7 @@ func TestStateChangeDuringPoolReset(t *testing.T) { gasLimitFunc := func() *big.Int { return big.NewInt(1000000000) } - txpool := NewTxPool(params.TestChainConfig, mux, stateFunc, gasLimitFunc) + txpool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, mux, stateFunc, gasLimitFunc) txpool.resetState() nonce := txpool.State().GetNonce(address) @@ -151,9 +155,9 @@ func TestInvalidTransactions(t *testing.T) { } tx = transaction(1, big.NewInt(100000), key) - pool.minGasPrice = big.NewInt(1000) - if err := pool.Add(tx); err != ErrCheap { - t.Error("expected", ErrCheap, "got", err) + pool.gasPrice = big.NewInt(1000) + if err := pool.Add(tx); err != ErrUnderpriced { + t.Error("expected", ErrUnderpriced, "got", err) } pool.SetLocal(tx) @@ -171,7 +175,7 @@ func TestTransactionQueue(t *testing.T) { pool.resetState() pool.enqueueTx(tx.Hash(), tx) - pool.promoteExecutables(currentState) + pool.promoteExecutables(currentState, []common.Address{from}) if len(pool.pending) != 1 { t.Error("expected valid txs to be 1 is", len(pool.pending)) } @@ -180,7 +184,7 @@ func TestTransactionQueue(t *testing.T) { from, _ = deriveSender(tx) currentState.SetNonce(from, 2) pool.enqueueTx(tx.Hash(), tx) - pool.promoteExecutables(currentState) + pool.promoteExecutables(currentState, []common.Address{from}) if _, ok := pool.pending[from].txs.items[tx.Nonce()]; ok { t.Error("expected transaction to be in tx pool") } @@ -202,7 +206,7 @@ func TestTransactionQueue(t *testing.T) { pool.enqueueTx(tx2.Hash(), tx2) pool.enqueueTx(tx3.Hash(), tx3) - pool.promoteExecutables(currentState) + pool.promoteExecutables(currentState, []common.Address{from}) if len(pool.pending) != 1 { t.Error("expected tx pool to be 1, got", len(pool.pending)) @@ -262,14 +266,14 @@ func TestTransactionChainFork(t *testing.T) { resetState() tx := transaction(0, big.NewInt(100000), key) - if err := pool.add(tx); err != nil { + if _, err := pool.add(tx); err != nil { t.Error("didn't expect error", err) } pool.RemoveBatch([]*types.Transaction{tx}) // reset the pool's internal state resetState() - if err := pool.add(tx); err != nil { + if _, err := pool.add(tx); err != nil { t.Error("didn't expect error", err) } } @@ -293,25 +297,23 @@ func TestTransactionDoubleNonce(t *testing.T) { tx3, _ := types.SignTx(types.NewTransaction(0, common.Address{}, big.NewInt(100), big.NewInt(1000000), big.NewInt(1), nil), signer, key) // Add the first two transaction, ensure higher priced stays only - if err := pool.add(tx1); err != nil { - t.Error("didn't expect error", err) + if replace, err := pool.add(tx1); err != nil || replace { + t.Errorf("first transaction insert failed (%v) or reported replacement (%v)", err, replace) } - if err := pool.add(tx2); err != nil { - t.Error("didn't expect error", err) + if replace, err := pool.add(tx2); err != nil || !replace { + t.Errorf("second transaction insert failed (%v) or not reported replacement (%v)", err, replace) } state, _ := pool.currentState() - pool.promoteExecutables(state) + pool.promoteExecutables(state, []common.Address{addr}) if pool.pending[addr].Len() != 1 { t.Error("expected 1 pending transactions, got", pool.pending[addr].Len()) } if tx := pool.pending[addr].txs.items[0]; tx.Hash() != tx2.Hash() { t.Errorf("transaction mismatch: have %x, want %x", tx.Hash(), tx2.Hash()) } - // Add the thid transaction and ensure it's not saved (smaller price) - if err := pool.add(tx3); err != nil { - t.Error("didn't expect error", err) - } - pool.promoteExecutables(state) + // Add the third transaction and ensure it's not saved (smaller price) + pool.add(tx3) + pool.promoteExecutables(state, []common.Address{addr}) if pool.pending[addr].Len() != 1 { t.Error("expected 1 pending transactions, got", pool.pending[addr].Len()) } @@ -330,7 +332,7 @@ func TestMissingNonce(t *testing.T) { currentState, _ := pool.currentState() currentState.AddBalance(addr, big.NewInt(100000000000000)) tx := transaction(1, big.NewInt(100000), key) - if err := pool.add(tx); err != nil { + if _, err := pool.add(tx); err != nil { t.Error("didn't expect error", err) } if len(pool.pending) != 0 { @@ -395,49 +397,78 @@ func TestTransactionDropping(t *testing.T) { var ( tx0 = transaction(0, big.NewInt(100), key) tx1 = transaction(1, big.NewInt(200), key) + tx2 = transaction(2, big.NewInt(300), key) tx10 = transaction(10, big.NewInt(100), key) tx11 = transaction(11, big.NewInt(200), key) + tx12 = transaction(12, big.NewInt(300), key) ) pool.promoteTx(account, tx0.Hash(), tx0) pool.promoteTx(account, tx1.Hash(), tx1) + pool.promoteTx(account, tx1.Hash(), tx2) pool.enqueueTx(tx10.Hash(), tx10) pool.enqueueTx(tx11.Hash(), tx11) + pool.enqueueTx(tx11.Hash(), tx12) // Check that pre and post validations leave the pool as is - if pool.pending[account].Len() != 2 { - t.Errorf("pending transaction mismatch: have %d, want %d", pool.pending[account].Len(), 2) + if pool.pending[account].Len() != 3 { + t.Errorf("pending transaction mismatch: have %d, want %d", pool.pending[account].Len(), 3) } - if pool.queue[account].Len() != 2 { - t.Errorf("queued transaction mismatch: have %d, want %d", pool.queue[account].Len(), 2) + if pool.queue[account].Len() != 3 { + t.Errorf("queued transaction mismatch: have %d, want %d", pool.queue[account].Len(), 3) } if len(pool.all) != 4 { t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), 4) } pool.resetState() - if pool.pending[account].Len() != 2 { - t.Errorf("pending transaction mismatch: have %d, want %d", pool.pending[account].Len(), 2) + if pool.pending[account].Len() != 3 { + t.Errorf("pending transaction mismatch: have %d, want %d", pool.pending[account].Len(), 3) } - if pool.queue[account].Len() != 2 { - t.Errorf("queued transaction mismatch: have %d, want %d", pool.queue[account].Len(), 2) + if pool.queue[account].Len() != 3 { + t.Errorf("queued transaction mismatch: have %d, want %d", pool.queue[account].Len(), 3) } if len(pool.all) != 4 { t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), 4) } // Reduce the balance of the account, and check that invalidated transactions are dropped - state.AddBalance(account, big.NewInt(-750)) + state.AddBalance(account, big.NewInt(-650)) + pool.resetState() + + if _, ok := pool.pending[account].txs.items[tx0.Nonce()]; !ok { + t.Errorf("funded pending transaction missing: %v", tx0) + } + if _, ok := pool.pending[account].txs.items[tx1.Nonce()]; !ok { + t.Errorf("funded pending transaction missing: %v", tx0) + } + if _, ok := pool.pending[account].txs.items[tx2.Nonce()]; ok { + t.Errorf("out-of-fund pending transaction present: %v", tx1) + } + if _, ok := pool.queue[account].txs.items[tx10.Nonce()]; !ok { + t.Errorf("funded queued transaction missing: %v", tx10) + } + if _, ok := pool.queue[account].txs.items[tx11.Nonce()]; !ok { + t.Errorf("funded queued transaction missing: %v", tx10) + } + if _, ok := pool.queue[account].txs.items[tx12.Nonce()]; ok { + t.Errorf("out-of-fund queued transaction present: %v", tx11) + } + if len(pool.all) != 4 { + t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), 4) + } + // Reduce the block gas limit, check that invalidated transactions are dropped + pool.gasLimit = func() *big.Int { return big.NewInt(100) } pool.resetState() if _, ok := pool.pending[account].txs.items[tx0.Nonce()]; !ok { t.Errorf("funded pending transaction missing: %v", tx0) } if _, ok := pool.pending[account].txs.items[tx1.Nonce()]; ok { - t.Errorf("out-of-fund pending transaction present: %v", tx1) + t.Errorf("over-gased pending transaction present: %v", tx1) } if _, ok := pool.queue[account].txs.items[tx10.Nonce()]; !ok { t.Errorf("funded queued transaction missing: %v", tx10) } if _, ok := pool.queue[account].txs.items[tx11.Nonce()]; ok { - t.Errorf("out-of-fund queued transaction present: %v", tx11) + t.Errorf("over-gased queued transaction present: %v", tx11) } if len(pool.all) != 2 { t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), 2) @@ -531,25 +562,25 @@ func TestTransactionQueueAccountLimiting(t *testing.T) { pool.resetState() // Keep queuing up transactions and make sure all above a limit are dropped - for i := uint64(1); i <= maxQueuedPerAccount+5; i++ { + for i := uint64(1); i <= DefaultTxPoolConfig.AccountQueue+5; i++ { if err := pool.Add(transaction(i, big.NewInt(100000), key)); err != nil { t.Fatalf("tx %d: failed to add transaction: %v", i, err) } if len(pool.pending) != 0 { t.Errorf("tx %d: pending pool size mismatch: have %d, want %d", i, len(pool.pending), 0) } - if i <= maxQueuedPerAccount { + if i <= DefaultTxPoolConfig.AccountQueue { if pool.queue[account].Len() != int(i) { t.Errorf("tx %d: queue size mismatch: have %d, want %d", i, pool.queue[account].Len(), i) } } else { - if pool.queue[account].Len() != int(maxQueuedPerAccount) { - t.Errorf("tx %d: queue limit mismatch: have %d, want %d", i, pool.queue[account].Len(), maxQueuedPerAccount) + if pool.queue[account].Len() != int(DefaultTxPoolConfig.AccountQueue) { + t.Errorf("tx %d: queue limit mismatch: have %d, want %d", i, pool.queue[account].Len(), DefaultTxPoolConfig.AccountQueue) } } } - if len(pool.all) != int(maxQueuedPerAccount) { - t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), maxQueuedPerAccount) + if len(pool.all) != int(DefaultTxPoolConfig.AccountQueue) { + t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), DefaultTxPoolConfig.AccountQueue) } } @@ -557,14 +588,14 @@ func TestTransactionQueueAccountLimiting(t *testing.T) { // some threshold, the higher transactions are dropped to prevent DOS attacks. func TestTransactionQueueGlobalLimiting(t *testing.T) { // Reduce the queue limits to shorten test time - defer func(old uint64) { maxQueuedInTotal = old }(maxQueuedInTotal) - maxQueuedInTotal = maxQueuedPerAccount * 3 + defer func(old uint64) { DefaultTxPoolConfig.GlobalQueue = old }(DefaultTxPoolConfig.GlobalQueue) + DefaultTxPoolConfig.GlobalQueue = DefaultTxPoolConfig.AccountQueue * 3 // Create the pool to test the limit enforcement with db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, db) - pool := NewTxPool(params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) + pool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) pool.resetState() // Create a number of test accounts and fund them @@ -578,7 +609,7 @@ func TestTransactionQueueGlobalLimiting(t *testing.T) { // Generate and queue a batch of transactions nonces := make(map[common.Address]uint64) - txs := make(types.Transactions, 0, 3*maxQueuedInTotal) + txs := make(types.Transactions, 0, 3*DefaultTxPoolConfig.GlobalQueue) for len(txs) < cap(txs) { key := keys[rand.Intn(len(keys))] addr := crypto.PubkeyToAddress(key.PublicKey) @@ -591,13 +622,13 @@ func TestTransactionQueueGlobalLimiting(t *testing.T) { queued := 0 for addr, list := range pool.queue { - if list.Len() > int(maxQueuedPerAccount) { - t.Errorf("addr %x: queued accounts overflown allowance: %d > %d", addr, list.Len(), maxQueuedPerAccount) + if list.Len() > int(DefaultTxPoolConfig.AccountQueue) { + t.Errorf("addr %x: queued accounts overflown allowance: %d > %d", addr, list.Len(), DefaultTxPoolConfig.AccountQueue) } queued += list.Len() } - if queued > int(maxQueuedInTotal) { - t.Fatalf("total transactions overflow allowance: %d > %d", queued, maxQueuedInTotal) + if queued > int(DefaultTxPoolConfig.GlobalQueue) { + t.Fatalf("total transactions overflow allowance: %d > %d", queued, DefaultTxPoolConfig.GlobalQueue) } } @@ -606,9 +637,9 @@ func TestTransactionQueueGlobalLimiting(t *testing.T) { // on shuffling them around. func TestTransactionQueueTimeLimiting(t *testing.T) { // Reduce the queue limits to shorten test time - defer func(old time.Duration) { maxQueuedLifetime = old }(maxQueuedLifetime) + defer func(old time.Duration) { DefaultTxPoolConfig.Lifetime = old }(DefaultTxPoolConfig.Lifetime) defer func(old time.Duration) { evictionInterval = old }(evictionInterval) - maxQueuedLifetime = time.Second + DefaultTxPoolConfig.Lifetime = time.Second evictionInterval = time.Second // Create a test account and fund it @@ -619,7 +650,7 @@ func TestTransactionQueueTimeLimiting(t *testing.T) { state.AddBalance(account, big.NewInt(1000000)) // Queue up a batch of transactions - for i := uint64(1); i <= maxQueuedPerAccount; i++ { + for i := uint64(1); i <= DefaultTxPoolConfig.AccountQueue; i++ { if err := pool.Add(transaction(i, big.NewInt(100000), key)); err != nil { t.Fatalf("tx %d: failed to add transaction: %v", i, err) } @@ -644,7 +675,7 @@ func TestTransactionPendingLimiting(t *testing.T) { pool.resetState() // Keep queuing up transactions and make sure all above a limit are dropped - for i := uint64(0); i < maxQueuedPerAccount+5; i++ { + for i := uint64(0); i < DefaultTxPoolConfig.AccountQueue+5; i++ { if err := pool.Add(transaction(i, big.NewInt(100000), key)); err != nil { t.Fatalf("tx %d: failed to add transaction: %v", i, err) } @@ -655,8 +686,8 @@ func TestTransactionPendingLimiting(t *testing.T) { t.Errorf("tx %d: queue size mismatch: have %d, want %d", i, pool.queue[account].Len(), 0) } } - if len(pool.all) != int(maxQueuedPerAccount+5) { - t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), maxQueuedPerAccount+5) + if len(pool.all) != int(DefaultTxPoolConfig.AccountQueue+5) { + t.Errorf("total transaction mismatch: have %d, want %d", len(pool.all), DefaultTxPoolConfig.AccountQueue+5) } } @@ -672,7 +703,7 @@ func testTransactionLimitingEquivalency(t *testing.T, origin uint64) { state1, _ := pool1.currentState() state1.AddBalance(account1, big.NewInt(1000000)) - for i := uint64(0); i < maxQueuedPerAccount+5; i++ { + for i := uint64(0); i < DefaultTxPoolConfig.AccountQueue+5; i++ { if err := pool1.Add(transaction(origin+i, big.NewInt(100000), key1)); err != nil { t.Fatalf("tx %d: failed to add transaction: %v", i, err) } @@ -684,7 +715,7 @@ func testTransactionLimitingEquivalency(t *testing.T, origin uint64) { state2.AddBalance(account2, big.NewInt(1000000)) txns := []*types.Transaction{} - for i := uint64(0); i < maxQueuedPerAccount+5; i++ { + for i := uint64(0); i < DefaultTxPoolConfig.AccountQueue+5; i++ { txns = append(txns, transaction(origin+i, big.NewInt(100000), key2)) } pool2.AddBatch(txns) @@ -706,14 +737,14 @@ func testTransactionLimitingEquivalency(t *testing.T, origin uint64) { // attacks. func TestTransactionPendingGlobalLimiting(t *testing.T) { // Reduce the queue limits to shorten test time - defer func(old uint64) { maxPendingTotal = old }(maxPendingTotal) - maxPendingTotal = minPendingPerAccount * 10 + defer func(old uint64) { DefaultTxPoolConfig.GlobalSlots = old }(DefaultTxPoolConfig.GlobalSlots) + DefaultTxPoolConfig.GlobalSlots = DefaultTxPoolConfig.AccountSlots * 10 // Create the pool to test the limit enforcement with db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, db) - pool := NewTxPool(params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) + pool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) pool.resetState() // Create a number of test accounts and fund them @@ -730,7 +761,7 @@ func TestTransactionPendingGlobalLimiting(t *testing.T) { txs := types.Transactions{} for _, key := range keys { addr := crypto.PubkeyToAddress(key.PublicKey) - for j := 0; j < int(maxPendingTotal)/len(keys)*2; j++ { + for j := 0; j < int(DefaultTxPoolConfig.GlobalSlots)/len(keys)*2; j++ { txs = append(txs, transaction(nonces[addr], big.NewInt(100000), key)) nonces[addr]++ } @@ -742,8 +773,8 @@ func TestTransactionPendingGlobalLimiting(t *testing.T) { for _, list := range pool.pending { pending += list.Len() } - if pending > int(maxPendingTotal) { - t.Fatalf("total pending transactions overflow allowance: %d > %d", pending, maxPendingTotal) + if pending > int(DefaultTxPoolConfig.GlobalSlots) { + t.Fatalf("total pending transactions overflow allowance: %d > %d", pending, DefaultTxPoolConfig.GlobalSlots) } } @@ -752,14 +783,14 @@ func TestTransactionPendingGlobalLimiting(t *testing.T) { // the transactions are still kept. func TestTransactionPendingMinimumAllowance(t *testing.T) { // Reduce the queue limits to shorten test time - defer func(old uint64) { maxPendingTotal = old }(maxPendingTotal) - maxPendingTotal = 0 + defer func(old uint64) { DefaultTxPoolConfig.GlobalSlots = old }(DefaultTxPoolConfig.GlobalSlots) + DefaultTxPoolConfig.GlobalSlots = 0 // Create the pool to test the limit enforcement with db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, db) - pool := NewTxPool(params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) + pool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) pool.resetState() // Create a number of test accounts and fund them @@ -776,7 +807,7 @@ func TestTransactionPendingMinimumAllowance(t *testing.T) { txs := types.Transactions{} for _, key := range keys { addr := crypto.PubkeyToAddress(key.PublicKey) - for j := 0; j < int(minPendingPerAccount)*2; j++ { + for j := 0; j < int(DefaultTxPoolConfig.AccountSlots)*2; j++ { txs = append(txs, transaction(nonces[addr], big.NewInt(100000), key)) nonces[addr]++ } @@ -785,12 +816,233 @@ func TestTransactionPendingMinimumAllowance(t *testing.T) { pool.AddBatch(txs) for addr, list := range pool.pending { - if list.Len() != int(minPendingPerAccount) { - t.Errorf("addr %x: total pending transactions mismatch: have %d, want %d", addr, list.Len(), minPendingPerAccount) + if list.Len() != int(DefaultTxPoolConfig.AccountSlots) { + t.Errorf("addr %x: total pending transactions mismatch: have %d, want %d", addr, list.Len(), DefaultTxPoolConfig.AccountSlots) } } } +// Tests that setting the transaction pool gas price to a higher value correctly +// discards everything cheaper than that and moves any gapped transactions back +// from the pending pool to the queue. +// +// Note, local transactions are never allowed to be dropped. +func TestTransactionPoolRepricing(t *testing.T) { + // Create the pool to test the pricing enforcement with + db, _ := ethdb.NewMemDatabase() + statedb, _ := state.New(common.Hash{}, db) + + pool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) + pool.resetState() + + // Create a number of test accounts and fund them + state, _ := pool.currentState() + + keys := make([]*ecdsa.PrivateKey, 3) + for i := 0; i < len(keys); i++ { + keys[i], _ = crypto.GenerateKey() + state.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) + } + // Generate and queue a batch of transactions, both pending and queued + txs := types.Transactions{} + + txs = append(txs, pricedTransaction(0, big.NewInt(100000), big.NewInt(2), keys[0])) + txs = append(txs, pricedTransaction(1, big.NewInt(100000), big.NewInt(1), keys[0])) + txs = append(txs, pricedTransaction(2, big.NewInt(100000), big.NewInt(2), keys[0])) + + txs = append(txs, pricedTransaction(1, big.NewInt(100000), big.NewInt(2), keys[1])) + txs = append(txs, pricedTransaction(2, big.NewInt(100000), big.NewInt(1), keys[1])) + txs = append(txs, pricedTransaction(3, big.NewInt(100000), big.NewInt(2), keys[1])) + + txs = append(txs, pricedTransaction(0, big.NewInt(100000), big.NewInt(1), keys[2])) + pool.SetLocal(txs[len(txs)-1]) // prevent this one from ever being dropped + + // Import the batch and that both pending and queued transactions match up + pool.AddBatch(txs) + + pending, queued := pool.stats() + if pending != 4 { + t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 4) + } + if queued != 3 { + t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 3) + } + // Reprice the pool and check that underpriced transactions get dropped + pool.SetGasPrice(big.NewInt(2)) + + pending, queued = pool.stats() + if pending != 2 { + t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 2) + } + if queued != 3 { + t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 3) + } + // Check that we can't add the old transactions back + if err := pool.Add(pricedTransaction(1, big.NewInt(100000), big.NewInt(1), keys[0])); err != ErrUnderpriced { + t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, ErrUnderpriced) + } + if err := pool.Add(pricedTransaction(2, big.NewInt(100000), big.NewInt(1), keys[1])); err != ErrUnderpriced { + t.Fatalf("adding underpriced queued transaction error mismatch: have %v, want %v", err, ErrUnderpriced) + } + // However we can add local underpriced transactions + tx := pricedTransaction(1, big.NewInt(100000), big.NewInt(1), keys[2]) + + pool.SetLocal(tx) // prevent this one from ever being dropped + if err := pool.Add(tx); err != nil { + t.Fatalf("failed to add underpriced local transaction: %v", err) + } + if pending, _ = pool.stats(); pending != 3 { + t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 3) + } +} + +// Tests that when the pool reaches its global transaction limit, underpriced +// transactions are gradually shifted out for more expensive ones and any gapped +// pending transactions are moved into te queue. +// +// Note, local transactions are never allowed to be dropped. +func TestTransactionPoolUnderpricing(t *testing.T) { + // Reduce the queue limits to shorten test time + defer func(old uint64) { DefaultTxPoolConfig.GlobalSlots = old }(DefaultTxPoolConfig.GlobalSlots) + DefaultTxPoolConfig.GlobalSlots = 2 + + defer func(old uint64) { DefaultTxPoolConfig.GlobalQueue = old }(DefaultTxPoolConfig.GlobalQueue) + DefaultTxPoolConfig.GlobalQueue = 2 + + // Create the pool to test the pricing enforcement with + db, _ := ethdb.NewMemDatabase() + statedb, _ := state.New(common.Hash{}, db) + + pool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) + pool.resetState() + + // Create a number of test accounts and fund them + state, _ := pool.currentState() + + keys := make([]*ecdsa.PrivateKey, 3) + for i := 0; i < len(keys); i++ { + keys[i], _ = crypto.GenerateKey() + state.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) + } + // Generate and queue a batch of transactions, both pending and queued + txs := types.Transactions{} + + txs = append(txs, pricedTransaction(0, big.NewInt(100000), big.NewInt(1), keys[0])) + txs = append(txs, pricedTransaction(1, big.NewInt(100000), big.NewInt(2), keys[0])) + + txs = append(txs, pricedTransaction(1, big.NewInt(100000), big.NewInt(1), keys[1])) + + txs = append(txs, pricedTransaction(0, big.NewInt(100000), big.NewInt(1), keys[2])) + pool.SetLocal(txs[len(txs)-1]) // prevent this one from ever being dropped + + // Import the batch and that both pending and queued transactions match up + pool.AddBatch(txs) + + pending, queued := pool.stats() + if pending != 3 { + t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 3) + } + if queued != 1 { + t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 1) + } + // Ensure that adding an underpriced transaction on block limit fails + if err := pool.Add(pricedTransaction(0, big.NewInt(100000), big.NewInt(1), keys[1])); err != ErrUnderpriced { + t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, ErrUnderpriced) + } + // Ensure that adding high priced transactions drops cheap ones, but not own + if err := pool.Add(pricedTransaction(0, big.NewInt(100000), big.NewInt(3), keys[1])); err != nil { + t.Fatalf("failed to add well priced transaction: %v", err) + } + if err := pool.Add(pricedTransaction(2, big.NewInt(100000), big.NewInt(4), keys[1])); err != nil { + t.Fatalf("failed to add well priced transaction: %v", err) + } + if err := pool.Add(pricedTransaction(3, big.NewInt(100000), big.NewInt(5), keys[1])); err != nil { + t.Fatalf("failed to add well priced transaction: %v", err) + } + pending, queued = pool.stats() + if pending != 2 { + t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 2) + } + if queued != 2 { + t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 2) + } + // Ensure that adding local transactions can push out even higher priced ones + tx := pricedTransaction(1, big.NewInt(100000), big.NewInt(0), keys[2]) + + pool.SetLocal(tx) // prevent this one from ever being dropped + if err := pool.Add(tx); err != nil { + t.Fatalf("failed to add underpriced local transaction: %v", err) + } + pending, queued = pool.stats() + if pending != 2 { + t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 2) + } + if queued != 2 { + t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 2) + } +} + +// Tests that the pool rejects replacement transactions that don't meet the minimum +// price bump required. +func TestTransactionReplacement(t *testing.T) { + // Create the pool to test the pricing enforcement with + db, _ := ethdb.NewMemDatabase() + statedb, _ := state.New(common.Hash{}, db) + + pool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) + pool.resetState() + + // Create a a test account to add transactions with + key, _ := crypto.GenerateKey() + + state, _ := pool.currentState() + state.AddBalance(crypto.PubkeyToAddress(key.PublicKey), big.NewInt(1000000000)) + + // Add pending transactions, ensuring the minimum price bump is enforced for replacement (for ultra low prices too) + price := int64(100) + threshold := (price * (100 + int64(DefaultTxPoolConfig.PriceBump))) / 100 + + if err := pool.Add(pricedTransaction(0, big.NewInt(100000), big.NewInt(1), key)); err != nil { + t.Fatalf("failed to add original cheap pending transaction: %v", err) + } + if err := pool.Add(pricedTransaction(0, big.NewInt(100001), big.NewInt(1), key)); err != ErrReplaceUnderpriced { + t.Fatalf("original cheap pending transaction replacement error mismatch: have %v, want %v", err, ErrReplaceUnderpriced) + } + if err := pool.Add(pricedTransaction(0, big.NewInt(100000), big.NewInt(2), key)); err != nil { + t.Fatalf("failed to replace original cheap pending transaction: %v", err) + } + + if err := pool.Add(pricedTransaction(0, big.NewInt(100000), big.NewInt(price), key)); err != nil { + t.Fatalf("failed to add original proper pending transaction: %v", err) + } + if err := pool.Add(pricedTransaction(0, big.NewInt(100000), big.NewInt(threshold), key)); err != ErrReplaceUnderpriced { + t.Fatalf("original proper pending transaction replacement error mismatch: have %v, want %v", err, ErrReplaceUnderpriced) + } + if err := pool.Add(pricedTransaction(0, big.NewInt(100000), big.NewInt(threshold+1), key)); err != nil { + t.Fatalf("failed to replace original proper pending transaction: %v", err) + } + // Add queued transactions, ensuring the minimum price bump is enforced for replacement (for ultra low prices too) + if err := pool.Add(pricedTransaction(2, big.NewInt(100000), big.NewInt(1), key)); err != nil { + t.Fatalf("failed to add original queued transaction: %v", err) + } + if err := pool.Add(pricedTransaction(2, big.NewInt(100001), big.NewInt(1), key)); err != ErrReplaceUnderpriced { + t.Fatalf("original queued transaction replacement error mismatch: have %v, want %v", err, ErrReplaceUnderpriced) + } + if err := pool.Add(pricedTransaction(2, big.NewInt(100000), big.NewInt(2), key)); err != nil { + t.Fatalf("failed to replace original queued transaction: %v", err) + } + + if err := pool.Add(pricedTransaction(2, big.NewInt(100000), big.NewInt(price), key)); err != nil { + t.Fatalf("failed to add original queued transaction: %v", err) + } + if err := pool.Add(pricedTransaction(2, big.NewInt(100001), big.NewInt(threshold), key)); err != ErrReplaceUnderpriced { + t.Fatalf("original queued transaction replacement error mismatch: have %v, want %v", err, ErrReplaceUnderpriced) + } + if err := pool.Add(pricedTransaction(2, big.NewInt(100000), big.NewInt(threshold+1), key)); err != nil { + t.Fatalf("failed to replace original queued transaction: %v", err) + } +} + // Benchmarks the speed of validating the contents of the pending queue of the // transaction pool. func BenchmarkPendingDemotion100(b *testing.B) { benchmarkPendingDemotion(b, 100) } @@ -835,7 +1087,7 @@ func benchmarkFuturePromotion(b *testing.B, size int) { // Benchmark the speed of pool validation b.ResetTimer() for i := 0; i < b.N; i++ { - pool.promoteExecutables(state) + pool.promoteExecutables(state, nil) } } diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index 0281c6d376..9b4f4a19cd 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -27,7 +27,12 @@ import ( "github.com/expanse-org/go-expanse/params" ) -var ErrInvalidChainId = errors.New("invalid chaid id for signer") +var ( + ErrInvalidChainId = errors.New("invalid chaid id for signer") + + errAbstractSigner = errors.New("abstract signer") + abstractSignerAddress = common.HexToAddress("ffffffffffffffffffffffffffffffffffffffff") +) // sigCache is used to cache the derived sender and contains // the signer used to derive it. diff --git a/core/types/transaction_test.go b/core/types/transaction_test.go index 09704d9f0e..4839caeb12 100644 --- a/core/types/transaction_test.go +++ b/core/types/transaction_test.go @@ -79,7 +79,7 @@ func decodeTx(data []byte) (*Transaction, error) { } func defaultTestKey() (*ecdsa.PrivateKey, common.Address) { - key := crypto.ToECDSA(common.Hex2Bytes("45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8")) + key, _ := crypto.HexToECDSA("45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8") addr := crypto.PubkeyToAddress(key.PublicKey) return key, addr } diff --git a/core/vm/analysis.go b/core/vm/analysis.go index 2f2c3305b5..6b91351d44 100644 --- a/core/vm/analysis.go +++ b/core/vm/analysis.go @@ -22,41 +22,39 @@ import ( "github.com/expanse-org/go-expanse/common" ) -var bigMaxUint64 = new(big.Int).SetUint64(^uint64(0)) - // destinations stores one map per contract (keyed by hash of code). // The maps contain an entry for each location of a JUMPDEST // instruction. -type destinations map[common.Hash]map[uint64]struct{} +type destinations map[common.Hash][]byte // has checks whether code has a JUMPDEST at dest. func (d destinations) has(codehash common.Hash, code []byte, dest *big.Int) bool { - // PC cannot go beyond len(code) and certainly can't be bigger than 64bits. + // PC cannot go beyond len(code) and certainly can't be bigger than 63bits. // Don't bother checking for JUMPDEST in that case. - if dest.Cmp(bigMaxUint64) > 0 { + udest := dest.Uint64() + if dest.BitLen() >= 63 || udest >= uint64(len(code)) { return false } + m, analysed := d[codehash] if !analysed { m = jumpdests(code) d[codehash] = m } - _, ok := m[dest.Uint64()] - return ok + return (m[udest/8] & (1 << (udest % 8))) != 0 } // jumpdests creates a map that contains an entry for each // PC location that is a JUMPDEST instruction. -func jumpdests(code []byte) map[uint64]struct{} { - m := make(map[uint64]struct{}) +func jumpdests(code []byte) []byte { + m := make([]byte, len(code)/8+1) for pc := uint64(0); pc < uint64(len(code)); pc++ { - var op OpCode = OpCode(code[pc]) - switch op { - case PUSH1, PUSH2, PUSH3, PUSH4, PUSH5, PUSH6, PUSH7, PUSH8, PUSH9, PUSH10, PUSH11, PUSH12, PUSH13, PUSH14, PUSH15, PUSH16, PUSH17, PUSH18, PUSH19, PUSH20, PUSH21, PUSH22, PUSH23, PUSH24, PUSH25, PUSH26, PUSH27, PUSH28, PUSH29, PUSH30, PUSH31, PUSH32: + op := OpCode(code[pc]) + if op == JUMPDEST { + m[pc/8] |= 1 << (pc % 8) + } else if op >= PUSH1 && op <= PUSH32 { a := uint64(op) - uint64(PUSH1) + 1 pc += a - case JUMPDEST: - m[pc] = struct{}{} } } return m diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 877eda247a..bc0c84a4a3 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -18,6 +18,7 @@ package vm import ( "crypto/sha256" + "errors" "math/big" "github.com/expanse-org/go-expanse/common" @@ -27,15 +28,17 @@ import ( "golang.org/x/crypto/ripemd160" ) +var errBadPrecompileInput = errors.New("bad pre compile input") + // Precompiled contract is the basic interface for native Go contracts. The implementation // requires a deterministic gas count based on the input size of the Run method of the // contract. type PrecompiledContract interface { - RequiredGas(inputSize int) uint64 // RequiredPrice calculates the contract gas use - Run(input []byte) []byte // Run runs the precompiled contract + RequiredGas(input []byte) uint64 // RequiredPrice calculates the contract gas use + Run(input []byte) ([]byte, error) // Run runs the precompiled contract } -// Precompiled contains the default set of ethereum contracts +// PrecompiledContracts contains the default set of ethereum contracts var PrecompiledContracts = map[common.Address]PrecompiledContract{ common.BytesToAddress([]byte{1}): &ecrecover{}, common.BytesToAddress([]byte{2}): &sha256hash{}, @@ -45,11 +48,9 @@ var PrecompiledContracts = map[common.Address]PrecompiledContract{ // RunPrecompile runs and evaluate the output of a precompiled contract defined in contracts.go func RunPrecompiledContract(p PrecompiledContract, input []byte, contract *Contract) (ret []byte, err error) { - gas := p.RequiredGas(len(input)) + gas := p.RequiredGas(input) if contract.UseGas(gas) { - ret = p.Run(input) - - return ret, nil + return p.Run(input) } else { return nil, ErrOutOfGas } @@ -58,11 +59,11 @@ func RunPrecompiledContract(p PrecompiledContract, input []byte, contract *Contr // ECRECOVER implemented as a native contract type ecrecover struct{} -func (c *ecrecover) RequiredGas(inputSize int) uint64 { +func (c *ecrecover) RequiredGas(input []byte) uint64 { return params.EcrecoverGas } -func (c *ecrecover) Run(in []byte) []byte { +func (c *ecrecover) Run(in []byte) ([]byte, error) { const ecRecoverInputLength = 128 in = common.RightPadBytes(in, ecRecoverInputLength) @@ -76,18 +77,18 @@ func (c *ecrecover) Run(in []byte) []byte { // tighter sig s values in homestead only apply to tx sigs if !allZero(in[32:63]) || !crypto.ValidateSignatureValues(v, r, s, false) { log.Trace("ECRECOVER error: v, r or s value invalid") - return nil + return nil, nil } // v needs to be at the end for libsecp256k1 pubKey, err := crypto.Ecrecover(in[:32], append(in[64:128], v)) // make sure the public key is a valid one if err != nil { log.Trace("ECRECOVER failed", "err", err) - return nil + return nil, nil } // the first byte of pubkey is bitcoin heritage - return common.LeftPadBytes(crypto.Keccak256(pubKey[1:])[12:], 32) + return common.LeftPadBytes(crypto.Keccak256(pubKey[1:])[12:], 32), nil } // SHA256 implemented as a native contract @@ -97,12 +98,12 @@ type sha256hash struct{} // // This method does not require any overflow checking as the input size gas costs // required for anything significant is so high it's impossible to pay for. -func (c *sha256hash) RequiredGas(inputSize int) uint64 { - return uint64(inputSize+31)/32*params.Sha256WordGas + params.Sha256Gas +func (c *sha256hash) RequiredGas(input []byte) uint64 { + return uint64(len(input)+31)/32*params.Sha256WordGas + params.Sha256Gas } -func (c *sha256hash) Run(in []byte) []byte { +func (c *sha256hash) Run(in []byte) ([]byte, error) { h := sha256.Sum256(in) - return h[:] + return h[:], nil } // RIPMED160 implemented as a native contract @@ -112,13 +113,13 @@ type ripemd160hash struct{} // // This method does not require any overflow checking as the input size gas costs // required for anything significant is so high it's impossible to pay for. -func (c *ripemd160hash) RequiredGas(inputSize int) uint64 { - return uint64(inputSize+31)/32*params.Ripemd160WordGas + params.Ripemd160Gas +func (c *ripemd160hash) RequiredGas(input []byte) uint64 { + return uint64(len(input)+31)/32*params.Ripemd160WordGas + params.Ripemd160Gas } -func (c *ripemd160hash) Run(in []byte) []byte { +func (c *ripemd160hash) Run(in []byte) ([]byte, error) { ripemd := ripemd160.New() ripemd.Write(in) - return common.LeftPadBytes(ripemd.Sum(nil), 32) + return common.LeftPadBytes(ripemd.Sum(nil), 32), nil } // data copy implemented as a native contract @@ -128,9 +129,9 @@ type dataCopy struct{} // // This method does not require any overflow checking as the input size gas costs // required for anything significant is so high it's impossible to pay for. -func (c *dataCopy) RequiredGas(inputSize int) uint64 { - return uint64(inputSize+31)/32*params.IdentityWordGas + params.IdentityGas +func (c *dataCopy) RequiredGas(input []byte) uint64 { + return uint64(len(input)+31)/32*params.IdentityWordGas + params.IdentityGas } -func (c *dataCopy) Run(in []byte) []byte { - return in +func (c *dataCopy) Run(in []byte) ([]byte, error) { + return in, nil } diff --git a/core/vm/contracts_test.go b/core/vm/contracts_test.go new file mode 100644 index 0000000000..830a8f69d9 --- /dev/null +++ b/core/vm/contracts_test.go @@ -0,0 +1 @@ +package vm diff --git a/core/vm/evm.go b/core/vm/evm.go index 11b8031099..749422ce1f 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -33,7 +33,20 @@ type ( GetHashFunc func(uint64) common.Hash ) -// Context provides the EVM with auxiliary information. Once provided it shouldn't be modified. +// run runs the given contract and takes care of running precompiles with a fallback to the byte code interpreter. +func run(evm *EVM, snapshot int, contract *Contract, input []byte) ([]byte, error) { + if contract.CodeAddr != nil { + precompiledContracts := PrecompiledContracts + if p := precompiledContracts[*contract.CodeAddr]; p != nil { + return RunPrecompiledContract(p, input, contract) + } + } + + return evm.interpreter.Run(snapshot, contract, input) +} + +// Context provides the EVM with auxiliary information. Once provided +// it shouldn't be modified. type Context struct { // CanTransfer returns whether the account contains // sufficient ether to transfer the value @@ -55,7 +68,13 @@ type Context struct { Difficulty *big.Int // Provides information for DIFFICULTY } -// EVM provides information about external sources for the EVM +// EVM is the Ethereum Virtual Machine base object and provides +// the necessary tools to run a contract on the given state with +// the provided context. It should be noted that any error +// generated through any of the calls should be considered a +// revert-state-and-consume-all-gas operation, no checks on +// specific errors should ever be performed. The interpreter makes +// sure that any errors generated are to be considered faulty code. // // The EVM should never be reused and is not thread safe. type EVM struct { @@ -68,6 +87,8 @@ type EVM struct { // chainConfig contains information about the current chain chainConfig *params.ChainConfig + // chain rules contains the chain rules for the current epoch + chainRules params.Rules // virtual machine configuration options used to initialise the // evm. vmConfig Config @@ -79,21 +100,23 @@ type EVM struct { abort int32 } -// NewEVM retutrns a new EVM evmironment. +// NewEVM retutrns a new EVM evmironment. The returned EVM is not thread safe +// and should only ever be used *once*. func NewEVM(ctx Context, statedb StateDB, chainConfig *params.ChainConfig, vmConfig Config) *EVM { evm := &EVM{ Context: ctx, StateDB: statedb, vmConfig: vmConfig, chainConfig: chainConfig, + chainRules: chainConfig.Rules(ctx.BlockNumber), } evm.interpreter = NewInterpreter(evm, vmConfig) return evm } -// Cancel cancels any running EVM operation. This may be called concurrently and it's safe to be -// called multiple times. +// Cancel cancels any running EVM operation. This may be called concurrently and +// it's safe to be called multiple times. func (evm *EVM) Cancel() { atomic.StoreInt32(&evm.abort, 1) } @@ -134,13 +157,12 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas contract := NewContract(caller, to, value, gas) contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr)) - ret, err = evm.interpreter.Run(contract, input) + ret, err = run(evm, snapshot, contract, input) // When an error was returned by the EVM or when setting the creation code // above we revert to the snapshot and consume any gas remaining. Additionally // when we're in homestead this also counts for code storage gas errors. if err != nil { contract.UseGas(contract.Gas) - evm.StateDB.RevertToSnapshot(snapshot) } return ret, contract.Gas, err @@ -175,10 +197,9 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte, contract := NewContract(caller, to, value, gas) contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr)) - ret, err = evm.interpreter.Run(contract, input) + ret, err = run(evm, snapshot, contract, input) if err != nil { contract.UseGas(contract.Gas) - evm.StateDB.RevertToSnapshot(snapshot) } @@ -210,10 +231,9 @@ func (evm *EVM) DelegateCall(caller ContractRef, addr common.Address, input []by contract := NewContract(caller, to, nil, gas).AsDelegate() contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr)) - ret, err = evm.interpreter.Run(contract, input) + ret, err = run(evm, snapshot, contract, input) if err != nil { contract.UseGas(contract.Gas) - evm.StateDB.RevertToSnapshot(snapshot) } @@ -253,8 +273,7 @@ func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.I contract := NewContract(caller, AccountRef(contractAddr), value, gas) contract.SetCallCode(&contractAddr, crypto.Keccak256Hash(code), code) - ret, err = evm.interpreter.Run(contract, nil) - + ret, err = run(evm, snapshot, contract, nil) // check whether the max code size has been exceeded maxCodeSizeExceeded := len(ret) > params.MaxCodeSize // if the contract creation ran successfully and no errors were returned @@ -275,10 +294,8 @@ func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.I // when we're in homestead this also counts for code storage gas errors. if maxCodeSizeExceeded || (err != nil && (evm.ChainConfig().IsHomestead(evm.BlockNumber) || err != ErrCodeStoreOutOfGas)) { + contract.UseGas(contract.Gas) evm.StateDB.RevertToSnapshot(snapshot) - - // Nothing should be returned when an error is thrown. - return nil, contractAddr, 0, err } // If the vm returned with an error the return value should be set to nil. // This isn't consensus critical but merely to for behaviour reasons such as diff --git a/core/vm/instructions.go b/core/vm/instructions.go index ec67cff819..237a05388e 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -27,7 +27,9 @@ import ( "github.com/expanse-org/go-expanse/params" ) -var bigZero = new(big.Int) +var ( + bigZero = new(big.Int) +) func opAdd(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { x, y := stack.pop(), stack.pop() @@ -599,7 +601,7 @@ func opCall(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Sta contract.Gas += returnGas evm.interpreter.intPool.put(addr, value, inOffset, inSize, retOffset, retSize) - return nil, nil + return ret, nil } func opCallCode(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { @@ -633,16 +635,10 @@ func opCallCode(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack contract.Gas += returnGas evm.interpreter.intPool.put(addr, value, inOffset, inSize, retOffset, retSize) - return nil, nil + return ret, nil } func opDelegateCall(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - // if not homestead return an error. DELEGATECALL is not supported - // during pre-homestead. - if !evm.ChainConfig().IsHomestead(evm.BlockNumber) { - return nil, fmt.Errorf("invalid opcode %x", DELEGATECALL) - } - gas, to, inOffset, inSize, outOffset, outSize := stack.pop().Uint64(), stack.pop(), stack.pop(), stack.pop(), stack.pop(), stack.pop() toAddr := common.BigToAddress(to) @@ -658,7 +654,7 @@ func opDelegateCall(pc *uint64, evm *EVM, contract *Contract, memory *Memory, st contract.Gas += returnGas evm.interpreter.intPool.put(to, inOffset, inSize, outOffset, outSize) - return nil, nil + return ret, nil } func opReturn(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { @@ -666,6 +662,7 @@ func opReturn(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *S ret := memory.GetPtr(offset.Int64(), size.Int64()) evm.interpreter.intPool.put(offset, size) + return ret, nil } @@ -709,10 +706,23 @@ func makeLog(size int) executionFunc { } // make push instruction function -func makePush(size uint64, bsize *big.Int) executionFunc { +func makePush(size uint64, pushByteSize int) executionFunc { return func(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - byts := getData(contract.Code, evm.interpreter.intPool.get().SetUint64(*pc+1), bsize) - stack.push(new(big.Int).SetBytes(byts)) + codeLen := len(contract.Code) + + startMin := codeLen + if int(*pc+1) < startMin { + startMin = int(*pc + 1) + } + + endMin := codeLen + if startMin+pushByteSize < endMin { + endMin = startMin + pushByteSize + } + + integer := evm.interpreter.intPool.get() + stack.push(integer.SetBytes(common.RightPadBytes(contract.Code[startMin:endMin], pushByteSize))) + *pc += size return nil, nil } @@ -721,7 +731,7 @@ func makePush(size uint64, bsize *big.Int) executionFunc { // make push instruction function func makeDup(size int64) executionFunc { return func(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - stack.dup(int(size)) + stack.dup(evm.interpreter.intPool, int(size)) return nil, nil } } diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index df76bcc47e..2f47e08790 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -52,43 +52,53 @@ type Config struct { } // Interpreter is used to run Ethereum based contracts and will utilise the -// passed environment to query external sources for state information. +// passed evmironment to query external sources for state information. // The Interpreter will run the byte code VM or JIT VM based on the passed // configuration. type Interpreter struct { - env *EVM + evm *EVM cfg Config gasTable params.GasTable intPool *intPool + + readonly bool } // NewInterpreter returns a new instance of the Interpreter. -func NewInterpreter(env *EVM, cfg Config) *Interpreter { +func NewInterpreter(evm *EVM, cfg Config) *Interpreter { // We use the STOP instruction whether to see // the jump table was initialised. If it was not // we'll set the default jump table. if !cfg.JumpTable[STOP].valid { - cfg.JumpTable = defaultJumpTable + switch { + case evm.ChainConfig().IsHomestead(evm.BlockNumber): + cfg.JumpTable = homesteadInstructionSet + default: + cfg.JumpTable = frontierInstructionSet + } } return &Interpreter{ - env: env, + evm: evm, cfg: cfg, - gasTable: env.ChainConfig().GasTable(env.BlockNumber), + gasTable: evm.ChainConfig().GasTable(evm.BlockNumber), intPool: newIntPool(), } } -// Run loops and evaluates the contract's code with the given input data -func (evm *Interpreter) Run(contract *Contract, input []byte) (ret []byte, err error) { - evm.env.depth++ - defer func() { evm.env.depth-- }() +func (in *Interpreter) enforceRestrictions(op OpCode, operation operation, stack *Stack) error { + return nil +} - if contract.CodeAddr != nil { - if p := PrecompiledContracts[*contract.CodeAddr]; p != nil { - return RunPrecompiledContract(p, input, contract) - } - } +// Run loops and evaluates the contract's code with the given input data and returns +// the return byte-slice and an error if one occurred. +// +// It's important to note that any errors returned by the interpreter should be +// considered a revert-and-consume-all-gas operation. No error specific checks +// should be handled to reduce complexity and errors further down the in. +func (in *Interpreter) Run(snapshot int, contract *Contract, input []byte) (ret []byte, err error) { + in.evm.depth++ + defer func() { in.evm.depth-- }() // Don't bother with the execution if there's no code. if len(contract.Code) == 0 { @@ -105,7 +115,8 @@ func (evm *Interpreter) Run(contract *Contract, input []byte) (ret []byte, err e mem = NewMemory() // bound memory stack = newstack() // local stack // For optimisation reason we're using uint64 as the program counter. - // It's theoretically possible to go above 2^64. The YP defines the PC to be uint256. Practically much less so feasible. + // It's theoretically possible to go above 2^64. The YP defines the PC + // to be uint256. Practically much less so feasible. pc = uint64(0) // program counter cost uint64 ) @@ -113,31 +124,34 @@ func (evm *Interpreter) Run(contract *Contract, input []byte) (ret []byte, err e // User defer pattern to check for an error and, based on the error being nil or not, use all gas and return. defer func() { - if err != nil && evm.cfg.Debug { + if err != nil && in.cfg.Debug { // XXX For debugging //fmt.Printf("%04d: %8v cost = %-8d stack = %-8d ERR = %v\n", pc, op, cost, stack.len(), err) - evm.cfg.Tracer.CaptureState(evm.env, pc, op, contract.Gas, cost, mem, stack, contract, evm.env.depth, err) + in.cfg.Tracer.CaptureState(in.evm, pc, op, contract.Gas, cost, mem, stack, contract, in.evm.depth, err) } }() - log.Debug("EVM running contract", "hash", codehash[:]) + log.Debug("interpreter running contract", "hash", codehash[:]) tstart := time.Now() - defer log.Debug("EVM finished running contract", "hash", codehash[:], "elapsed", time.Since(tstart)) + defer log.Debug("interpreter finished running contract", "hash", codehash[:], "elapsed", time.Since(tstart)) // The Interpreter main run loop (contextual). This loop runs until either an // explicit STOP, RETURN or SELFDESTRUCT is executed, an error occurred during - // the execution of one of the operations or until the evm.done is set by - // the parent context.Context. - for atomic.LoadInt32(&evm.env.abort) == 0 { + // the execution of one of the operations or until the done flag is set by the + // parent context. + for atomic.LoadInt32(&in.evm.abort) == 0 { // Get the memory location of pc op = contract.GetOp(pc) // get the operation from the jump table matching the opcode - operation := evm.cfg.JumpTable[op] + operation := in.cfg.JumpTable[op] + if err := in.enforceRestrictions(op, operation, stack); err != nil { + return nil, err + } // if the op is invalid abort the process and return an error if !operation.valid { - return nil, fmt.Errorf("invalid opcode %x", op) + return nil, fmt.Errorf("invalid opcode 0x%x", int(op)) } // validate the stack and make sure there enough stack items available @@ -161,10 +175,10 @@ func (evm *Interpreter) Run(contract *Contract, input []byte) (ret []byte, err e } } - if !evm.cfg.DisableGasMetering { + if !in.cfg.DisableGasMetering { // consume the gas and return an error if not enough gas is available. // cost is explicitly set so that the capture state defer method cas get the proper cost - cost, err = operation.gasCost(evm.gasTable, evm.env, contract, stack, mem, memorySize) + cost, err = operation.gasCost(in.gasTable, in.evm, contract, stack, mem, memorySize) if err != nil || !contract.UseGas(cost) { return nil, ErrOutOfGas } @@ -173,19 +187,20 @@ func (evm *Interpreter) Run(contract *Contract, input []byte) (ret []byte, err e mem.Resize(memorySize) } - if evm.cfg.Debug { - evm.cfg.Tracer.CaptureState(evm.env, pc, op, contract.Gas, cost, mem, stack, contract, evm.env.depth, err) + if in.cfg.Debug { + in.cfg.Tracer.CaptureState(in.evm, pc, op, contract.Gas, cost, mem, stack, contract, in.evm.depth, err) } // XXX For debugging //fmt.Printf("%04d: %8v cost = %-8d stack = %-8d\n", pc, op, cost, stack.len()) // execute the operation - res, err := operation.execute(&pc, evm.env, contract, mem, stack) + res, err := operation.execute(&pc, in.evm, contract, mem, stack) // verifyPool is a build flag. Pool verification makes sure the integrity // of the integer pool by comparing values to a default value. if verifyPool { - verifyIntegerPool(evm.intPool) + verifyIntegerPool(in.intPool) } + switch { case err != nil: return nil, err @@ -194,6 +209,11 @@ func (evm *Interpreter) Run(contract *Contract, input []byte) (ret []byte, err e case !operation.jumps: pc++ } + // if the operation returned a value make sure that is also set + // the last return data. + if res != nil { + mem.lastReturn = ret + } } return nil, nil } diff --git a/core/vm/intpool.go b/core/vm/intpool.go index 4f1228e149..384f5df59b 100644 --- a/core/vm/intpool.go +++ b/core/vm/intpool.go @@ -20,6 +20,8 @@ import "math/big" var checkVal = big.NewInt(-42) +const poolLimit = 256 + // intPool is a pool of big integers that // can be reused for all big.Int operations. type intPool struct { @@ -37,6 +39,10 @@ func (p *intPool) get() *big.Int { return new(big.Int) } func (p *intPool) put(is ...*big.Int) { + if len(p.pool.data) > poolLimit { + return + } + for _, i := range is { // verifyPool is a build flag. Pool verification makes sure the integrity // of the integer pool by comparing values to a default value. diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index a9af1f37ea..96e72ddf67 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -47,13 +47,36 @@ type operation struct { // jumps indicates whether operation made a jump. This prevents the program // counter from further incrementing. jumps bool + // writes determines whether this a state modifying operation + writes bool // valid is used to check whether the retrieved operation is valid and known valid bool + // reverts determined whether the operation reverts state + reverts bool } -var defaultJumpTable = NewJumpTable() +var ( + frontierInstructionSet = NewFrontierInstructionSet() + homesteadInstructionSet = NewHomesteadInstructionSet() +) -func NewJumpTable() [256]operation { +// NewHomesteadInstructionSet returns the frontier and homestead +// instructions that can be executed during the homestead phase. +func NewHomesteadInstructionSet() [256]operation { + instructionSet := NewFrontierInstructionSet() + instructionSet[DELEGATECALL] = operation{ + execute: opDelegateCall, + gasCost: gasDelegateCall, + validateStack: makeStackFunc(6, 1), + memorySize: memoryDelegateCall, + valid: true, + } + return instructionSet +} + +// NewFrontierInstructionSet returns the frontier instructions +// that can be executed during the frontier phase. +func NewFrontierInstructionSet() [256]operation { return [256]operation{ STOP: { execute: opStop, @@ -357,6 +380,7 @@ func NewJumpTable() [256]operation { gasCost: gasSStore, validateStack: makeStackFunc(2, 0), valid: true, + writes: true, }, JUMP: { execute: opJump, @@ -397,193 +421,193 @@ func NewJumpTable() [256]operation { valid: true, }, PUSH1: { - execute: makePush(1, big.NewInt(1)), + execute: makePush(1, 1), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH2: { - execute: makePush(2, big.NewInt(2)), + execute: makePush(2, 2), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH3: { - execute: makePush(3, big.NewInt(3)), + execute: makePush(3, 3), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH4: { - execute: makePush(4, big.NewInt(4)), + execute: makePush(4, 4), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH5: { - execute: makePush(5, big.NewInt(5)), + execute: makePush(5, 5), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH6: { - execute: makePush(6, big.NewInt(6)), + execute: makePush(6, 6), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH7: { - execute: makePush(7, big.NewInt(7)), + execute: makePush(7, 7), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH8: { - execute: makePush(8, big.NewInt(8)), + execute: makePush(8, 8), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH9: { - execute: makePush(9, big.NewInt(9)), + execute: makePush(9, 9), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH10: { - execute: makePush(10, big.NewInt(10)), + execute: makePush(10, 10), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH11: { - execute: makePush(11, big.NewInt(11)), + execute: makePush(11, 11), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH12: { - execute: makePush(12, big.NewInt(12)), + execute: makePush(12, 12), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH13: { - execute: makePush(13, big.NewInt(13)), + execute: makePush(13, 13), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH14: { - execute: makePush(14, big.NewInt(14)), + execute: makePush(14, 14), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH15: { - execute: makePush(15, big.NewInt(15)), + execute: makePush(15, 15), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH16: { - execute: makePush(16, big.NewInt(16)), + execute: makePush(16, 16), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH17: { - execute: makePush(17, big.NewInt(17)), + execute: makePush(17, 17), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH18: { - execute: makePush(18, big.NewInt(18)), + execute: makePush(18, 18), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH19: { - execute: makePush(19, big.NewInt(19)), + execute: makePush(19, 19), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH20: { - execute: makePush(20, big.NewInt(20)), + execute: makePush(20, 20), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH21: { - execute: makePush(21, big.NewInt(21)), + execute: makePush(21, 21), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH22: { - execute: makePush(22, big.NewInt(22)), + execute: makePush(22, 22), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH23: { - execute: makePush(23, big.NewInt(23)), + execute: makePush(23, 23), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH24: { - execute: makePush(24, big.NewInt(24)), + execute: makePush(24, 24), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH25: { - execute: makePush(25, big.NewInt(25)), + execute: makePush(25, 25), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH26: { - execute: makePush(26, big.NewInt(26)), + execute: makePush(26, 26), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH27: { - execute: makePush(27, big.NewInt(27)), + execute: makePush(27, 27), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH28: { - execute: makePush(28, big.NewInt(28)), + execute: makePush(28, 28), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH29: { - execute: makePush(29, big.NewInt(29)), + execute: makePush(29, 29), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH30: { - execute: makePush(30, big.NewInt(30)), + execute: makePush(30, 30), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH31: { - execute: makePush(31, big.NewInt(31)), + execute: makePush(31, 31), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, }, PUSH32: { - execute: makePush(32, big.NewInt(32)), + execute: makePush(32, 32), gasCost: gasPush, validateStack: makeStackFunc(0, 1), valid: true, @@ -821,6 +845,7 @@ func NewJumpTable() [256]operation { validateStack: makeStackFunc(3, 1), memorySize: memoryCreate, valid: true, + writes: true, }, CALL: { execute: opCall, @@ -844,19 +869,13 @@ func NewJumpTable() [256]operation { halts: true, valid: true, }, - DELEGATECALL: { - execute: opDelegateCall, - gasCost: gasDelegateCall, - validateStack: makeStackFunc(6, 1), - memorySize: memoryDelegateCall, - valid: true, - }, SELFDESTRUCT: { execute: opSuicide, gasCost: gasSuicide, validateStack: makeStackFunc(1, 0), halts: true, valid: true, + writes: true, }, } } diff --git a/core/vm/memory.go b/core/vm/memory.go index 99a84d2271..6dbee94eff 100644 --- a/core/vm/memory.go +++ b/core/vm/memory.go @@ -22,6 +22,7 @@ import "fmt" type Memory struct { store []byte lastGasCost uint64 + lastReturn []byte } func NewMemory() *Memory { diff --git a/core/vm/stack.go b/core/vm/stack.go index 2d1b7bb82d..9c10d50ad1 100644 --- a/core/vm/stack.go +++ b/core/vm/stack.go @@ -29,7 +29,7 @@ type Stack struct { } func newstack() *Stack { - return &Stack{} + return &Stack{data: make([]*big.Int, 0, 1024)} } func (st *Stack) Data() []*big.Int { @@ -60,8 +60,8 @@ func (st *Stack) swap(n int) { st.data[st.len()-n], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-n] } -func (st *Stack) dup(n int) { - st.push(new(big.Int).Set(st.data[st.len()-n])) +func (st *Stack) dup(pool *intPool, n int) { + st.push(pool.get().Set(st.data[st.len()-n])) } func (st *Stack) peek() *big.Int { diff --git a/crypto/bn256/bn256.go b/crypto/bn256/bn256.go new file mode 100644 index 0000000000..92418369b0 --- /dev/null +++ b/crypto/bn256/bn256.go @@ -0,0 +1,428 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package bn256 implements a particular bilinear group at the 128-bit security level. +// +// Bilinear groups are the basis of many of the new cryptographic protocols +// that have been proposed over the past decade. They consist of a triplet of +// groups (G₁, G₂ and GT) such that there exists a function e(g₁ˣ,g₂ʸ)=gTˣʸ +// (where gₓ is a generator of the respective group). That function is called +// a pairing function. +// +// This package specifically implements the Optimal Ate pairing over a 256-bit +// Barreto-Naehrig curve as described in +// http://cryptojedi.org/papers/dclxvi-20100714.pdf. Its output is compatible +// with the implementation described in that paper. +package bn256 + +import ( + "crypto/rand" + "io" + "math/big" +) + +// BUG(agl): this implementation is not constant time. +// TODO(agl): keep GF(p²) elements in Mongomery form. + +// G1 is an abstract cyclic group. The zero value is suitable for use as the +// output of an operation, but cannot be used as an input. +type G1 struct { + p *curvePoint +} + +// RandomG1 returns x and g₁ˣ where x is a random, non-zero number read from r. +func RandomG1(r io.Reader) (*big.Int, *G1, error) { + var k *big.Int + var err error + + for { + k, err = rand.Int(r, Order) + if err != nil { + return nil, nil, err + } + if k.Sign() > 0 { + break + } + } + + return k, new(G1).ScalarBaseMult(k), nil +} + +func (g *G1) String() string { + return "bn256.G1" + g.p.String() +} + +// CurvePoints returns p's curve points in big integer +func (e *G1) CurvePoints() (*big.Int, *big.Int, *big.Int, *big.Int) { + return e.p.x, e.p.y, e.p.z, e.p.t +} + +// ScalarBaseMult sets e to g*k where g is the generator of the group and +// then returns e. +func (e *G1) ScalarBaseMult(k *big.Int) *G1 { + if e.p == nil { + e.p = newCurvePoint(nil) + } + e.p.Mul(curveGen, k, new(bnPool)) + return e +} + +// ScalarMult sets e to a*k and then returns e. +func (e *G1) ScalarMult(a *G1, k *big.Int) *G1 { + if e.p == nil { + e.p = newCurvePoint(nil) + } + e.p.Mul(a.p, k, new(bnPool)) + return e +} + +// Add sets e to a+b and then returns e. +// BUG(agl): this function is not complete: a==b fails. +func (e *G1) Add(a, b *G1) *G1 { + if e.p == nil { + e.p = newCurvePoint(nil) + } + e.p.Add(a.p, b.p, new(bnPool)) + return e +} + +// Neg sets e to -a and then returns e. +func (e *G1) Neg(a *G1) *G1 { + if e.p == nil { + e.p = newCurvePoint(nil) + } + e.p.Negative(a.p) + return e +} + +// Marshal converts n to a byte slice. +func (n *G1) Marshal() []byte { + n.p.MakeAffine(nil) + + xBytes := new(big.Int).Mod(n.p.x, P).Bytes() + yBytes := new(big.Int).Mod(n.p.y, P).Bytes() + + // Each value is a 256-bit number. + const numBytes = 256 / 8 + + ret := make([]byte, numBytes*2) + copy(ret[1*numBytes-len(xBytes):], xBytes) + copy(ret[2*numBytes-len(yBytes):], yBytes) + + return ret +} + +// Unmarshal sets e to the result of converting the output of Marshal back into +// a group element and then returns e. +func (e *G1) Unmarshal(m []byte) (*G1, bool) { + // Each value is a 256-bit number. + const numBytes = 256 / 8 + + if len(m) != 2*numBytes { + return nil, false + } + + if e.p == nil { + e.p = newCurvePoint(nil) + } + + e.p.x.SetBytes(m[0*numBytes : 1*numBytes]) + e.p.y.SetBytes(m[1*numBytes : 2*numBytes]) + + if e.p.x.Sign() == 0 && e.p.y.Sign() == 0 { + // This is the point at infinity. + e.p.y.SetInt64(1) + e.p.z.SetInt64(0) + e.p.t.SetInt64(0) + } else { + e.p.z.SetInt64(1) + e.p.t.SetInt64(1) + + if !e.p.IsOnCurve() { + return nil, false + } + } + + return e, true +} + +// G2 is an abstract cyclic group. The zero value is suitable for use as the +// output of an operation, but cannot be used as an input. +type G2 struct { + p *twistPoint +} + +// RandomG1 returns x and g₂ˣ where x is a random, non-zero number read from r. +func RandomG2(r io.Reader) (*big.Int, *G2, error) { + var k *big.Int + var err error + + for { + k, err = rand.Int(r, Order) + if err != nil { + return nil, nil, err + } + if k.Sign() > 0 { + break + } + } + + return k, new(G2).ScalarBaseMult(k), nil +} + +func (g *G2) String() string { + return "bn256.G2" + g.p.String() +} + +// CurvePoints returns the curve points of p which includes the real +// and imaginary parts of the curve point. +func (e *G2) CurvePoints() (*gfP2, *gfP2, *gfP2, *gfP2) { + return e.p.x, e.p.y, e.p.z, e.p.t +} + +// ScalarBaseMult sets e to g*k where g is the generator of the group and +// then returns out. +func (e *G2) ScalarBaseMult(k *big.Int) *G2 { + if e.p == nil { + e.p = newTwistPoint(nil) + } + e.p.Mul(twistGen, k, new(bnPool)) + return e +} + +// ScalarMult sets e to a*k and then returns e. +func (e *G2) ScalarMult(a *G2, k *big.Int) *G2 { + if e.p == nil { + e.p = newTwistPoint(nil) + } + e.p.Mul(a.p, k, new(bnPool)) + return e +} + +// Add sets e to a+b and then returns e. +// BUG(agl): this function is not complete: a==b fails. +func (e *G2) Add(a, b *G2) *G2 { + if e.p == nil { + e.p = newTwistPoint(nil) + } + e.p.Add(a.p, b.p, new(bnPool)) + return e +} + +// Marshal converts n into a byte slice. +func (n *G2) Marshal() []byte { + n.p.MakeAffine(nil) + + xxBytes := new(big.Int).Mod(n.p.x.x, P).Bytes() + xyBytes := new(big.Int).Mod(n.p.x.y, P).Bytes() + yxBytes := new(big.Int).Mod(n.p.y.x, P).Bytes() + yyBytes := new(big.Int).Mod(n.p.y.y, P).Bytes() + + // Each value is a 256-bit number. + const numBytes = 256 / 8 + + ret := make([]byte, numBytes*4) + copy(ret[1*numBytes-len(xxBytes):], xxBytes) + copy(ret[2*numBytes-len(xyBytes):], xyBytes) + copy(ret[3*numBytes-len(yxBytes):], yxBytes) + copy(ret[4*numBytes-len(yyBytes):], yyBytes) + + return ret +} + +// Unmarshal sets e to the result of converting the output of Marshal back into +// a group element and then returns e. +func (e *G2) Unmarshal(m []byte) (*G2, bool) { + // Each value is a 256-bit number. + const numBytes = 256 / 8 + + if len(m) != 4*numBytes { + return nil, false + } + + if e.p == nil { + e.p = newTwistPoint(nil) + } + + e.p.x.x.SetBytes(m[0*numBytes : 1*numBytes]) + e.p.x.y.SetBytes(m[1*numBytes : 2*numBytes]) + e.p.y.x.SetBytes(m[2*numBytes : 3*numBytes]) + e.p.y.y.SetBytes(m[3*numBytes : 4*numBytes]) + + if e.p.x.x.Sign() == 0 && + e.p.x.y.Sign() == 0 && + e.p.y.x.Sign() == 0 && + e.p.y.y.Sign() == 0 { + // This is the point at infinity. + e.p.y.SetOne() + e.p.z.SetZero() + e.p.t.SetZero() + } else { + e.p.z.SetOne() + e.p.t.SetOne() + + if !e.p.IsOnCurve() { + return nil, false + } + } + + return e, true +} + +// GT is an abstract cyclic group. The zero value is suitable for use as the +// output of an operation, but cannot be used as an input. +type GT struct { + p *gfP12 +} + +func (g *GT) String() string { + return "bn256.GT" + g.p.String() +} + +// ScalarMult sets e to a*k and then returns e. +func (e *GT) ScalarMult(a *GT, k *big.Int) *GT { + if e.p == nil { + e.p = newGFp12(nil) + } + e.p.Exp(a.p, k, new(bnPool)) + return e +} + +// Add sets e to a+b and then returns e. +func (e *GT) Add(a, b *GT) *GT { + if e.p == nil { + e.p = newGFp12(nil) + } + e.p.Mul(a.p, b.p, new(bnPool)) + return e +} + +// Neg sets e to -a and then returns e. +func (e *GT) Neg(a *GT) *GT { + if e.p == nil { + e.p = newGFp12(nil) + } + e.p.Invert(a.p, new(bnPool)) + return e +} + +// Marshal converts n into a byte slice. +func (n *GT) Marshal() []byte { + n.p.Minimal() + + xxxBytes := n.p.x.x.x.Bytes() + xxyBytes := n.p.x.x.y.Bytes() + xyxBytes := n.p.x.y.x.Bytes() + xyyBytes := n.p.x.y.y.Bytes() + xzxBytes := n.p.x.z.x.Bytes() + xzyBytes := n.p.x.z.y.Bytes() + yxxBytes := n.p.y.x.x.Bytes() + yxyBytes := n.p.y.x.y.Bytes() + yyxBytes := n.p.y.y.x.Bytes() + yyyBytes := n.p.y.y.y.Bytes() + yzxBytes := n.p.y.z.x.Bytes() + yzyBytes := n.p.y.z.y.Bytes() + + // Each value is a 256-bit number. + const numBytes = 256 / 8 + + ret := make([]byte, numBytes*12) + copy(ret[1*numBytes-len(xxxBytes):], xxxBytes) + copy(ret[2*numBytes-len(xxyBytes):], xxyBytes) + copy(ret[3*numBytes-len(xyxBytes):], xyxBytes) + copy(ret[4*numBytes-len(xyyBytes):], xyyBytes) + copy(ret[5*numBytes-len(xzxBytes):], xzxBytes) + copy(ret[6*numBytes-len(xzyBytes):], xzyBytes) + copy(ret[7*numBytes-len(yxxBytes):], yxxBytes) + copy(ret[8*numBytes-len(yxyBytes):], yxyBytes) + copy(ret[9*numBytes-len(yyxBytes):], yyxBytes) + copy(ret[10*numBytes-len(yyyBytes):], yyyBytes) + copy(ret[11*numBytes-len(yzxBytes):], yzxBytes) + copy(ret[12*numBytes-len(yzyBytes):], yzyBytes) + + return ret +} + +// Unmarshal sets e to the result of converting the output of Marshal back into +// a group element and then returns e. +func (e *GT) Unmarshal(m []byte) (*GT, bool) { + // Each value is a 256-bit number. + const numBytes = 256 / 8 + + if len(m) != 12*numBytes { + return nil, false + } + + if e.p == nil { + e.p = newGFp12(nil) + } + + e.p.x.x.x.SetBytes(m[0*numBytes : 1*numBytes]) + e.p.x.x.y.SetBytes(m[1*numBytes : 2*numBytes]) + e.p.x.y.x.SetBytes(m[2*numBytes : 3*numBytes]) + e.p.x.y.y.SetBytes(m[3*numBytes : 4*numBytes]) + e.p.x.z.x.SetBytes(m[4*numBytes : 5*numBytes]) + e.p.x.z.y.SetBytes(m[5*numBytes : 6*numBytes]) + e.p.y.x.x.SetBytes(m[6*numBytes : 7*numBytes]) + e.p.y.x.y.SetBytes(m[7*numBytes : 8*numBytes]) + e.p.y.y.x.SetBytes(m[8*numBytes : 9*numBytes]) + e.p.y.y.y.SetBytes(m[9*numBytes : 10*numBytes]) + e.p.y.z.x.SetBytes(m[10*numBytes : 11*numBytes]) + e.p.y.z.y.SetBytes(m[11*numBytes : 12*numBytes]) + + return e, true +} + +// Pair calculates an Optimal Ate pairing. +func Pair(g1 *G1, g2 *G2) *GT { + return >{optimalAte(g2.p, g1.p, new(bnPool))} +} + +func PairingCheck(a []*G1, b []*G2) bool { + pool := new(bnPool) + e := newGFp12(pool) + e.SetOne() + for i := 0; i < len(a); i++ { + new_e := miller(b[i].p, a[i].p, pool) + e.Mul(e, new_e, pool) + } + ret := finalExponentiation(e, pool) + e.Put(pool) + return ret.IsOne() +} + +// bnPool implements a tiny cache of *big.Int objects that's used to reduce the +// number of allocations made during processing. +type bnPool struct { + bns []*big.Int + count int +} + +func (pool *bnPool) Get() *big.Int { + if pool == nil { + return new(big.Int) + } + + pool.count++ + l := len(pool.bns) + if l == 0 { + return new(big.Int) + } + + bn := pool.bns[l-1] + pool.bns = pool.bns[:l-1] + return bn +} + +func (pool *bnPool) Put(bn *big.Int) { + if pool == nil { + return + } + pool.bns = append(pool.bns, bn) + pool.count-- +} + +func (pool *bnPool) Count() int { + return pool.count +} diff --git a/crypto/bn256/bn256_test.go b/crypto/bn256/bn256_test.go new file mode 100644 index 0000000000..866065d0ca --- /dev/null +++ b/crypto/bn256/bn256_test.go @@ -0,0 +1,304 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bn256 + +import ( + "bytes" + "crypto/rand" + "math/big" + "testing" +) + +func TestGFp2Invert(t *testing.T) { + pool := new(bnPool) + + a := newGFp2(pool) + a.x.SetString("23423492374", 10) + a.y.SetString("12934872398472394827398470", 10) + + inv := newGFp2(pool) + inv.Invert(a, pool) + + b := newGFp2(pool).Mul(inv, a, pool) + if b.x.Int64() != 0 || b.y.Int64() != 1 { + t.Fatalf("bad result for a^-1*a: %s %s", b.x, b.y) + } + + a.Put(pool) + b.Put(pool) + inv.Put(pool) + + if c := pool.Count(); c > 0 { + t.Errorf("Pool count non-zero: %d\n", c) + } +} + +func isZero(n *big.Int) bool { + return new(big.Int).Mod(n, P).Int64() == 0 +} + +func isOne(n *big.Int) bool { + return new(big.Int).Mod(n, P).Int64() == 1 +} + +func TestGFp6Invert(t *testing.T) { + pool := new(bnPool) + + a := newGFp6(pool) + a.x.x.SetString("239487238491", 10) + a.x.y.SetString("2356249827341", 10) + a.y.x.SetString("082659782", 10) + a.y.y.SetString("182703523765", 10) + a.z.x.SetString("978236549263", 10) + a.z.y.SetString("64893242", 10) + + inv := newGFp6(pool) + inv.Invert(a, pool) + + b := newGFp6(pool).Mul(inv, a, pool) + if !isZero(b.x.x) || + !isZero(b.x.y) || + !isZero(b.y.x) || + !isZero(b.y.y) || + !isZero(b.z.x) || + !isOne(b.z.y) { + t.Fatalf("bad result for a^-1*a: %s", b) + } + + a.Put(pool) + b.Put(pool) + inv.Put(pool) + + if c := pool.Count(); c > 0 { + t.Errorf("Pool count non-zero: %d\n", c) + } +} + +func TestGFp12Invert(t *testing.T) { + pool := new(bnPool) + + a := newGFp12(pool) + a.x.x.x.SetString("239846234862342323958623", 10) + a.x.x.y.SetString("2359862352529835623", 10) + a.x.y.x.SetString("928836523", 10) + a.x.y.y.SetString("9856234", 10) + a.x.z.x.SetString("235635286", 10) + a.x.z.y.SetString("5628392833", 10) + a.y.x.x.SetString("252936598265329856238956532167968", 10) + a.y.x.y.SetString("23596239865236954178968", 10) + a.y.y.x.SetString("95421692834", 10) + a.y.y.y.SetString("236548", 10) + a.y.z.x.SetString("924523", 10) + a.y.z.y.SetString("12954623", 10) + + inv := newGFp12(pool) + inv.Invert(a, pool) + + b := newGFp12(pool).Mul(inv, a, pool) + if !isZero(b.x.x.x) || + !isZero(b.x.x.y) || + !isZero(b.x.y.x) || + !isZero(b.x.y.y) || + !isZero(b.x.z.x) || + !isZero(b.x.z.y) || + !isZero(b.y.x.x) || + !isZero(b.y.x.y) || + !isZero(b.y.y.x) || + !isZero(b.y.y.y) || + !isZero(b.y.z.x) || + !isOne(b.y.z.y) { + t.Fatalf("bad result for a^-1*a: %s", b) + } + + a.Put(pool) + b.Put(pool) + inv.Put(pool) + + if c := pool.Count(); c > 0 { + t.Errorf("Pool count non-zero: %d\n", c) + } +} + +func TestCurveImpl(t *testing.T) { + pool := new(bnPool) + + g := &curvePoint{ + pool.Get().SetInt64(1), + pool.Get().SetInt64(-2), + pool.Get().SetInt64(1), + pool.Get().SetInt64(0), + } + + x := pool.Get().SetInt64(32498273234) + X := newCurvePoint(pool).Mul(g, x, pool) + + y := pool.Get().SetInt64(98732423523) + Y := newCurvePoint(pool).Mul(g, y, pool) + + s1 := newCurvePoint(pool).Mul(X, y, pool).MakeAffine(pool) + s2 := newCurvePoint(pool).Mul(Y, x, pool).MakeAffine(pool) + + if s1.x.Cmp(s2.x) != 0 || + s2.x.Cmp(s1.x) != 0 { + t.Errorf("DH points don't match: (%s, %s) (%s, %s)", s1.x, s1.y, s2.x, s2.y) + } + + pool.Put(x) + X.Put(pool) + pool.Put(y) + Y.Put(pool) + s1.Put(pool) + s2.Put(pool) + g.Put(pool) + + if c := pool.Count(); c > 0 { + t.Errorf("Pool count non-zero: %d\n", c) + } +} + +func TestOrderG1(t *testing.T) { + g := new(G1).ScalarBaseMult(Order) + if !g.p.IsInfinity() { + t.Error("G1 has incorrect order") + } + + one := new(G1).ScalarBaseMult(new(big.Int).SetInt64(1)) + g.Add(g, one) + g.p.MakeAffine(nil) + if g.p.x.Cmp(one.p.x) != 0 || g.p.y.Cmp(one.p.y) != 0 { + t.Errorf("1+0 != 1 in G1") + } +} + +func TestOrderG2(t *testing.T) { + g := new(G2).ScalarBaseMult(Order) + if !g.p.IsInfinity() { + t.Error("G2 has incorrect order") + } + + one := new(G2).ScalarBaseMult(new(big.Int).SetInt64(1)) + g.Add(g, one) + g.p.MakeAffine(nil) + if g.p.x.x.Cmp(one.p.x.x) != 0 || + g.p.x.y.Cmp(one.p.x.y) != 0 || + g.p.y.x.Cmp(one.p.y.x) != 0 || + g.p.y.y.Cmp(one.p.y.y) != 0 { + t.Errorf("1+0 != 1 in G2") + } +} + +func TestOrderGT(t *testing.T) { + gt := Pair(&G1{curveGen}, &G2{twistGen}) + g := new(GT).ScalarMult(gt, Order) + if !g.p.IsOne() { + t.Error("GT has incorrect order") + } +} + +func TestBilinearity(t *testing.T) { + for i := 0; i < 2; i++ { + a, p1, _ := RandomG1(rand.Reader) + b, p2, _ := RandomG2(rand.Reader) + e1 := Pair(p1, p2) + + e2 := Pair(&G1{curveGen}, &G2{twistGen}) + e2.ScalarMult(e2, a) + e2.ScalarMult(e2, b) + + minusE2 := new(GT).Neg(e2) + e1.Add(e1, minusE2) + + if !e1.p.IsOne() { + t.Fatalf("bad pairing result: %s", e1) + } + } +} + +func TestG1Marshal(t *testing.T) { + g := new(G1).ScalarBaseMult(new(big.Int).SetInt64(1)) + form := g.Marshal() + _, ok := new(G1).Unmarshal(form) + if !ok { + t.Fatalf("failed to unmarshal") + } + + g.ScalarBaseMult(Order) + form = g.Marshal() + g2, ok := new(G1).Unmarshal(form) + if !ok { + t.Fatalf("failed to unmarshal ∞") + } + if !g2.p.IsInfinity() { + t.Fatalf("∞ unmarshaled incorrectly") + } +} + +func TestG2Marshal(t *testing.T) { + g := new(G2).ScalarBaseMult(new(big.Int).SetInt64(1)) + form := g.Marshal() + _, ok := new(G2).Unmarshal(form) + if !ok { + t.Fatalf("failed to unmarshal") + } + + g.ScalarBaseMult(Order) + form = g.Marshal() + g2, ok := new(G2).Unmarshal(form) + if !ok { + t.Fatalf("failed to unmarshal ∞") + } + if !g2.p.IsInfinity() { + t.Fatalf("∞ unmarshaled incorrectly") + } +} + +func TestG1Identity(t *testing.T) { + g := new(G1).ScalarBaseMult(new(big.Int).SetInt64(0)) + if !g.p.IsInfinity() { + t.Error("failure") + } +} + +func TestG2Identity(t *testing.T) { + g := new(G2).ScalarBaseMult(new(big.Int).SetInt64(0)) + if !g.p.IsInfinity() { + t.Error("failure") + } +} + +func TestTripartiteDiffieHellman(t *testing.T) { + a, _ := rand.Int(rand.Reader, Order) + b, _ := rand.Int(rand.Reader, Order) + c, _ := rand.Int(rand.Reader, Order) + + pa, _ := new(G1).Unmarshal(new(G1).ScalarBaseMult(a).Marshal()) + qa, _ := new(G2).Unmarshal(new(G2).ScalarBaseMult(a).Marshal()) + pb, _ := new(G1).Unmarshal(new(G1).ScalarBaseMult(b).Marshal()) + qb, _ := new(G2).Unmarshal(new(G2).ScalarBaseMult(b).Marshal()) + pc, _ := new(G1).Unmarshal(new(G1).ScalarBaseMult(c).Marshal()) + qc, _ := new(G2).Unmarshal(new(G2).ScalarBaseMult(c).Marshal()) + + k1 := Pair(pb, qc) + k1.ScalarMult(k1, a) + k1Bytes := k1.Marshal() + + k2 := Pair(pc, qa) + k2.ScalarMult(k2, b) + k2Bytes := k2.Marshal() + + k3 := Pair(pa, qb) + k3.ScalarMult(k3, c) + k3Bytes := k3.Marshal() + + if !bytes.Equal(k1Bytes, k2Bytes) || !bytes.Equal(k2Bytes, k3Bytes) { + t.Errorf("keys didn't agree") + } +} + +func BenchmarkPairing(b *testing.B) { + for i := 0; i < b.N; i++ { + Pair(&G1{curveGen}, &G2{twistGen}) + } +} diff --git a/crypto/bn256/constants.go b/crypto/bn256/constants.go new file mode 100644 index 0000000000..ab649d7f3f --- /dev/null +++ b/crypto/bn256/constants.go @@ -0,0 +1,44 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bn256 + +import ( + "math/big" +) + +func bigFromBase10(s string) *big.Int { + n, _ := new(big.Int).SetString(s, 10) + return n +} + +// u is the BN parameter that determines the prime: 1868033³. +var u = bigFromBase10("4965661367192848881") + +// p is a prime over which we form a basic field: 36u⁴+36u³+24u²+6u+1. +var P = bigFromBase10("21888242871839275222246405745257275088696311157297823662689037894645226208583") + +// Order is the number of elements in both G₁ and G₂: 36u⁴+36u³+18u²+6u+1. +var Order = bigFromBase10("21888242871839275222246405745257275088548364400416034343698204186575808495617") + +// xiToPMinus1Over6 is ξ^((p-1)/6) where ξ = i+9. +var xiToPMinus1Over6 = &gfP2{bigFromBase10("16469823323077808223889137241176536799009286646108169935659301613961712198316"), bigFromBase10("8376118865763821496583973867626364092589906065868298776909617916018768340080")} + +// xiToPMinus1Over3 is ξ^((p-1)/3) where ξ = i+9. +var xiToPMinus1Over3 = &gfP2{bigFromBase10("10307601595873709700152284273816112264069230130616436755625194854815875713954"), bigFromBase10("21575463638280843010398324269430826099269044274347216827212613867836435027261")} + +// xiToPMinus1Over2 is ξ^((p-1)/2) where ξ = i+9. +var xiToPMinus1Over2 = &gfP2{bigFromBase10("3505843767911556378687030309984248845540243509899259641013678093033130930403"), bigFromBase10("2821565182194536844548159561693502659359617185244120367078079554186484126554")} + +// xiToPSquaredMinus1Over3 is ξ^((p²-1)/3) where ξ = i+9. +var xiToPSquaredMinus1Over3 = bigFromBase10("21888242871839275220042445260109153167277707414472061641714758635765020556616") + +// xiTo2PSquaredMinus2Over3 is ξ^((2p²-2)/3) where ξ = i+9 (a cubic root of unity, mod p). +var xiTo2PSquaredMinus2Over3 = bigFromBase10("2203960485148121921418603742825762020974279258880205651966") + +// xiToPSquaredMinus1Over6 is ξ^((1p²-1)/6) where ξ = i+9 (a cubic root of -1, mod p). +var xiToPSquaredMinus1Over6 = bigFromBase10("21888242871839275220042445260109153167277707414472061641714758635765020556617") + +// xiTo2PMinus2Over3 is ξ^((2p-2)/3) where ξ = i+9. +var xiTo2PMinus2Over3 = &gfP2{bigFromBase10("19937756971775647987995932169929341994314640652964949448313374472400716661030"), bigFromBase10("2581911344467009335267311115468803099551665605076196740867805258568234346338")} diff --git a/crypto/bn256/curve.go b/crypto/bn256/curve.go new file mode 100644 index 0000000000..233b1f2521 --- /dev/null +++ b/crypto/bn256/curve.go @@ -0,0 +1,278 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bn256 + +import ( + "math/big" +) + +// curvePoint implements the elliptic curve y²=x³+3. Points are kept in +// Jacobian form and t=z² when valid. G₁ is the set of points of this curve on +// GF(p). +type curvePoint struct { + x, y, z, t *big.Int +} + +var curveB = new(big.Int).SetInt64(3) + +// curveGen is the generator of G₁. +var curveGen = &curvePoint{ + new(big.Int).SetInt64(1), + new(big.Int).SetInt64(-2), + new(big.Int).SetInt64(1), + new(big.Int).SetInt64(1), +} + +func newCurvePoint(pool *bnPool) *curvePoint { + return &curvePoint{ + pool.Get(), + pool.Get(), + pool.Get(), + pool.Get(), + } +} + +func (c *curvePoint) String() string { + c.MakeAffine(new(bnPool)) + return "(" + c.x.String() + ", " + c.y.String() + ")" +} + +func (c *curvePoint) Put(pool *bnPool) { + pool.Put(c.x) + pool.Put(c.y) + pool.Put(c.z) + pool.Put(c.t) +} + +func (c *curvePoint) Set(a *curvePoint) { + c.x.Set(a.x) + c.y.Set(a.y) + c.z.Set(a.z) + c.t.Set(a.t) +} + +// IsOnCurve returns true iff c is on the curve where c must be in affine form. +func (c *curvePoint) IsOnCurve() bool { + yy := new(big.Int).Mul(c.y, c.y) + xxx := new(big.Int).Mul(c.x, c.x) + xxx.Mul(xxx, c.x) + yy.Sub(yy, xxx) + yy.Sub(yy, curveB) + if yy.Sign() < 0 || yy.Cmp(P) >= 0 { + yy.Mod(yy, P) + } + return yy.Sign() == 0 +} + +func (c *curvePoint) SetInfinity() { + c.z.SetInt64(0) +} + +func (c *curvePoint) IsInfinity() bool { + return c.z.Sign() == 0 +} + +func (c *curvePoint) Add(a, b *curvePoint, pool *bnPool) { + if a.IsInfinity() { + c.Set(b) + return + } + if b.IsInfinity() { + c.Set(a) + return + } + + // See http://hyperelliptic.org/EFD/g1p/auto-code/shortw/jacobian-0/addition/add-2007-bl.op3 + + // Normalize the points by replacing a = [x1:y1:z1] and b = [x2:y2:z2] + // by [u1:s1:z1·z2] and [u2:s2:z1·z2] + // where u1 = x1·z2², s1 = y1·z2³ and u1 = x2·z1², s2 = y2·z1³ + z1z1 := pool.Get().Mul(a.z, a.z) + z1z1.Mod(z1z1, P) + z2z2 := pool.Get().Mul(b.z, b.z) + z2z2.Mod(z2z2, P) + u1 := pool.Get().Mul(a.x, z2z2) + u1.Mod(u1, P) + u2 := pool.Get().Mul(b.x, z1z1) + u2.Mod(u2, P) + + t := pool.Get().Mul(b.z, z2z2) + t.Mod(t, P) + s1 := pool.Get().Mul(a.y, t) + s1.Mod(s1, P) + + t.Mul(a.z, z1z1) + t.Mod(t, P) + s2 := pool.Get().Mul(b.y, t) + s2.Mod(s2, P) + + // Compute x = (2h)²(s²-u1-u2) + // where s = (s2-s1)/(u2-u1) is the slope of the line through + // (u1,s1) and (u2,s2). The extra factor 2h = 2(u2-u1) comes from the value of z below. + // This is also: + // 4(s2-s1)² - 4h²(u1+u2) = 4(s2-s1)² - 4h³ - 4h²(2u1) + // = r² - j - 2v + // with the notations below. + h := pool.Get().Sub(u2, u1) + xEqual := h.Sign() == 0 + + t.Add(h, h) + // i = 4h² + i := pool.Get().Mul(t, t) + i.Mod(i, P) + // j = 4h³ + j := pool.Get().Mul(h, i) + j.Mod(j, P) + + t.Sub(s2, s1) + yEqual := t.Sign() == 0 + if xEqual && yEqual { + c.Double(a, pool) + return + } + r := pool.Get().Add(t, t) + + v := pool.Get().Mul(u1, i) + v.Mod(v, P) + + // t4 = 4(s2-s1)² + t4 := pool.Get().Mul(r, r) + t4.Mod(t4, P) + t.Add(v, v) + t6 := pool.Get().Sub(t4, j) + c.x.Sub(t6, t) + + // Set y = -(2h)³(s1 + s*(x/4h²-u1)) + // This is also + // y = - 2·s1·j - (s2-s1)(2x - 2i·u1) = r(v-x) - 2·s1·j + t.Sub(v, c.x) // t7 + t4.Mul(s1, j) // t8 + t4.Mod(t4, P) + t6.Add(t4, t4) // t9 + t4.Mul(r, t) // t10 + t4.Mod(t4, P) + c.y.Sub(t4, t6) + + // Set z = 2(u2-u1)·z1·z2 = 2h·z1·z2 + t.Add(a.z, b.z) // t11 + t4.Mul(t, t) // t12 + t4.Mod(t4, P) + t.Sub(t4, z1z1) // t13 + t4.Sub(t, z2z2) // t14 + c.z.Mul(t4, h) + c.z.Mod(c.z, P) + + pool.Put(z1z1) + pool.Put(z2z2) + pool.Put(u1) + pool.Put(u2) + pool.Put(t) + pool.Put(s1) + pool.Put(s2) + pool.Put(h) + pool.Put(i) + pool.Put(j) + pool.Put(r) + pool.Put(v) + pool.Put(t4) + pool.Put(t6) +} + +func (c *curvePoint) Double(a *curvePoint, pool *bnPool) { + // See http://hyperelliptic.org/EFD/g1p/auto-code/shortw/jacobian-0/doubling/dbl-2009-l.op3 + A := pool.Get().Mul(a.x, a.x) + A.Mod(A, P) + B := pool.Get().Mul(a.y, a.y) + B.Mod(B, P) + C_ := pool.Get().Mul(B, B) + C_.Mod(C_, P) + + t := pool.Get().Add(a.x, B) + t2 := pool.Get().Mul(t, t) + t2.Mod(t2, P) + t.Sub(t2, A) + t2.Sub(t, C_) + d := pool.Get().Add(t2, t2) + t.Add(A, A) + e := pool.Get().Add(t, A) + f := pool.Get().Mul(e, e) + f.Mod(f, P) + + t.Add(d, d) + c.x.Sub(f, t) + + t.Add(C_, C_) + t2.Add(t, t) + t.Add(t2, t2) + c.y.Sub(d, c.x) + t2.Mul(e, c.y) + t2.Mod(t2, P) + c.y.Sub(t2, t) + + t.Mul(a.y, a.z) + t.Mod(t, P) + c.z.Add(t, t) + + pool.Put(A) + pool.Put(B) + pool.Put(C_) + pool.Put(t) + pool.Put(t2) + pool.Put(d) + pool.Put(e) + pool.Put(f) +} + +func (c *curvePoint) Mul(a *curvePoint, scalar *big.Int, pool *bnPool) *curvePoint { + sum := newCurvePoint(pool) + sum.SetInfinity() + t := newCurvePoint(pool) + + for i := scalar.BitLen(); i >= 0; i-- { + t.Double(sum, pool) + if scalar.Bit(i) != 0 { + sum.Add(t, a, pool) + } else { + sum.Set(t) + } + } + + c.Set(sum) + sum.Put(pool) + t.Put(pool) + return c +} + +func (c *curvePoint) MakeAffine(pool *bnPool) *curvePoint { + if words := c.z.Bits(); len(words) == 1 && words[0] == 1 { + return c + } + + zInv := pool.Get().ModInverse(c.z, P) + t := pool.Get().Mul(c.y, zInv) + t.Mod(t, P) + zInv2 := pool.Get().Mul(zInv, zInv) + zInv2.Mod(zInv2, P) + c.y.Mul(t, zInv2) + c.y.Mod(c.y, P) + t.Mul(c.x, zInv2) + t.Mod(t, P) + c.x.Set(t) + c.z.SetInt64(1) + c.t.SetInt64(1) + + pool.Put(zInv) + pool.Put(t) + pool.Put(zInv2) + + return c +} + +func (c *curvePoint) Negative(a *curvePoint) { + c.x.Set(a.x) + c.y.Neg(a.y) + c.z.Set(a.z) + c.t.SetInt64(0) +} diff --git a/crypto/bn256/example_test.go b/crypto/bn256/example_test.go new file mode 100644 index 0000000000..b2d19807a2 --- /dev/null +++ b/crypto/bn256/example_test.go @@ -0,0 +1,43 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bn256 + +import ( + "crypto/rand" +) + +func ExamplePair() { + // This implements the tripartite Diffie-Hellman algorithm from "A One + // Round Protocol for Tripartite Diffie-Hellman", A. Joux. + // http://www.springerlink.com/content/cddc57yyva0hburb/fulltext.pdf + + // Each of three parties, a, b and c, generate a private value. + a, _ := rand.Int(rand.Reader, Order) + b, _ := rand.Int(rand.Reader, Order) + c, _ := rand.Int(rand.Reader, Order) + + // Then each party calculates g₁ and g₂ times their private value. + pa := new(G1).ScalarBaseMult(a) + qa := new(G2).ScalarBaseMult(a) + + pb := new(G1).ScalarBaseMult(b) + qb := new(G2).ScalarBaseMult(b) + + pc := new(G1).ScalarBaseMult(c) + qc := new(G2).ScalarBaseMult(c) + + // Now each party exchanges its public values with the other two and + // all parties can calculate the shared key. + k1 := Pair(pb, qc) + k1.ScalarMult(k1, a) + + k2 := Pair(pc, qa) + k2.ScalarMult(k2, b) + + k3 := Pair(pa, qb) + k3.ScalarMult(k3, c) + + // k1, k2 and k3 will all be equal. +} diff --git a/crypto/bn256/gfp12.go b/crypto/bn256/gfp12.go new file mode 100644 index 0000000000..f084eddf21 --- /dev/null +++ b/crypto/bn256/gfp12.go @@ -0,0 +1,200 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bn256 + +// For details of the algorithms used, see "Multiplication and Squaring on +// Pairing-Friendly Fields, Devegili et al. +// http://eprint.iacr.org/2006/471.pdf. + +import ( + "math/big" +) + +// gfP12 implements the field of size p¹² as a quadratic extension of gfP6 +// where ω²=τ. +type gfP12 struct { + x, y *gfP6 // value is xω + y +} + +func newGFp12(pool *bnPool) *gfP12 { + return &gfP12{newGFp6(pool), newGFp6(pool)} +} + +func (e *gfP12) String() string { + return "(" + e.x.String() + "," + e.y.String() + ")" +} + +func (e *gfP12) Put(pool *bnPool) { + e.x.Put(pool) + e.y.Put(pool) +} + +func (e *gfP12) Set(a *gfP12) *gfP12 { + e.x.Set(a.x) + e.y.Set(a.y) + return e +} + +func (e *gfP12) SetZero() *gfP12 { + e.x.SetZero() + e.y.SetZero() + return e +} + +func (e *gfP12) SetOne() *gfP12 { + e.x.SetZero() + e.y.SetOne() + return e +} + +func (e *gfP12) Minimal() { + e.x.Minimal() + e.y.Minimal() +} + +func (e *gfP12) IsZero() bool { + e.Minimal() + return e.x.IsZero() && e.y.IsZero() +} + +func (e *gfP12) IsOne() bool { + e.Minimal() + return e.x.IsZero() && e.y.IsOne() +} + +func (e *gfP12) Conjugate(a *gfP12) *gfP12 { + e.x.Negative(a.x) + e.y.Set(a.y) + return a +} + +func (e *gfP12) Negative(a *gfP12) *gfP12 { + e.x.Negative(a.x) + e.y.Negative(a.y) + return e +} + +// Frobenius computes (xω+y)^p = x^p ω·ξ^((p-1)/6) + y^p +func (e *gfP12) Frobenius(a *gfP12, pool *bnPool) *gfP12 { + e.x.Frobenius(a.x, pool) + e.y.Frobenius(a.y, pool) + e.x.MulScalar(e.x, xiToPMinus1Over6, pool) + return e +} + +// FrobeniusP2 computes (xω+y)^p² = x^p² ω·ξ^((p²-1)/6) + y^p² +func (e *gfP12) FrobeniusP2(a *gfP12, pool *bnPool) *gfP12 { + e.x.FrobeniusP2(a.x) + e.x.MulGFP(e.x, xiToPSquaredMinus1Over6) + e.y.FrobeniusP2(a.y) + return e +} + +func (e *gfP12) Add(a, b *gfP12) *gfP12 { + e.x.Add(a.x, b.x) + e.y.Add(a.y, b.y) + return e +} + +func (e *gfP12) Sub(a, b *gfP12) *gfP12 { + e.x.Sub(a.x, b.x) + e.y.Sub(a.y, b.y) + return e +} + +func (e *gfP12) Mul(a, b *gfP12, pool *bnPool) *gfP12 { + tx := newGFp6(pool) + tx.Mul(a.x, b.y, pool) + t := newGFp6(pool) + t.Mul(b.x, a.y, pool) + tx.Add(tx, t) + + ty := newGFp6(pool) + ty.Mul(a.y, b.y, pool) + t.Mul(a.x, b.x, pool) + t.MulTau(t, pool) + e.y.Add(ty, t) + e.x.Set(tx) + + tx.Put(pool) + ty.Put(pool) + t.Put(pool) + return e +} + +func (e *gfP12) MulScalar(a *gfP12, b *gfP6, pool *bnPool) *gfP12 { + e.x.Mul(e.x, b, pool) + e.y.Mul(e.y, b, pool) + return e +} + +func (c *gfP12) Exp(a *gfP12, power *big.Int, pool *bnPool) *gfP12 { + sum := newGFp12(pool) + sum.SetOne() + t := newGFp12(pool) + + for i := power.BitLen() - 1; i >= 0; i-- { + t.Square(sum, pool) + if power.Bit(i) != 0 { + sum.Mul(t, a, pool) + } else { + sum.Set(t) + } + } + + c.Set(sum) + + sum.Put(pool) + t.Put(pool) + + return c +} + +func (e *gfP12) Square(a *gfP12, pool *bnPool) *gfP12 { + // Complex squaring algorithm + v0 := newGFp6(pool) + v0.Mul(a.x, a.y, pool) + + t := newGFp6(pool) + t.MulTau(a.x, pool) + t.Add(a.y, t) + ty := newGFp6(pool) + ty.Add(a.x, a.y) + ty.Mul(ty, t, pool) + ty.Sub(ty, v0) + t.MulTau(v0, pool) + ty.Sub(ty, t) + + e.y.Set(ty) + e.x.Double(v0) + + v0.Put(pool) + t.Put(pool) + ty.Put(pool) + + return e +} + +func (e *gfP12) Invert(a *gfP12, pool *bnPool) *gfP12 { + // See "Implementing cryptographic pairings", M. Scott, section 3.2. + // ftp://136.206.11.249/pub/crypto/pairings.pdf + t1 := newGFp6(pool) + t2 := newGFp6(pool) + + t1.Square(a.x, pool) + t2.Square(a.y, pool) + t1.MulTau(t1, pool) + t1.Sub(t2, t1) + t2.Invert(t1, pool) + + e.x.Negative(a.x) + e.y.Set(a.y) + e.MulScalar(e, t2, pool) + + t1.Put(pool) + t2.Put(pool) + + return e +} diff --git a/crypto/bn256/gfp2.go b/crypto/bn256/gfp2.go new file mode 100644 index 0000000000..3981f6cb4f --- /dev/null +++ b/crypto/bn256/gfp2.go @@ -0,0 +1,227 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bn256 + +// For details of the algorithms used, see "Multiplication and Squaring on +// Pairing-Friendly Fields, Devegili et al. +// http://eprint.iacr.org/2006/471.pdf. + +import ( + "math/big" +) + +// gfP2 implements a field of size p² as a quadratic extension of the base +// field where i²=-1. +type gfP2 struct { + x, y *big.Int // value is xi+y. +} + +func newGFp2(pool *bnPool) *gfP2 { + return &gfP2{pool.Get(), pool.Get()} +} + +func (e *gfP2) String() string { + x := new(big.Int).Mod(e.x, P) + y := new(big.Int).Mod(e.y, P) + return "(" + x.String() + "," + y.String() + ")" +} + +func (e *gfP2) Put(pool *bnPool) { + pool.Put(e.x) + pool.Put(e.y) +} + +func (e *gfP2) Set(a *gfP2) *gfP2 { + e.x.Set(a.x) + e.y.Set(a.y) + return e +} + +func (e *gfP2) SetZero() *gfP2 { + e.x.SetInt64(0) + e.y.SetInt64(0) + return e +} + +func (e *gfP2) SetOne() *gfP2 { + e.x.SetInt64(0) + e.y.SetInt64(1) + return e +} + +func (e *gfP2) Minimal() { + if e.x.Sign() < 0 || e.x.Cmp(P) >= 0 { + e.x.Mod(e.x, P) + } + if e.y.Sign() < 0 || e.y.Cmp(P) >= 0 { + e.y.Mod(e.y, P) + } +} + +func (e *gfP2) IsZero() bool { + return e.x.Sign() == 0 && e.y.Sign() == 0 +} + +func (e *gfP2) IsOne() bool { + if e.x.Sign() != 0 { + return false + } + words := e.y.Bits() + return len(words) == 1 && words[0] == 1 +} + +func (e *gfP2) Conjugate(a *gfP2) *gfP2 { + e.y.Set(a.y) + e.x.Neg(a.x) + return e +} + +func (e *gfP2) Negative(a *gfP2) *gfP2 { + e.x.Neg(a.x) + e.y.Neg(a.y) + return e +} + +func (e *gfP2) Add(a, b *gfP2) *gfP2 { + e.x.Add(a.x, b.x) + e.y.Add(a.y, b.y) + return e +} + +func (e *gfP2) Sub(a, b *gfP2) *gfP2 { + e.x.Sub(a.x, b.x) + e.y.Sub(a.y, b.y) + return e +} + +func (e *gfP2) Double(a *gfP2) *gfP2 { + e.x.Lsh(a.x, 1) + e.y.Lsh(a.y, 1) + return e +} + +func (c *gfP2) Exp(a *gfP2, power *big.Int, pool *bnPool) *gfP2 { + sum := newGFp2(pool) + sum.SetOne() + t := newGFp2(pool) + + for i := power.BitLen() - 1; i >= 0; i-- { + t.Square(sum, pool) + if power.Bit(i) != 0 { + sum.Mul(t, a, pool) + } else { + sum.Set(t) + } + } + + c.Set(sum) + + sum.Put(pool) + t.Put(pool) + + return c +} + +// See "Multiplication and Squaring in Pairing-Friendly Fields", +// http://eprint.iacr.org/2006/471.pdf +func (e *gfP2) Mul(a, b *gfP2, pool *bnPool) *gfP2 { + tx := pool.Get().Mul(a.x, b.y) + t := pool.Get().Mul(b.x, a.y) + tx.Add(tx, t) + tx.Mod(tx, P) + + ty := pool.Get().Mul(a.y, b.y) + t.Mul(a.x, b.x) + ty.Sub(ty, t) + e.y.Mod(ty, P) + e.x.Set(tx) + + pool.Put(tx) + pool.Put(ty) + pool.Put(t) + + return e +} + +func (e *gfP2) MulScalar(a *gfP2, b *big.Int) *gfP2 { + e.x.Mul(a.x, b) + e.y.Mul(a.y, b) + return e +} + +// MulXi sets e=ξa where ξ=i+9 and then returns e. +func (e *gfP2) MulXi(a *gfP2, pool *bnPool) *gfP2 { + // (xi+y)(i+3) = (9x+y)i+(9y-x) + tx := pool.Get().Lsh(a.x, 3) + tx.Add(tx, a.x) + tx.Add(tx, a.y) + + ty := pool.Get().Lsh(a.y, 3) + ty.Add(ty, a.y) + ty.Sub(ty, a.x) + + e.x.Set(tx) + e.y.Set(ty) + + pool.Put(tx) + pool.Put(ty) + + return e +} + +func (e *gfP2) Square(a *gfP2, pool *bnPool) *gfP2 { + // Complex squaring algorithm: + // (xi+b)² = (x+y)(y-x) + 2*i*x*y + t1 := pool.Get().Sub(a.y, a.x) + t2 := pool.Get().Add(a.x, a.y) + ty := pool.Get().Mul(t1, t2) + ty.Mod(ty, P) + + t1.Mul(a.x, a.y) + t1.Lsh(t1, 1) + + e.x.Mod(t1, P) + e.y.Set(ty) + + pool.Put(t1) + pool.Put(t2) + pool.Put(ty) + + return e +} + +func (e *gfP2) Invert(a *gfP2, pool *bnPool) *gfP2 { + // See "Implementing cryptographic pairings", M. Scott, section 3.2. + // ftp://136.206.11.249/pub/crypto/pairings.pdf + t := pool.Get() + t.Mul(a.y, a.y) + t2 := pool.Get() + t2.Mul(a.x, a.x) + t.Add(t, t2) + + inv := pool.Get() + inv.ModInverse(t, P) + + e.x.Neg(a.x) + e.x.Mul(e.x, inv) + e.x.Mod(e.x, P) + + e.y.Mul(a.y, inv) + e.y.Mod(e.y, P) + + pool.Put(t) + pool.Put(t2) + pool.Put(inv) + + return e +} + +func (e *gfP2) Real() *big.Int { + return e.x +} + +func (e *gfP2) Imag() *big.Int { + return e.y +} diff --git a/crypto/bn256/gfp6.go b/crypto/bn256/gfp6.go new file mode 100644 index 0000000000..218856617c --- /dev/null +++ b/crypto/bn256/gfp6.go @@ -0,0 +1,296 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bn256 + +// For details of the algorithms used, see "Multiplication and Squaring on +// Pairing-Friendly Fields, Devegili et al. +// http://eprint.iacr.org/2006/471.pdf. + +import ( + "math/big" +) + +// gfP6 implements the field of size p⁶ as a cubic extension of gfP2 where τ³=ξ +// and ξ=i+9. +type gfP6 struct { + x, y, z *gfP2 // value is xτ² + yτ + z +} + +func newGFp6(pool *bnPool) *gfP6 { + return &gfP6{newGFp2(pool), newGFp2(pool), newGFp2(pool)} +} + +func (e *gfP6) String() string { + return "(" + e.x.String() + "," + e.y.String() + "," + e.z.String() + ")" +} + +func (e *gfP6) Put(pool *bnPool) { + e.x.Put(pool) + e.y.Put(pool) + e.z.Put(pool) +} + +func (e *gfP6) Set(a *gfP6) *gfP6 { + e.x.Set(a.x) + e.y.Set(a.y) + e.z.Set(a.z) + return e +} + +func (e *gfP6) SetZero() *gfP6 { + e.x.SetZero() + e.y.SetZero() + e.z.SetZero() + return e +} + +func (e *gfP6) SetOne() *gfP6 { + e.x.SetZero() + e.y.SetZero() + e.z.SetOne() + return e +} + +func (e *gfP6) Minimal() { + e.x.Minimal() + e.y.Minimal() + e.z.Minimal() +} + +func (e *gfP6) IsZero() bool { + return e.x.IsZero() && e.y.IsZero() && e.z.IsZero() +} + +func (e *gfP6) IsOne() bool { + return e.x.IsZero() && e.y.IsZero() && e.z.IsOne() +} + +func (e *gfP6) Negative(a *gfP6) *gfP6 { + e.x.Negative(a.x) + e.y.Negative(a.y) + e.z.Negative(a.z) + return e +} + +func (e *gfP6) Frobenius(a *gfP6, pool *bnPool) *gfP6 { + e.x.Conjugate(a.x) + e.y.Conjugate(a.y) + e.z.Conjugate(a.z) + + e.x.Mul(e.x, xiTo2PMinus2Over3, pool) + e.y.Mul(e.y, xiToPMinus1Over3, pool) + return e +} + +// FrobeniusP2 computes (xτ²+yτ+z)^(p²) = xτ^(2p²) + yτ^(p²) + z +func (e *gfP6) FrobeniusP2(a *gfP6) *gfP6 { + // τ^(2p²) = τ²τ^(2p²-2) = τ²ξ^((2p²-2)/3) + e.x.MulScalar(a.x, xiTo2PSquaredMinus2Over3) + // τ^(p²) = ττ^(p²-1) = τξ^((p²-1)/3) + e.y.MulScalar(a.y, xiToPSquaredMinus1Over3) + e.z.Set(a.z) + return e +} + +func (e *gfP6) Add(a, b *gfP6) *gfP6 { + e.x.Add(a.x, b.x) + e.y.Add(a.y, b.y) + e.z.Add(a.z, b.z) + return e +} + +func (e *gfP6) Sub(a, b *gfP6) *gfP6 { + e.x.Sub(a.x, b.x) + e.y.Sub(a.y, b.y) + e.z.Sub(a.z, b.z) + return e +} + +func (e *gfP6) Double(a *gfP6) *gfP6 { + e.x.Double(a.x) + e.y.Double(a.y) + e.z.Double(a.z) + return e +} + +func (e *gfP6) Mul(a, b *gfP6, pool *bnPool) *gfP6 { + // "Multiplication and Squaring on Pairing-Friendly Fields" + // Section 4, Karatsuba method. + // http://eprint.iacr.org/2006/471.pdf + + v0 := newGFp2(pool) + v0.Mul(a.z, b.z, pool) + v1 := newGFp2(pool) + v1.Mul(a.y, b.y, pool) + v2 := newGFp2(pool) + v2.Mul(a.x, b.x, pool) + + t0 := newGFp2(pool) + t0.Add(a.x, a.y) + t1 := newGFp2(pool) + t1.Add(b.x, b.y) + tz := newGFp2(pool) + tz.Mul(t0, t1, pool) + + tz.Sub(tz, v1) + tz.Sub(tz, v2) + tz.MulXi(tz, pool) + tz.Add(tz, v0) + + t0.Add(a.y, a.z) + t1.Add(b.y, b.z) + ty := newGFp2(pool) + ty.Mul(t0, t1, pool) + ty.Sub(ty, v0) + ty.Sub(ty, v1) + t0.MulXi(v2, pool) + ty.Add(ty, t0) + + t0.Add(a.x, a.z) + t1.Add(b.x, b.z) + tx := newGFp2(pool) + tx.Mul(t0, t1, pool) + tx.Sub(tx, v0) + tx.Add(tx, v1) + tx.Sub(tx, v2) + + e.x.Set(tx) + e.y.Set(ty) + e.z.Set(tz) + + t0.Put(pool) + t1.Put(pool) + tx.Put(pool) + ty.Put(pool) + tz.Put(pool) + v0.Put(pool) + v1.Put(pool) + v2.Put(pool) + return e +} + +func (e *gfP6) MulScalar(a *gfP6, b *gfP2, pool *bnPool) *gfP6 { + e.x.Mul(a.x, b, pool) + e.y.Mul(a.y, b, pool) + e.z.Mul(a.z, b, pool) + return e +} + +func (e *gfP6) MulGFP(a *gfP6, b *big.Int) *gfP6 { + e.x.MulScalar(a.x, b) + e.y.MulScalar(a.y, b) + e.z.MulScalar(a.z, b) + return e +} + +// MulTau computes τ·(aτ²+bτ+c) = bτ²+cτ+aξ +func (e *gfP6) MulTau(a *gfP6, pool *bnPool) { + tz := newGFp2(pool) + tz.MulXi(a.x, pool) + ty := newGFp2(pool) + ty.Set(a.y) + e.y.Set(a.z) + e.x.Set(ty) + e.z.Set(tz) + tz.Put(pool) + ty.Put(pool) +} + +func (e *gfP6) Square(a *gfP6, pool *bnPool) *gfP6 { + v0 := newGFp2(pool).Square(a.z, pool) + v1 := newGFp2(pool).Square(a.y, pool) + v2 := newGFp2(pool).Square(a.x, pool) + + c0 := newGFp2(pool).Add(a.x, a.y) + c0.Square(c0, pool) + c0.Sub(c0, v1) + c0.Sub(c0, v2) + c0.MulXi(c0, pool) + c0.Add(c0, v0) + + c1 := newGFp2(pool).Add(a.y, a.z) + c1.Square(c1, pool) + c1.Sub(c1, v0) + c1.Sub(c1, v1) + xiV2 := newGFp2(pool).MulXi(v2, pool) + c1.Add(c1, xiV2) + + c2 := newGFp2(pool).Add(a.x, a.z) + c2.Square(c2, pool) + c2.Sub(c2, v0) + c2.Add(c2, v1) + c2.Sub(c2, v2) + + e.x.Set(c2) + e.y.Set(c1) + e.z.Set(c0) + + v0.Put(pool) + v1.Put(pool) + v2.Put(pool) + c0.Put(pool) + c1.Put(pool) + c2.Put(pool) + xiV2.Put(pool) + + return e +} + +func (e *gfP6) Invert(a *gfP6, pool *bnPool) *gfP6 { + // See "Implementing cryptographic pairings", M. Scott, section 3.2. + // ftp://136.206.11.249/pub/crypto/pairings.pdf + + // Here we can give a short explanation of how it works: let j be a cubic root of + // unity in GF(p²) so that 1+j+j²=0. + // Then (xτ² + yτ + z)(xj²τ² + yjτ + z)(xjτ² + yj²τ + z) + // = (xτ² + yτ + z)(Cτ²+Bτ+A) + // = (x³ξ²+y³ξ+z³-3ξxyz) = F is an element of the base field (the norm). + // + // On the other hand (xj²τ² + yjτ + z)(xjτ² + yj²τ + z) + // = τ²(y²-ξxz) + τ(ξx²-yz) + (z²-ξxy) + // + // So that's why A = (z²-ξxy), B = (ξx²-yz), C = (y²-ξxz) + t1 := newGFp2(pool) + + A := newGFp2(pool) + A.Square(a.z, pool) + t1.Mul(a.x, a.y, pool) + t1.MulXi(t1, pool) + A.Sub(A, t1) + + B := newGFp2(pool) + B.Square(a.x, pool) + B.MulXi(B, pool) + t1.Mul(a.y, a.z, pool) + B.Sub(B, t1) + + C_ := newGFp2(pool) + C_.Square(a.y, pool) + t1.Mul(a.x, a.z, pool) + C_.Sub(C_, t1) + + F := newGFp2(pool) + F.Mul(C_, a.y, pool) + F.MulXi(F, pool) + t1.Mul(A, a.z, pool) + F.Add(F, t1) + t1.Mul(B, a.x, pool) + t1.MulXi(t1, pool) + F.Add(F, t1) + + F.Invert(F, pool) + + e.x.Mul(C_, F, pool) + e.y.Mul(B, F, pool) + e.z.Mul(A, F, pool) + + t1.Put(pool) + A.Put(pool) + B.Put(pool) + C_.Put(pool) + F.Put(pool) + + return e +} diff --git a/crypto/bn256/main_test.go b/crypto/bn256/main_test.go new file mode 100644 index 0000000000..0230f1b199 --- /dev/null +++ b/crypto/bn256/main_test.go @@ -0,0 +1,71 @@ +package bn256 + +import ( + "testing" + + "crypto/rand" +) + +func TestRandomG2Marshal(t *testing.T) { + for i := 0; i < 10; i++ { + n, g2, err := RandomG2(rand.Reader) + if err != nil { + t.Error(err) + continue + } + t.Logf("%d: %x\n", n, g2.Marshal()) + } +} + +func TestPairings(t *testing.T) { + a1 := new(G1).ScalarBaseMult(bigFromBase10("1")) + a2 := new(G1).ScalarBaseMult(bigFromBase10("2")) + a37 := new(G1).ScalarBaseMult(bigFromBase10("37")) + an1 := new(G1).ScalarBaseMult(bigFromBase10("21888242871839275222246405745257275088548364400416034343698204186575808495616")) + + b0 := new(G2).ScalarBaseMult(bigFromBase10("0")) + b1 := new(G2).ScalarBaseMult(bigFromBase10("1")) + b2 := new(G2).ScalarBaseMult(bigFromBase10("2")) + b27 := new(G2).ScalarBaseMult(bigFromBase10("27")) + b999 := new(G2).ScalarBaseMult(bigFromBase10("999")) + bn1 := new(G2).ScalarBaseMult(bigFromBase10("21888242871839275222246405745257275088548364400416034343698204186575808495616")) + + p1 := Pair(a1, b1) + pn1 := Pair(a1, bn1) + np1 := Pair(an1, b1) + if pn1.String() != np1.String() { + t.Error("Pairing mismatch: e(a, -b) != e(-a, b)") + } + if !PairingCheck([]*G1{a1, an1}, []*G2{b1, b1}) { + t.Error("MultiAte check gave false negative!") + } + p0 := new(GT).Add(p1, pn1) + p0_2 := Pair(a1, b0) + if p0.String() != p0_2.String() { + t.Error("Pairing mismatch: e(a, b) * e(a, -b) != 1") + } + p0_3 := new(GT).ScalarMult(p1, bigFromBase10("21888242871839275222246405745257275088548364400416034343698204186575808495617")) + if p0.String() != p0_3.String() { + t.Error("Pairing mismatch: e(a, b) has wrong order") + } + p2 := Pair(a2, b1) + p2_2 := Pair(a1, b2) + p2_3 := new(GT).ScalarMult(p1, bigFromBase10("2")) + if p2.String() != p2_2.String() { + t.Error("Pairing mismatch: e(a, b * 2) != e(a * 2, b)") + } + if p2.String() != p2_3.String() { + t.Error("Pairing mismatch: e(a, b * 2) != e(a, b) ** 2") + } + if p2.String() == p1.String() { + t.Error("Pairing is degenerate!") + } + if PairingCheck([]*G1{a1, a1}, []*G2{b1, b1}) { + t.Error("MultiAte check gave false positive!") + } + p999 := Pair(a37, b27) + p999_2 := Pair(a1, b999) + if p999.String() != p999_2.String() { + t.Error("Pairing mismatch: e(a * 37, b * 27) != e(a, b * 999)") + } +} diff --git a/crypto/bn256/optate.go b/crypto/bn256/optate.go new file mode 100644 index 0000000000..68716b62b5 --- /dev/null +++ b/crypto/bn256/optate.go @@ -0,0 +1,398 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bn256 + +func lineFunctionAdd(r, p *twistPoint, q *curvePoint, r2 *gfP2, pool *bnPool) (a, b, c *gfP2, rOut *twistPoint) { + // See the mixed addition algorithm from "Faster Computation of the + // Tate Pairing", http://arxiv.org/pdf/0904.0854v3.pdf + + B := newGFp2(pool).Mul(p.x, r.t, pool) + + D := newGFp2(pool).Add(p.y, r.z) + D.Square(D, pool) + D.Sub(D, r2) + D.Sub(D, r.t) + D.Mul(D, r.t, pool) + + H := newGFp2(pool).Sub(B, r.x) + I := newGFp2(pool).Square(H, pool) + + E := newGFp2(pool).Add(I, I) + E.Add(E, E) + + J := newGFp2(pool).Mul(H, E, pool) + + L1 := newGFp2(pool).Sub(D, r.y) + L1.Sub(L1, r.y) + + V := newGFp2(pool).Mul(r.x, E, pool) + + rOut = newTwistPoint(pool) + rOut.x.Square(L1, pool) + rOut.x.Sub(rOut.x, J) + rOut.x.Sub(rOut.x, V) + rOut.x.Sub(rOut.x, V) + + rOut.z.Add(r.z, H) + rOut.z.Square(rOut.z, pool) + rOut.z.Sub(rOut.z, r.t) + rOut.z.Sub(rOut.z, I) + + t := newGFp2(pool).Sub(V, rOut.x) + t.Mul(t, L1, pool) + t2 := newGFp2(pool).Mul(r.y, J, pool) + t2.Add(t2, t2) + rOut.y.Sub(t, t2) + + rOut.t.Square(rOut.z, pool) + + t.Add(p.y, rOut.z) + t.Square(t, pool) + t.Sub(t, r2) + t.Sub(t, rOut.t) + + t2.Mul(L1, p.x, pool) + t2.Add(t2, t2) + a = newGFp2(pool) + a.Sub(t2, t) + + c = newGFp2(pool) + c.MulScalar(rOut.z, q.y) + c.Add(c, c) + + b = newGFp2(pool) + b.SetZero() + b.Sub(b, L1) + b.MulScalar(b, q.x) + b.Add(b, b) + + B.Put(pool) + D.Put(pool) + H.Put(pool) + I.Put(pool) + E.Put(pool) + J.Put(pool) + L1.Put(pool) + V.Put(pool) + t.Put(pool) + t2.Put(pool) + + return +} + +func lineFunctionDouble(r *twistPoint, q *curvePoint, pool *bnPool) (a, b, c *gfP2, rOut *twistPoint) { + // See the doubling algorithm for a=0 from "Faster Computation of the + // Tate Pairing", http://arxiv.org/pdf/0904.0854v3.pdf + + A := newGFp2(pool).Square(r.x, pool) + B := newGFp2(pool).Square(r.y, pool) + C_ := newGFp2(pool).Square(B, pool) + + D := newGFp2(pool).Add(r.x, B) + D.Square(D, pool) + D.Sub(D, A) + D.Sub(D, C_) + D.Add(D, D) + + E := newGFp2(pool).Add(A, A) + E.Add(E, A) + + G := newGFp2(pool).Square(E, pool) + + rOut = newTwistPoint(pool) + rOut.x.Sub(G, D) + rOut.x.Sub(rOut.x, D) + + rOut.z.Add(r.y, r.z) + rOut.z.Square(rOut.z, pool) + rOut.z.Sub(rOut.z, B) + rOut.z.Sub(rOut.z, r.t) + + rOut.y.Sub(D, rOut.x) + rOut.y.Mul(rOut.y, E, pool) + t := newGFp2(pool).Add(C_, C_) + t.Add(t, t) + t.Add(t, t) + rOut.y.Sub(rOut.y, t) + + rOut.t.Square(rOut.z, pool) + + t.Mul(E, r.t, pool) + t.Add(t, t) + b = newGFp2(pool) + b.SetZero() + b.Sub(b, t) + b.MulScalar(b, q.x) + + a = newGFp2(pool) + a.Add(r.x, E) + a.Square(a, pool) + a.Sub(a, A) + a.Sub(a, G) + t.Add(B, B) + t.Add(t, t) + a.Sub(a, t) + + c = newGFp2(pool) + c.Mul(rOut.z, r.t, pool) + c.Add(c, c) + c.MulScalar(c, q.y) + + A.Put(pool) + B.Put(pool) + C_.Put(pool) + D.Put(pool) + E.Put(pool) + G.Put(pool) + t.Put(pool) + + return +} + +func mulLine(ret *gfP12, a, b, c *gfP2, pool *bnPool) { + a2 := newGFp6(pool) + a2.x.SetZero() + a2.y.Set(a) + a2.z.Set(b) + a2.Mul(a2, ret.x, pool) + t3 := newGFp6(pool).MulScalar(ret.y, c, pool) + + t := newGFp2(pool) + t.Add(b, c) + t2 := newGFp6(pool) + t2.x.SetZero() + t2.y.Set(a) + t2.z.Set(t) + ret.x.Add(ret.x, ret.y) + + ret.y.Set(t3) + + ret.x.Mul(ret.x, t2, pool) + ret.x.Sub(ret.x, a2) + ret.x.Sub(ret.x, ret.y) + a2.MulTau(a2, pool) + ret.y.Add(ret.y, a2) + + a2.Put(pool) + t3.Put(pool) + t2.Put(pool) + t.Put(pool) +} + +// sixuPlus2NAF is 6u+2 in non-adjacent form. +var sixuPlus2NAF = []int8{0, 0, 0, 1, 0, 1, 0, -1, 0, 0, 1, -1, 0, 0, 1, 0, + 0, 1, 1, 0, -1, 0, 0, 1, 0, -1, 0, 0, 0, 0, 1, 1, + 1, 0, 0, -1, 0, 0, 1, 0, 0, 0, 0, 0, -1, 0, 0, 1, + 1, 0, 0, -1, 0, 0, 0, 1, 1, 0, -1, 0, 0, 1, 0, 1, 1} + +// miller implements the Miller loop for calculating the Optimal Ate pairing. +// See algorithm 1 from http://cryptojedi.org/papers/dclxvi-20100714.pdf +func miller(q *twistPoint, p *curvePoint, pool *bnPool) *gfP12 { + ret := newGFp12(pool) + ret.SetOne() + + aAffine := newTwistPoint(pool) + aAffine.Set(q) + aAffine.MakeAffine(pool) + + bAffine := newCurvePoint(pool) + bAffine.Set(p) + bAffine.MakeAffine(pool) + + minusA := newTwistPoint(pool) + minusA.Negative(aAffine, pool) + + r := newTwistPoint(pool) + r.Set(aAffine) + + r2 := newGFp2(pool) + r2.Square(aAffine.y, pool) + + for i := len(sixuPlus2NAF) - 1; i > 0; i-- { + a, b, c, newR := lineFunctionDouble(r, bAffine, pool) + if i != len(sixuPlus2NAF)-1 { + ret.Square(ret, pool) + } + + mulLine(ret, a, b, c, pool) + a.Put(pool) + b.Put(pool) + c.Put(pool) + r.Put(pool) + r = newR + + switch sixuPlus2NAF[i-1] { + case 1: + a, b, c, newR = lineFunctionAdd(r, aAffine, bAffine, r2, pool) + case -1: + a, b, c, newR = lineFunctionAdd(r, minusA, bAffine, r2, pool) + default: + continue + } + + mulLine(ret, a, b, c, pool) + a.Put(pool) + b.Put(pool) + c.Put(pool) + r.Put(pool) + r = newR + } + + // In order to calculate Q1 we have to convert q from the sextic twist + // to the full GF(p^12) group, apply the Frobenius there, and convert + // back. + // + // The twist isomorphism is (x', y') -> (xω², yω³). If we consider just + // x for a moment, then after applying the Frobenius, we have x̄ω^(2p) + // where x̄ is the conjugate of x. If we are going to apply the inverse + // isomorphism we need a value with a single coefficient of ω² so we + // rewrite this as x̄ω^(2p-2)ω². ξ⁶ = ω and, due to the construction of + // p, 2p-2 is a multiple of six. Therefore we can rewrite as + // x̄ξ^((p-1)/3)ω² and applying the inverse isomorphism eliminates the + // ω². + // + // A similar argument can be made for the y value. + + q1 := newTwistPoint(pool) + q1.x.Conjugate(aAffine.x) + q1.x.Mul(q1.x, xiToPMinus1Over3, pool) + q1.y.Conjugate(aAffine.y) + q1.y.Mul(q1.y, xiToPMinus1Over2, pool) + q1.z.SetOne() + q1.t.SetOne() + + // For Q2 we are applying the p² Frobenius. The two conjugations cancel + // out and we are left only with the factors from the isomorphism. In + // the case of x, we end up with a pure number which is why + // xiToPSquaredMinus1Over3 is ∈ GF(p). With y we get a factor of -1. We + // ignore this to end up with -Q2. + + minusQ2 := newTwistPoint(pool) + minusQ2.x.MulScalar(aAffine.x, xiToPSquaredMinus1Over3) + minusQ2.y.Set(aAffine.y) + minusQ2.z.SetOne() + minusQ2.t.SetOne() + + r2.Square(q1.y, pool) + a, b, c, newR := lineFunctionAdd(r, q1, bAffine, r2, pool) + mulLine(ret, a, b, c, pool) + a.Put(pool) + b.Put(pool) + c.Put(pool) + r.Put(pool) + r = newR + + r2.Square(minusQ2.y, pool) + a, b, c, newR = lineFunctionAdd(r, minusQ2, bAffine, r2, pool) + mulLine(ret, a, b, c, pool) + a.Put(pool) + b.Put(pool) + c.Put(pool) + r.Put(pool) + r = newR + + aAffine.Put(pool) + bAffine.Put(pool) + minusA.Put(pool) + r.Put(pool) + r2.Put(pool) + + return ret +} + +// finalExponentiation computes the (p¹²-1)/Order-th power of an element of +// GF(p¹²) to obtain an element of GT (steps 13-15 of algorithm 1 from +// http://cryptojedi.org/papers/dclxvi-20100714.pdf) +func finalExponentiation(in *gfP12, pool *bnPool) *gfP12 { + t1 := newGFp12(pool) + + // This is the p^6-Frobenius + t1.x.Negative(in.x) + t1.y.Set(in.y) + + inv := newGFp12(pool) + inv.Invert(in, pool) + t1.Mul(t1, inv, pool) + + t2 := newGFp12(pool).FrobeniusP2(t1, pool) + t1.Mul(t1, t2, pool) + + fp := newGFp12(pool).Frobenius(t1, pool) + fp2 := newGFp12(pool).FrobeniusP2(t1, pool) + fp3 := newGFp12(pool).Frobenius(fp2, pool) + + fu, fu2, fu3 := newGFp12(pool), newGFp12(pool), newGFp12(pool) + fu.Exp(t1, u, pool) + fu2.Exp(fu, u, pool) + fu3.Exp(fu2, u, pool) + + y3 := newGFp12(pool).Frobenius(fu, pool) + fu2p := newGFp12(pool).Frobenius(fu2, pool) + fu3p := newGFp12(pool).Frobenius(fu3, pool) + y2 := newGFp12(pool).FrobeniusP2(fu2, pool) + + y0 := newGFp12(pool) + y0.Mul(fp, fp2, pool) + y0.Mul(y0, fp3, pool) + + y1, y4, y5 := newGFp12(pool), newGFp12(pool), newGFp12(pool) + y1.Conjugate(t1) + y5.Conjugate(fu2) + y3.Conjugate(y3) + y4.Mul(fu, fu2p, pool) + y4.Conjugate(y4) + + y6 := newGFp12(pool) + y6.Mul(fu3, fu3p, pool) + y6.Conjugate(y6) + + t0 := newGFp12(pool) + t0.Square(y6, pool) + t0.Mul(t0, y4, pool) + t0.Mul(t0, y5, pool) + t1.Mul(y3, y5, pool) + t1.Mul(t1, t0, pool) + t0.Mul(t0, y2, pool) + t1.Square(t1, pool) + t1.Mul(t1, t0, pool) + t1.Square(t1, pool) + t0.Mul(t1, y1, pool) + t1.Mul(t1, y0, pool) + t0.Square(t0, pool) + t0.Mul(t0, t1, pool) + + inv.Put(pool) + t1.Put(pool) + t2.Put(pool) + fp.Put(pool) + fp2.Put(pool) + fp3.Put(pool) + fu.Put(pool) + fu2.Put(pool) + fu3.Put(pool) + fu2p.Put(pool) + fu3p.Put(pool) + y0.Put(pool) + y1.Put(pool) + y2.Put(pool) + y3.Put(pool) + y4.Put(pool) + y5.Put(pool) + y6.Put(pool) + + return t0 +} + +func optimalAte(a *twistPoint, b *curvePoint, pool *bnPool) *gfP12 { + e := miller(a, b, pool) + ret := finalExponentiation(e, pool) + e.Put(pool) + + if a.IsInfinity() || b.IsInfinity() { + ret.SetOne() + } + + return ret +} diff --git a/crypto/bn256/twist.go b/crypto/bn256/twist.go new file mode 100644 index 0000000000..95b966e2ed --- /dev/null +++ b/crypto/bn256/twist.go @@ -0,0 +1,249 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bn256 + +import ( + "math/big" +) + +// twistPoint implements the elliptic curve y²=x³+3/ξ over GF(p²). Points are +// kept in Jacobian form and t=z² when valid. The group G₂ is the set of +// n-torsion points of this curve over GF(p²) (where n = Order) +type twistPoint struct { + x, y, z, t *gfP2 +} + +var twistB = &gfP2{ + bigFromBase10("266929791119991161246907387137283842545076965332900288569378510910307636690"), + bigFromBase10("19485874751759354771024239261021720505790618469301721065564631296452457478373"), +} + +// twistGen is the generator of group G₂. +var twistGen = &twistPoint{ + &gfP2{ + bigFromBase10("11559732032986387107991004021392285783925812861821192530917403151452391805634"), + bigFromBase10("10857046999023057135944570762232829481370756359578518086990519993285655852781"), + }, + &gfP2{ + bigFromBase10("4082367875863433681332203403145435568316851327593401208105741076214120093531"), + bigFromBase10("8495653923123431417604973247489272438418190587263600148770280649306958101930"), + }, + &gfP2{ + bigFromBase10("0"), + bigFromBase10("1"), + }, + &gfP2{ + bigFromBase10("0"), + bigFromBase10("1"), + }, +} + +func newTwistPoint(pool *bnPool) *twistPoint { + return &twistPoint{ + newGFp2(pool), + newGFp2(pool), + newGFp2(pool), + newGFp2(pool), + } +} + +func (c *twistPoint) String() string { + return "(" + c.x.String() + ", " + c.y.String() + ", " + c.z.String() + ")" +} + +func (c *twistPoint) Put(pool *bnPool) { + c.x.Put(pool) + c.y.Put(pool) + c.z.Put(pool) + c.t.Put(pool) +} + +func (c *twistPoint) Set(a *twistPoint) { + c.x.Set(a.x) + c.y.Set(a.y) + c.z.Set(a.z) + c.t.Set(a.t) +} + +// IsOnCurve returns true iff c is on the curve where c must be in affine form. +func (c *twistPoint) IsOnCurve() bool { + pool := new(bnPool) + yy := newGFp2(pool).Square(c.y, pool) + xxx := newGFp2(pool).Square(c.x, pool) + xxx.Mul(xxx, c.x, pool) + yy.Sub(yy, xxx) + yy.Sub(yy, twistB) + yy.Minimal() + return yy.x.Sign() == 0 && yy.y.Sign() == 0 +} + +func (c *twistPoint) SetInfinity() { + c.z.SetZero() +} + +func (c *twistPoint) IsInfinity() bool { + return c.z.IsZero() +} + +func (c *twistPoint) Add(a, b *twistPoint, pool *bnPool) { + // For additional comments, see the same function in curve.go. + + if a.IsInfinity() { + c.Set(b) + return + } + if b.IsInfinity() { + c.Set(a) + return + } + + // See http://hyperelliptic.org/EFD/g1p/auto-code/shortw/jacobian-0/addition/add-2007-bl.op3 + z1z1 := newGFp2(pool).Square(a.z, pool) + z2z2 := newGFp2(pool).Square(b.z, pool) + u1 := newGFp2(pool).Mul(a.x, z2z2, pool) + u2 := newGFp2(pool).Mul(b.x, z1z1, pool) + + t := newGFp2(pool).Mul(b.z, z2z2, pool) + s1 := newGFp2(pool).Mul(a.y, t, pool) + + t.Mul(a.z, z1z1, pool) + s2 := newGFp2(pool).Mul(b.y, t, pool) + + h := newGFp2(pool).Sub(u2, u1) + xEqual := h.IsZero() + + t.Add(h, h) + i := newGFp2(pool).Square(t, pool) + j := newGFp2(pool).Mul(h, i, pool) + + t.Sub(s2, s1) + yEqual := t.IsZero() + if xEqual && yEqual { + c.Double(a, pool) + return + } + r := newGFp2(pool).Add(t, t) + + v := newGFp2(pool).Mul(u1, i, pool) + + t4 := newGFp2(pool).Square(r, pool) + t.Add(v, v) + t6 := newGFp2(pool).Sub(t4, j) + c.x.Sub(t6, t) + + t.Sub(v, c.x) // t7 + t4.Mul(s1, j, pool) // t8 + t6.Add(t4, t4) // t9 + t4.Mul(r, t, pool) // t10 + c.y.Sub(t4, t6) + + t.Add(a.z, b.z) // t11 + t4.Square(t, pool) // t12 + t.Sub(t4, z1z1) // t13 + t4.Sub(t, z2z2) // t14 + c.z.Mul(t4, h, pool) + + z1z1.Put(pool) + z2z2.Put(pool) + u1.Put(pool) + u2.Put(pool) + t.Put(pool) + s1.Put(pool) + s2.Put(pool) + h.Put(pool) + i.Put(pool) + j.Put(pool) + r.Put(pool) + v.Put(pool) + t4.Put(pool) + t6.Put(pool) +} + +func (c *twistPoint) Double(a *twistPoint, pool *bnPool) { + // See http://hyperelliptic.org/EFD/g1p/auto-code/shortw/jacobian-0/doubling/dbl-2009-l.op3 + A := newGFp2(pool).Square(a.x, pool) + B := newGFp2(pool).Square(a.y, pool) + C_ := newGFp2(pool).Square(B, pool) + + t := newGFp2(pool).Add(a.x, B) + t2 := newGFp2(pool).Square(t, pool) + t.Sub(t2, A) + t2.Sub(t, C_) + d := newGFp2(pool).Add(t2, t2) + t.Add(A, A) + e := newGFp2(pool).Add(t, A) + f := newGFp2(pool).Square(e, pool) + + t.Add(d, d) + c.x.Sub(f, t) + + t.Add(C_, C_) + t2.Add(t, t) + t.Add(t2, t2) + c.y.Sub(d, c.x) + t2.Mul(e, c.y, pool) + c.y.Sub(t2, t) + + t.Mul(a.y, a.z, pool) + c.z.Add(t, t) + + A.Put(pool) + B.Put(pool) + C_.Put(pool) + t.Put(pool) + t2.Put(pool) + d.Put(pool) + e.Put(pool) + f.Put(pool) +} + +func (c *twistPoint) Mul(a *twistPoint, scalar *big.Int, pool *bnPool) *twistPoint { + sum := newTwistPoint(pool) + sum.SetInfinity() + t := newTwistPoint(pool) + + for i := scalar.BitLen(); i >= 0; i-- { + t.Double(sum, pool) + if scalar.Bit(i) != 0 { + sum.Add(t, a, pool) + } else { + sum.Set(t) + } + } + + c.Set(sum) + sum.Put(pool) + t.Put(pool) + return c +} + +func (c *twistPoint) MakeAffine(pool *bnPool) *twistPoint { + if c.z.IsOne() { + return c + } + + zInv := newGFp2(pool).Invert(c.z, pool) + t := newGFp2(pool).Mul(c.y, zInv, pool) + zInv2 := newGFp2(pool).Square(zInv, pool) + c.y.Mul(t, zInv2, pool) + t.Mul(c.x, zInv2, pool) + c.x.Set(t) + c.z.SetOne() + c.t.SetOne() + + zInv.Put(pool) + t.Put(pool) + zInv2.Put(pool) + + return c +} + +func (c *twistPoint) Negative(a *twistPoint, pool *bnPool) { + c.x.Set(a.x) + c.y.SetZero() + c.y.Sub(c.y, a.y) + c.z.Set(a.z) + c.t.SetZero() +} diff --git a/crypto/crypto.go b/crypto/crypto.go index 37f9ca803c..8d6699baa3 100644 --- a/crypto/crypto.go +++ b/crypto/crypto.go @@ -22,12 +22,14 @@ import ( "crypto/rand" "encoding/hex" "errors" + "fmt" "io" "io/ioutil" "math/big" "os" "github.com/expanse-org/go-expanse/common" + "github.com/expanse-org/go-expanse/common/math" "github.com/expanse-org/go-expanse/crypto/sha3" "github.com/expanse-org/go-expanse/rlp" ) @@ -66,9 +68,6 @@ func Keccak512(data ...[]byte) []byte { return d.Sum(nil) } -// Deprecated: For backward compatibility as other packages depend on these -func Sha3Hash(data ...[]byte) common.Hash { return Keccak256Hash(data...) } - // Creates an ethereum address given the bytes and the nonce func CreateAddress(b common.Address, nonce uint64) common.Address { data, _ := rlp.EncodeToBytes([]interface{}{b, nonce}) @@ -76,23 +75,38 @@ func CreateAddress(b common.Address, nonce uint64) common.Address { } // ToECDSA creates a private key with the given D value. -func ToECDSA(prv []byte) *ecdsa.PrivateKey { - if len(prv) == 0 { - return nil - } +func ToECDSA(d []byte) (*ecdsa.PrivateKey, error) { + return toECDSA(d, true) +} - priv := new(ecdsa.PrivateKey) - priv.PublicKey.Curve = S256() - priv.D = new(big.Int).SetBytes(prv) - priv.PublicKey.X, priv.PublicKey.Y = priv.PublicKey.Curve.ScalarBaseMult(prv) +// ToECDSAUnsafe blidly converts a binary blob to a private key. It should almost +// never be used unless you are sure the input is valid and want to avoid hitting +// errors due to bad origin encoding (0 prefixes cut off). +func ToECDSAUnsafe(d []byte) *ecdsa.PrivateKey { + priv, _ := toECDSA(d, false) return priv } -func FromECDSA(prv *ecdsa.PrivateKey) []byte { - if prv == nil { +// toECDSA creates a private key with the given D value. The strict parameter +// controls whether the key's length should be enforced at the curve size or +// it can also accept legacy encodings (0 prefixes). +func toECDSA(d []byte, strict bool) (*ecdsa.PrivateKey, error) { + priv := new(ecdsa.PrivateKey) + priv.PublicKey.Curve = S256() + if strict && 8*len(d) != priv.Params().BitSize { + return nil, fmt.Errorf("invalid length, need %d bits", priv.Params().BitSize) + } + priv.D = new(big.Int).SetBytes(d) + priv.PublicKey.X, priv.PublicKey.Y = priv.PublicKey.Curve.ScalarBaseMult(d) + return priv, nil +} + +// FromECDSA exports a private key into a binary dump. +func FromECDSA(priv *ecdsa.PrivateKey) []byte { + if priv == nil { return nil } - return prv.D.Bytes() + return math.PaddedBigBytes(priv.D, priv.Params().BitSize/8) } func ToECDSAPub(pub []byte) *ecdsa.PublicKey { @@ -116,14 +130,10 @@ func HexToECDSA(hexkey string) (*ecdsa.PrivateKey, error) { if err != nil { return nil, errors.New("invalid hex string") } - if len(b) != 32 { - return nil, errors.New("invalid length, need 256 bits") - } - return ToECDSA(b), nil + return ToECDSA(b) } // LoadECDSA loads a secp256k1 private key from the given file. -// The key data is expected to be hex-encoded. func LoadECDSA(file string) (*ecdsa.PrivateKey, error) { buf := make([]byte, 64) fd, err := os.Open(file) @@ -139,8 +149,7 @@ func LoadECDSA(file string) (*ecdsa.PrivateKey, error) { if err != nil { return nil, err } - - return ToECDSA(key), nil + return ToECDSA(key) } // SaveECDSA saves a secp256k1 private key to the given file with diff --git a/crypto/crypto_test.go b/crypto/crypto_test.go index 95fed0be6e..5fcb6338a1 100644 --- a/crypto/crypto_test.go +++ b/crypto/crypto_test.go @@ -36,7 +36,7 @@ var testPrivHex = "289c2857d4598e37fb9647507e47a309d6133539bf21a8b9cb6df88fd5232 // These tests are sanity checks. // They should ensure that we don't e.g. use Sha3-224 instead of Sha3-256 // and that the sha3 library uses keccak-f permutation. -func TestSha3Hash(t *testing.T) { +func TestKeccak256Hash(t *testing.T) { msg := []byte("abc") exp, _ := hex.DecodeString("4e03657aea45a94fc7d47ba826c8d667c0d1e6e33a64a036ec44f58fa12d6c45") checkhash(t, "Sha3-256-array", func(in []byte) []byte { h := Keccak256Hash(in); return h[:] }, msg, exp) diff --git a/eth/api.go b/eth/api.go index a670a5d302..f3e63643a6 100644 --- a/eth/api.go +++ b/eth/api.go @@ -20,7 +20,6 @@ import ( "bytes" "compress/gzip" "context" - "errors" "fmt" "io" "io/ioutil" @@ -41,6 +40,7 @@ import ( "github.com/expanse-org/go-expanse/params" "github.com/expanse-org/go-expanse/rlp" "github.com/expanse-org/go-expanse/rpc" + "github.com/expanse-org/go-expanse/trie" ) const defaultTraceTimeout = 5 * time.Second @@ -153,6 +153,12 @@ func (api *PrivateMinerAPI) Start(threads *int) error { } // Start the miner and return if !api.e.IsMining() { + // Propagate the initial price point to the transaction pool + api.e.lock.RLock() + price := api.e.gasPrice + api.e.lock.RUnlock() + + api.e.txPool.SetGasPrice(price) return api.e.StartMining(true) } return nil @@ -180,7 +186,11 @@ func (api *PrivateMinerAPI) SetExtra(extra string) (bool, error) { // SetGasPrice sets the minimum accepted gas price for the miner. func (api *PrivateMinerAPI) SetGasPrice(gasPrice hexutil.Big) bool { - api.e.Miner().SetGasPrice((*big.Int)(&gasPrice)) + api.e.lock.Lock() + api.e.gasPrice = (*big.Int)(&gasPrice) + api.e.lock.Unlock() + + api.e.txPool.SetGasPrice((*big.Int)(&gasPrice)) return true } @@ -526,59 +536,67 @@ func (api *PrivateDebugAPI) TraceTransaction(ctx context.Context, txHash common. if tx == nil { return nil, fmt.Errorf("transaction %x not found", txHash) } - block := api.eth.BlockChain().GetBlockByHash(blockHash) - if block == nil { - return nil, fmt.Errorf("block %x not found", blockHash) - } - // Create the state database to mutate and eventually trace - parent := api.eth.BlockChain().GetBlock(block.ParentHash(), block.NumberU64()-1) - if parent == nil { - return nil, fmt.Errorf("block parent %x not found", block.ParentHash()) - } - stateDb, err := api.eth.BlockChain().StateAt(parent.Root()) + msg, context, statedb, err := api.computeTxEnv(blockHash, int(txIndex)) if err != nil { return nil, err } - signer := types.MakeSigner(api.config, block.Number()) - // Mutate the state and trace the selected transaction - for idx, tx := range block.Transactions() { - // Assemble the transaction call message - msg, err := tx.AsMessage(signer) - if err != nil { - return nil, fmt.Errorf("sender retrieval failed: %v", err) - } - context := core.NewEVMContext(msg, block.Header(), api.eth.BlockChain(), nil) - - // Mutate the state if we haven't reached the tracing transaction yet - if uint64(idx) < txIndex { - vmenv := vm.NewEVM(context, stateDb, api.config, vm.Config{}) - _, _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(tx.Gas())) - if err != nil { - return nil, fmt.Errorf("mutation failed: %v", err) - } - stateDb.DeleteSuicides() - continue - } - - vmenv := vm.NewEVM(context, stateDb, api.config, vm.Config{Debug: true, Tracer: tracer}) - ret, gas, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(tx.Gas())) - if err != nil { - return nil, fmt.Errorf("tracing failed: %v", err) - } - - switch tracer := tracer.(type) { - case *vm.StructLogger: - return ðapi.ExecutionResult{ - Gas: gas, - ReturnValue: fmt.Sprintf("%x", ret), - StructLogs: ethapi.FormatLogs(tracer.StructLogs()), - }, nil - case *ethapi.JavascriptTracer: - return tracer.GetResult() - } + // Run the transaction with tracing enabled. + vmenv := vm.NewEVM(context, statedb, api.config, vm.Config{Debug: true, Tracer: tracer}) + ret, gas, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(tx.Gas())) + if err != nil { + return nil, fmt.Errorf("tracing failed: %v", err) } - return nil, errors.New("database inconsistency") + switch tracer := tracer.(type) { + case *vm.StructLogger: + return ðapi.ExecutionResult{ + Gas: gas, + ReturnValue: fmt.Sprintf("%x", ret), + StructLogs: ethapi.FormatLogs(tracer.StructLogs()), + }, nil + case *ethapi.JavascriptTracer: + return tracer.GetResult() + default: + panic(fmt.Sprintf("bad tracer type %T", tracer)) + } +} + +// computeTxEnv returns the execution environment of a certain transaction. +func (api *PrivateDebugAPI) computeTxEnv(blockHash common.Hash, txIndex int) (core.Message, vm.Context, *state.StateDB, error) { + // Create the parent state. + block := api.eth.BlockChain().GetBlockByHash(blockHash) + if block == nil { + return nil, vm.Context{}, nil, fmt.Errorf("block %x not found", blockHash) + } + parent := api.eth.BlockChain().GetBlock(block.ParentHash(), block.NumberU64()-1) + if parent == nil { + return nil, vm.Context{}, nil, fmt.Errorf("block parent %x not found", block.ParentHash()) + } + statedb, err := api.eth.BlockChain().StateAt(parent.Root()) + if err != nil { + return nil, vm.Context{}, nil, err + } + txs := block.Transactions() + + // Recompute transactions up to the target index. + signer := types.MakeSigner(api.config, block.Number()) + for idx, tx := range txs { + // Assemble the transaction call message + msg, _ := tx.AsMessage(signer) + context := core.NewEVMContext(msg, block.Header(), api.eth.BlockChain(), nil) + if idx == txIndex { + return msg, context, statedb, nil + } + + vmenv := vm.NewEVM(context, statedb, api.config, vm.Config{}) + gp := new(core.GasPool).AddGas(tx.Gas()) + _, _, err := core.ApplyMessage(vmenv, msg, gp) + if err != nil { + return nil, vm.Context{}, nil, fmt.Errorf("tx %x failed: %v", tx.Hash(), err) + } + statedb.DeleteSuicides() + } + return nil, vm.Context{}, nil, fmt.Errorf("tx index %d out of range for block %x", txIndex, blockHash) } // Preimage is a debug API function that returns the preimage for a sha3 hash, if known. @@ -592,3 +610,48 @@ func (api *PrivateDebugAPI) Preimage(ctx context.Context, hash common.Hash) (hex func (api *PrivateDebugAPI) GetBadBlocks(ctx context.Context) ([]core.BadBlockArgs, error) { return api.eth.BlockChain().BadBlocks() } + +// StorageRangeResult is the result of a debug_storageRangeAt API call. +type StorageRangeResult struct { + Storage storageMap `json:"storage"` + NextKey *common.Hash `json:"nextKey"` // nil if Storage includes the last key in the trie. +} + +type storageMap map[common.Hash]storageEntry + +type storageEntry struct { + Key *common.Hash `json:"key"` + Value common.Hash `json:"value"` +} + +// StorageRangeAt returns the storage at the given block height and transaction index. +func (api *PrivateDebugAPI) StorageRangeAt(ctx context.Context, blockHash common.Hash, txIndex int, contractAddress common.Address, keyStart hexutil.Bytes, maxResult int) (StorageRangeResult, error) { + _, _, statedb, err := api.computeTxEnv(blockHash, txIndex) + if err != nil { + return StorageRangeResult{}, err + } + st := statedb.StorageTrie(contractAddress) + if st == nil { + return StorageRangeResult{}, fmt.Errorf("account %x doesn't exist", contractAddress) + } + return storageRangeAt(st, keyStart, maxResult), nil +} + +func storageRangeAt(st *trie.SecureTrie, start []byte, maxResult int) StorageRangeResult { + it := trie.NewIterator(st.NodeIterator(start)) + result := StorageRangeResult{Storage: storageMap{}} + for i := 0; i < maxResult && it.Next(); i++ { + e := storageEntry{Value: common.BytesToHash(it.Value)} + if preimage := st.GetKey(it.Key); preimage != nil { + preimage := common.BytesToHash(preimage) + e.Key = &preimage + } + result.Storage[common.BytesToHash(it.Key)] = e + } + // Add the 'next key' so clients can continue downloading. + if it.Next() { + next := common.BytesToHash(it.Key) + result.NextKey = &next + } + return result +} diff --git a/eth/api_test.go b/eth/api_test.go new file mode 100644 index 0000000000..bcebd0c177 --- /dev/null +++ b/eth/api_test.go @@ -0,0 +1,88 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package eth + +import ( + "reflect" + "testing" + + "github.com/davecgh/go-spew/spew" + "github.com/expanse-org/go-expanse/common" + "github.com/expanse-org/go-expanse/core/state" + "github.com/expanse-org/go-expanse/ethdb" +) + +var dumper = spew.ConfigState{Indent: " "} + +func TestStorageRangeAt(t *testing.T) { + // Create a state where account 0x010000... has a few storage entries. + var ( + db, _ = ethdb.NewMemDatabase() + state, _ = state.New(common.Hash{}, db) + addr = common.Address{0x01} + keys = []common.Hash{ // hashes of Keys of storage + common.HexToHash("340dd630ad21bf010b4e676dbfa9ba9a02175262d1fa356232cfde6cb5b47ef2"), + common.HexToHash("426fcb404ab2d5d8e61a3d918108006bbb0a9be65e92235bb10eefbdb6dcd053"), + common.HexToHash("48078cfed56339ea54962e72c37c7f588fc4f8e5bc173827ba75cb10a63a96a5"), + common.HexToHash("5723d2c3a83af9b735e3b7f21531e5623d183a9095a56604ead41f3582fdfb75"), + } + storage = storageMap{ + keys[0]: {Key: &common.Hash{0x02}, Value: common.Hash{0x01}}, + keys[1]: {Key: &common.Hash{0x04}, Value: common.Hash{0x02}}, + keys[2]: {Key: &common.Hash{0x01}, Value: common.Hash{0x03}}, + keys[3]: {Key: &common.Hash{0x03}, Value: common.Hash{0x04}}, + } + ) + for _, entry := range storage { + state.SetState(addr, *entry.Key, entry.Value) + } + + // Check a few combinations of limit and start/end. + tests := []struct { + start []byte + limit int + want StorageRangeResult + }{ + { + start: []byte{}, limit: 0, + want: StorageRangeResult{storageMap{}, &keys[0]}, + }, + { + start: []byte{}, limit: 100, + want: StorageRangeResult{storage, nil}, + }, + { + start: []byte{}, limit: 2, + want: StorageRangeResult{storageMap{keys[0]: storage[keys[0]], keys[1]: storage[keys[1]]}, &keys[2]}, + }, + { + start: []byte{0x00}, limit: 4, + want: StorageRangeResult{storage, nil}, + }, + { + start: []byte{0x40}, limit: 2, + want: StorageRangeResult{storageMap{keys[1]: storage[keys[1]], keys[2]: storage[keys[2]]}, &keys[3]}, + }, + } + for _, test := range tests { + result := storageRangeAt(state.StorageTrie(addr), test.start, test.limit) + if !reflect.DeepEqual(result, test.want) { + t.Fatalf("wrong result for range 0x%x.., limit %d:\ngot %s\nwant %s", + test.start, test.limit, dumper.Sdump(result), dumper.Sdump(&test.want)) + } + } +} diff --git a/eth/backend.go b/eth/backend.go index eacd44e9c4..398a803cc3 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -20,6 +20,7 @@ package eth import ( "errors" "fmt" + "math/big" "runtime" "sync" "sync/atomic" @@ -75,13 +76,14 @@ type Ethereum struct { ApiBackend *EthApiBackend - miner *miner.Miner - Mining bool - MinerThreads int - etherbase common.Address + miner *miner.Miner + gasPrice *big.Int + etherbase common.Address - netVersionId int + networkId uint64 netRPCService *ethapi.PublicNetAPI + + lock sync.RWMutex // Protects the variadic fields (e.g. gas price and etherbase) } func (s *Ethereum) AddLesServer(ls LesServer) { @@ -118,9 +120,9 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { engine: CreateConsensusEngine(ctx, config, chainConfig, chainDb), shutdownChan: make(chan bool), stopDbUpgrade: stopDbUpgrade, - netVersionId: config.NetworkId, + networkId: config.NetworkId, + gasPrice: config.GasPrice, etherbase: config.Etherbase, - MinerThreads: config.MinerThreads, } if err := addMipmapBloomBins(chainDb); err != nil { @@ -148,7 +150,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { core.WriteChainConfig(chainDb, genesisHash, chainConfig) } - newPool := core.NewTxPool(eth.chainConfig, eth.EventMux(), eth.blockchain.State, eth.blockchain.GasLimit) + newPool := core.NewTxPool(config.TxPool, eth.chainConfig, eth.EventMux(), eth.blockchain.State, eth.blockchain.GasLimit) eth.txPool = newPool maxPeers := config.MaxPeers @@ -167,7 +169,6 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { } eth.miner = miner.New(eth, eth.chainConfig, eth.EventMux(), eth.engine) - eth.miner.SetGasPrice(config.GasPrice) eth.miner.SetExtra(makeExtraData(config.ExtraData)) eth.ApiBackend = &EthApiBackend{eth, nil} @@ -313,8 +314,12 @@ func (s *Ethereum) ResetWithGenesisBlock(gb *types.Block) { } func (s *Ethereum) Etherbase() (eb common.Address, err error) { - if s.etherbase != (common.Address{}) { - return s.etherbase, nil + s.lock.RLock() + etherbase := s.etherbase + s.lock.RUnlock() + + if etherbase != (common.Address{}) { + return etherbase, nil } if wallets := s.AccountManager().Wallets(); len(wallets) > 0 { if accounts := wallets[0].Accounts(); len(accounts) > 0 { @@ -326,7 +331,10 @@ func (s *Ethereum) Etherbase() (eb common.Address, err error) { // set in js console via admin interface or wrapper from cli flags func (self *Ethereum) SetEtherbase(etherbase common.Address) { + self.lock.Lock() self.etherbase = etherbase + self.lock.Unlock() + self.miner.SetEtherbase(etherbase) } @@ -367,7 +375,7 @@ func (s *Ethereum) Engine() consensus.Engine { return s.engine } func (s *Ethereum) ChainDb() ethdb.Database { return s.chainDb } func (s *Ethereum) IsListening() bool { return true } // Always listening func (s *Ethereum) EthVersion() int { return int(s.protocolManager.SubProtocols[0].Version) } -func (s *Ethereum) NetVersion() int { return s.netVersionId } +func (s *Ethereum) NetVersion() uint64 { return s.networkId } func (s *Ethereum) Downloader() *downloader.Downloader { return s.protocolManager.downloader } // Protocols implements node.Service, returning all the currently configured diff --git a/eth/bind.go b/eth/bind.go index 5b11413e37..aaf7b41549 100644 --- a/eth/bind.go +++ b/eth/bind.go @@ -48,7 +48,7 @@ func NewContractBackend(apiBackend ethapi.Backend) *ContractBackend { return &ContractBackend{ eapi: ethapi.NewPublicEthereumAPI(apiBackend), bcapi: ethapi.NewPublicBlockChainAPI(apiBackend), - txapi: ethapi.NewPublicTransactionPoolAPI(apiBackend), + txapi: ethapi.NewPublicTransactionPoolAPI(apiBackend, new(ethapi.AddrLocker)), } } diff --git a/eth/config.go b/eth/config.go index 177f78537c..66fab095ab 100644 --- a/eth/config.go +++ b/eth/config.go @@ -42,8 +42,9 @@ var DefaultConfig = Config{ NetworkId: 1, LightPeers: 20, DatabaseCache: 128, - GasPrice: big.NewInt(20 * params.Shannon), + GasPrice: big.NewInt(18 * params.Shannon), + TxPool: core.DefaultTxPoolConfig, GPO: gasprice.Config{ Blocks: 10, Percentile: 50, @@ -72,7 +73,7 @@ type Config struct { Genesis *core.Genesis `toml:",omitempty"` // Protocol options - NetworkId int // Network ID to use for selecting peers to connect to + NetworkId uint64 // Network ID to use for selecting peers to connect to SyncMode downloader.SyncMode // Light client options @@ -99,6 +100,9 @@ type Config struct { EthashDatasetsInMem int EthashDatasetsOnDisk int + // Transaction pool options + TxPool core.TxPoolConfig + // Gas Price Oracle options GPO gasprice.Config diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 580ccab63a..4f168ab23e 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -1491,6 +1491,10 @@ func (d *Downloader) qosTuner() { func (d *Downloader) qosReduceConfidence() { // If we have a single peer, confidence is always 1 peers := uint64(d.peers.Len()) + if peers == 0 { + // Ensure peer connectivity races don't catch us off guard + return + } if peers == 1 { atomic.StoreUint64(&d.rttConfidence, 1000000) return diff --git a/eth/gen_config.go b/eth/gen_config.go index d9319833a6..07291f2126 100644 --- a/eth/gen_config.go +++ b/eth/gen_config.go @@ -15,7 +15,7 @@ import ( func (c Config) MarshalTOML() (interface{}, error) { type Config struct { Genesis *core.Genesis `toml:",omitempty"` - NetworkId int + NetworkId uint64 SyncMode downloader.SyncMode LightServ int `toml:",omitempty"` LightPeers int `toml:",omitempty"` @@ -33,6 +33,7 @@ func (c Config) MarshalTOML() (interface{}, error) { EthashDatasetDir string EthashDatasetsInMem int EthashDatasetsOnDisk int + TxPool core.TxPoolConfig GPO gasprice.Config EnablePreimageRecording bool DocRoot string `toml:"-"` @@ -60,6 +61,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.EthashDatasetDir = c.EthashDatasetDir enc.EthashDatasetsInMem = c.EthashDatasetsInMem enc.EthashDatasetsOnDisk = c.EthashDatasetsOnDisk + enc.TxPool = c.TxPool enc.GPO = c.GPO enc.EnablePreimageRecording = c.EnablePreimageRecording enc.DocRoot = c.DocRoot @@ -72,7 +74,7 @@ func (c Config) MarshalTOML() (interface{}, error) { func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { type Config struct { Genesis *core.Genesis `toml:",omitempty"` - NetworkId *int + NetworkId *uint64 SyncMode *downloader.SyncMode LightServ *int `toml:",omitempty"` LightPeers *int `toml:",omitempty"` @@ -90,6 +92,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { EthashDatasetDir *string EthashDatasetsInMem *int EthashDatasetsOnDisk *int + TxPool *core.TxPoolConfig GPO *gasprice.Config EnablePreimageRecording *bool DocRoot *string `toml:"-"` @@ -158,6 +161,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.EthashDatasetsOnDisk != nil { c.EthashDatasetsOnDisk = *dec.EthashDatasetsOnDisk } + if dec.TxPool != nil { + c.TxPool = *dec.TxPool + } if dec.GPO != nil { c.GPO = *dec.GPO } diff --git a/eth/handler.go b/eth/handler.go index d4fa4eee75..fb8bc0b08d 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -60,7 +60,7 @@ func errResp(code errCode, format string, v ...interface{}) error { } type ProtocolManager struct { - networkId int + networkId uint64 fastSync uint32 // Flag whether fast sync is enabled (gets disabled if we already have blocks) acceptTxs uint32 // Flag whether we're considered synchronised (enables transaction processing) @@ -96,7 +96,7 @@ type ProtocolManager struct { // NewProtocolManager returns a new ethereum sub protocol manager. The Ethereum sub protocol manages peers capable // with the ethereum network. -func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, networkId int, maxPeers int, mux *event.TypeMux, txpool txPool, engine consensus.Engine, blockchain *core.BlockChain, chaindb ethdb.Database) (*ProtocolManager, error) { +func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, networkId uint64, maxPeers int, mux *event.TypeMux, txpool txPool, engine consensus.Engine, blockchain *core.BlockChain, chaindb ethdb.Database) (*ProtocolManager, error) { // Create the protocol manager with the base fields manager := &ProtocolManager{ networkId: networkId, @@ -171,6 +171,11 @@ func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, ne return blockchain.CurrentBlock().NumberU64() } inserter := func(blocks types.Blocks) (int, error) { + // If fast sync is running, deny importing weird blocks + if atomic.LoadUint32(&manager.fastSync) == 1 { + log.Warn("Discarded bad propagated block", "number", blocks[0].Number(), "hash", blocks[0].Hash()) + return 0, nil + } atomic.StoreUint32(&manager.acceptTxs, 1) // Mark initial sync done on any fetcher import return manager.blockchain.InsertChain(blocks) } @@ -733,7 +738,7 @@ func (self *ProtocolManager) txBroadcastLoop() { // EthNodeInfo represents a short summary of the Ethereum sub-protocol metadata known // about the host peer. type EthNodeInfo struct { - Network int `json:"network"` // Ethereum network ID (1=Frontier, 2=Morden, Ropsten=3) + Network uint64 `json:"network"` // Ethereum network ID (1=Frontier, 2=Morden, Ropsten=3) Difficulty *big.Int `json:"difficulty"` // Total difficulty of the host's blockchain Genesis common.Hash `json:"genesis"` // SHA3 hash of the host's genesis block Head common.Hash `json:"head"` // SHA3 hash of the host's best owned block diff --git a/eth/helper_test.go b/eth/helper_test.go index 4637334ff4..fa49aa93bd 100644 --- a/eth/helper_test.go +++ b/eth/helper_test.go @@ -173,7 +173,7 @@ func newTestPeer(name string, version int, pm *ProtocolManager, shake bool) (*te func (p *testPeer) handshake(t *testing.T, td *big.Int, head common.Hash, genesis common.Hash) { msg := &statusData{ ProtocolVersion: uint32(p.version), - NetworkId: uint32(DefaultConfig.NetworkId), + NetworkId: DefaultConfig.NetworkId, TD: td, CurrentBlock: head, GenesisBlock: genesis, diff --git a/eth/peer.go b/eth/peer.go index 6ab6e33ce0..d44ecc8a30 100644 --- a/eth/peer.go +++ b/eth/peer.go @@ -230,7 +230,7 @@ func (p *peer) RequestReceipts(hashes []common.Hash) error { // Handshake executes the eth protocol handshake, negotiating version number, // network IDs, difficulties, head and genesis blocks. -func (p *peer) Handshake(network int, td *big.Int, head common.Hash, genesis common.Hash) error { +func (p *peer) Handshake(network uint64, td *big.Int, head common.Hash, genesis common.Hash) error { // Send out own handshake in a new thread errc := make(chan error, 2) var status statusData // safe to read after two values have been received from errc @@ -238,7 +238,7 @@ func (p *peer) Handshake(network int, td *big.Int, head common.Hash, genesis com go func() { errc <- p2p.Send(p.rw, StatusMsg, &statusData{ ProtocolVersion: uint32(p.version), - NetworkId: uint32(network), + NetworkId: network, TD: td, CurrentBlock: head, GenesisBlock: genesis, @@ -263,7 +263,7 @@ func (p *peer) Handshake(network int, td *big.Int, head common.Hash, genesis com return nil } -func (p *peer) readStatus(network int, status *statusData, genesis common.Hash) (err error) { +func (p *peer) readStatus(network uint64, status *statusData, genesis common.Hash) (err error) { msg, err := p.rw.ReadMsg() if err != nil { return err @@ -281,7 +281,7 @@ func (p *peer) readStatus(network int, status *statusData, genesis common.Hash) if status.GenesisBlock != genesis { return errResp(ErrGenesisBlockMismatch, "%x (!= %x)", status.GenesisBlock[:8], genesis[:8]) } - if int(status.NetworkId) != network { + if status.NetworkId != network { return errResp(ErrNetworkIdMismatch, "%d (!= %d)", status.NetworkId, network) } if int(status.ProtocolVersion) != p.version { diff --git a/eth/protocol.go b/eth/protocol.go index 1c0861566a..96c08dffea 100644 --- a/eth/protocol.go +++ b/eth/protocol.go @@ -105,7 +105,7 @@ type txPool interface { // statusData is the network packet for the status message. type statusData struct { ProtocolVersion uint32 - NetworkId uint32 + NetworkId uint64 TD *big.Int CurrentBlock common.Hash GenesisBlock common.Hash diff --git a/eth/protocol_test.go b/eth/protocol_test.go index a03fcc0bcd..90ba44da69 100644 --- a/eth/protocol_test.go +++ b/eth/protocol_test.go @@ -55,7 +55,7 @@ func testStatusMsgErrors(t *testing.T, protocol int) { wantError: errResp(ErrNoStatusMsg, "first msg has code 2 (!= 0)"), }, { - code: StatusMsg, data: statusData{10, uint32(DefaultConfig.NetworkId), td, currentBlock, genesis}, + code: StatusMsg, data: statusData{10, DefaultConfig.NetworkId, td, currentBlock, genesis}, wantError: errResp(ErrProtocolVersionMismatch, "10 (!= %d)", protocol), }, { @@ -63,7 +63,7 @@ func testStatusMsgErrors(t *testing.T, protocol int) { wantError: errResp(ErrNetworkIdMismatch, "999 (!= 1)"), }, { - code: StatusMsg, data: statusData{uint32(protocol), uint32(DefaultConfig.NetworkId), td, currentBlock, common.Hash{3}}, + code: StatusMsg, data: statusData{uint32(protocol), DefaultConfig.NetworkId, td, currentBlock, common.Hash{3}}, wantError: errResp(ErrGenesisBlockMismatch, "0300000000000000 (!= %x)", genesis[:8]), }, } diff --git a/eth/sync.go b/eth/sync.go index b876fb783f..014c3a90cd 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -183,6 +183,7 @@ func (pm *ProtocolManager) synchronise(peer *peer) { // The only scenario where this can happen is if the user manually (or via a // bad block) rolled back a fast sync node below the sync point. In this case // however it's safe to reenable fast sync. + atomic.StoreUint32(&pm.fastSync, 1) mode = downloader.FastSync } if err := pm.downloader.Synchronise(peer.id, pHead, pTd, mode); err != nil { diff --git a/ethstats/ethstats.go b/ethstats/ethstats.go index bbb65b4913..1732a5a04e 100644 --- a/ethstats/ethstats.go +++ b/ethstats/ethstats.go @@ -18,6 +18,7 @@ package ethstats import ( + "context" "encoding/json" "errors" "fmt" @@ -30,6 +31,7 @@ import ( "time" "github.com/expanse-org/go-expanse/common" + "github.com/expanse-org/go-expanse/common/mclock" "github.com/expanse-org/go-expanse/consensus" "github.com/expanse-org/go-expanse/core" "github.com/expanse-org/go-expanse/core/types" @@ -118,7 +120,7 @@ func (s *Service) Stop() error { // loop keeps trying to connect to the netstats server, reporting chain events // until termination. func (s *Service) loop() { - // Subscribe tso chain events to execute updates on + // Subscribe to chain events to execute updates on var emux *event.TypeMux if s.eth != nil { emux = s.eth.EventMux() @@ -131,6 +133,46 @@ func (s *Service) loop() { txSub := emux.Subscribe(core.TxPreEvent{}) defer txSub.Unsubscribe() + // Start a goroutine that exhausts the subsciptions to avoid events piling up + var ( + quitCh = make(chan struct{}) + headCh = make(chan *types.Block, 1) + txCh = make(chan struct{}, 1) + ) + go func() { + var lastTx mclock.AbsTime + + for { + select { + // Notify of chain head events, but drop if too frequent + case head, ok := <-headSub.Chan(): + if !ok { // node stopped + close(quitCh) + return + } + select { + case headCh <- head.Data.(core.ChainHeadEvent).Block: + default: + } + + // Notify of new transaction events, but drop if too frequent + case _, ok := <-txSub.Chan(): + if !ok { // node stopped + close(quitCh) + return + } + if time.Duration(mclock.Now()-lastTx) < time.Second { + continue + } + lastTx = mclock.Now() + + select { + case txCh <- struct{}{}: + default: + } + } + } + }() // Loop reporting until termination for { // Resolve the URL, defaulting to TLS, but falling back to none too @@ -150,7 +192,7 @@ func (s *Service) loop() { if conf, err = websocket.NewConfig(url, "http://localhost/"); err != nil { continue } - conf.Dialer = &net.Dialer{Timeout: 3 * time.Second} + conf.Dialer = &net.Dialer{Timeout: 5 * time.Second} if conn, err = websocket.DialConfig(conf); err == nil { break } @@ -180,6 +222,10 @@ func (s *Service) loop() { for err == nil { select { + case <-quitCh: + conn.Close() + return + case <-fullReport.C: if err = s.report(conn); err != nil { log.Warn("Full stats report failed", "err", err) @@ -188,30 +234,14 @@ func (s *Service) loop() { if err = s.reportHistory(conn, list); err != nil { log.Warn("Requested history report failed", "err", err) } - case head, ok := <-headSub.Chan(): - if !ok { // node stopped - conn.Close() - return - } - if err = s.reportBlock(conn, head.Data.(core.ChainHeadEvent).Block); err != nil { + case head := <-headCh: + if err = s.reportBlock(conn, head); err != nil { log.Warn("Block stats report failed", "err", err) } if err = s.reportPending(conn); err != nil { log.Warn("Post-block transaction stats report failed", "err", err) } - case _, ok := <-txSub.Chan(): - if !ok { // node stopped - conn.Close() - return - } - // Exhaust events to avoid reporting too frequently - for exhausted := false; !exhausted; { - select { - case <-headSub.Chan(): - default: - exhausted = true - } - } + case <-txCh: if err = s.reportPending(conn); err != nil { log.Warn("Transaction stats report failed", "err", err) } @@ -323,10 +353,10 @@ func (s *Service) login(conn *websocket.Conn) error { var network, protocol string if info := infos.Protocols["exp"]; info != nil { - network = strconv.Itoa(info.(*eth.EthNodeInfo).Network) - protocol = fmt.Sprintf("eth/%d", eth.ProtocolVersions[0]) + network = fmt.Sprintf("%d", info.(*eth.EthNodeInfo).Network) + protocol = fmt.Sprintf("exp/%d", eth.ProtocolVersions[0]) } else { - network = strconv.Itoa(infos.Protocols["les"].(*eth.EthNodeInfo).Network) + network = fmt.Sprintf("%d", infos.Protocols["les"].(*eth.EthNodeInfo).Network) protocol = fmt.Sprintf("les/%d", les.ProtocolVersions[0]) } auth := &authMsg{ @@ -397,7 +427,7 @@ func (s *Service) reportLatency(conn *websocket.Conn) error { select { case <-s.pongCh: // Pong delivered, report the latency - case <-time.After(3 * time.Second): + case <-time.After(5 * time.Second): // Ping timeout, abort return errors.New("ping timed out") } @@ -426,21 +456,15 @@ type blockStats struct { GasLimit *big.Int `json:"gasLimit"` Diff string `json:"difficulty"` TotalDiff string `json:"totalDifficulty"` - Txs txStats `json:"transactions"` + Txs []txStats `json:"transactions"` TxHash common.Hash `json:"transactionsRoot"` Root common.Hash `json:"stateRoot"` Uncles uncleStats `json:"uncles"` } -// txStats is a custom wrapper around a transaction array to force serializing -// empty arrays instead of returning null for them. -type txStats []*types.Transaction - -func (s txStats) MarshalJSON() ([]byte, error) { - if txs := ([]*types.Transaction)(s); len(txs) > 0 { - return json.Marshal(txs) - } - return []byte("[]"), nil +// txStats is the information to report about individual transactions. +type txStats struct { + Hash common.Hash `json:"hash"` } // uncleStats is a custom wrapper around an uncle array to force serializing @@ -479,7 +503,7 @@ func (s *Service) assembleBlockStats(block *types.Block) *blockStats { var ( header *types.Header td *big.Int - txs []*types.Transaction + txs []txStats uncles []*types.Header ) if s.eth != nil { @@ -490,7 +514,10 @@ func (s *Service) assembleBlockStats(block *types.Block) *blockStats { header = block.Header() td = s.eth.BlockChain().GetTd(header.Hash(), header.Number.Uint64()) - txs = block.Transactions() + txs = make([]txStats, len(block.Transactions())) + for i, tx := range block.Transactions() { + txs[i].Hash = tx.Hash() + } uncles = block.Uncles() } else { // Light nodes would need on-demand lookups for transactions/uncles, skip @@ -500,6 +527,7 @@ func (s *Service) assembleBlockStats(block *types.Block) *blockStats { header = s.les.BlockChain().CurrentHeader() } td = s.les.BlockChain().GetTd(header.Hash(), header.Number.Uint64()) + txs = []txStats{} } // Assemble and return the block stats author, _ := s.engine.Author(header) @@ -639,7 +667,8 @@ func (s *Service) reportStats(conn *websocket.Conn) error { sync := s.eth.Downloader().Progress() syncing = s.eth.BlockChain().CurrentHeader().Number.Uint64() >= sync.HighestBlock - gasprice = int(s.eth.Miner().GasPrice().Uint64()) + price, _ := s.eth.ApiBackend.SuggestPrice(context.Background()) + gasprice = int(price.Uint64()) } else { sync := s.les.Downloader().Progress() syncing = s.les.BlockChain().CurrentHeader().Number.Uint64() >= sync.HighestBlock diff --git a/internal/ethapi/addrlock.go b/internal/ethapi/addrlock.go new file mode 100644 index 0000000000..70e50d157a --- /dev/null +++ b/internal/ethapi/addrlock.go @@ -0,0 +1,53 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package ethapi + +import ( + "sync" + + "github.com/expanse-org/go-expanse/common" +) + +type AddrLocker struct { + mu sync.Mutex + locks map[common.Address]*sync.Mutex +} + +// lock returns the lock of the given address. +func (l *AddrLocker) lock(address common.Address) *sync.Mutex { + l.mu.Lock() + defer l.mu.Unlock() + if l.locks == nil { + l.locks = make(map[common.Address]*sync.Mutex) + } + if _, ok := l.locks[address]; !ok { + l.locks[address] = new(sync.Mutex) + } + return l.locks[address] +} + +// LockAddr locks an account's mutex. This is used to prevent another tx getting the +// same nonce until the lock is released. The mutex prevents the (an identical nonce) from +// being read again during the time that the first transaction is being signed. +func (l *AddrLocker) LockAddr(address common.Address) { + l.lock(address).Lock() +} + +// UnlockAddr unlocks the mutex of the given account. +func (l *AddrLocker) UnlockAddr(address common.Address) { + l.lock(address).Unlock() +} diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index e0324e9c53..f6108e9f2a 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -19,7 +19,6 @@ package ethapi import ( "bytes" "context" - "encoding/hex" "errors" "fmt" "math/big" @@ -118,16 +117,16 @@ func (s *PublicTxPoolAPI) Content() map[string]map[string]map[string]*RPCTransac // Flatten the pending transactions for account, txs := range pending { dump := make(map[string]*RPCTransaction) - for nonce, tx := range txs { - dump[fmt.Sprintf("%d", nonce)] = newRPCPendingTransaction(tx) + for _, tx := range txs { + dump[fmt.Sprintf("%d", tx.Nonce())] = newRPCPendingTransaction(tx) } content["pending"][account.Hex()] = dump } // Flatten the queued transactions for account, txs := range queue { dump := make(map[string]*RPCTransaction) - for nonce, tx := range txs { - dump[fmt.Sprintf("%d", nonce)] = newRPCPendingTransaction(tx) + for _, tx := range txs { + dump[fmt.Sprintf("%d", tx.Nonce())] = newRPCPendingTransaction(tx) } content["queued"][account.Hex()] = dump } @@ -162,16 +161,16 @@ func (s *PublicTxPoolAPI) Inspect() map[string]map[string]map[string]string { // Flatten the pending transactions for account, txs := range pending { dump := make(map[string]string) - for nonce, tx := range txs { - dump[fmt.Sprintf("%d", nonce)] = format(tx) + for _, tx := range txs { + dump[fmt.Sprintf("%d", tx.Nonce())] = format(tx) } content["pending"][account.Hex()] = dump } // Flatten the queued transactions for account, txs := range queue { dump := make(map[string]string) - for nonce, tx := range txs { - dump[fmt.Sprintf("%d", nonce)] = format(tx) + for _, tx := range txs { + dump[fmt.Sprintf("%d", tx.Nonce())] = format(tx) } content["queued"][account.Hex()] = dump } @@ -191,7 +190,7 @@ func NewPublicAccountAPI(am *accounts.Manager) *PublicAccountAPI { // Accounts returns the collection of accounts this node manages func (s *PublicAccountAPI) Accounts() []common.Address { - var addresses []common.Address + addresses := make([]common.Address, 0) // return [] instead of nil if empty for _, wallet := range s.am.Wallets() { for _, account := range wallet.Accounts() { addresses = append(addresses, account.Address) @@ -204,21 +203,23 @@ func (s *PublicAccountAPI) Accounts() []common.Address { // It offers methods to create, (un)lock en list accounts. Some methods accept // passwords and are therefore considered private by default. type PrivateAccountAPI struct { - am *accounts.Manager - b Backend + am *accounts.Manager + nonceLock *AddrLocker + b Backend } // NewPrivateAccountAPI create a new PrivateAccountAPI. -func NewPrivateAccountAPI(b Backend) *PrivateAccountAPI { +func NewPrivateAccountAPI(b Backend, nonceLock *AddrLocker) *PrivateAccountAPI { return &PrivateAccountAPI{ - am: b.AccountManager(), - b: b, + am: b.AccountManager(), + nonceLock: nonceLock, + b: b, } } // ListAccounts will return a list of addresses for accounts this node manages. func (s *PrivateAccountAPI) ListAccounts() []common.Address { - var addresses []common.Address + addresses := make([]common.Address, 0) // return [] instead of nil if empty for _, wallet := range s.am.Wallets() { for _, account := range wallet.Accounts() { addresses = append(addresses, account.Address) @@ -237,7 +238,7 @@ type rawWallet struct { // ListWallets will return a list of wallets this node manages. func (s *PrivateAccountAPI) ListWallets() []rawWallet { - var wallets []rawWallet + wallets := make([]rawWallet, 0) // return [] instead of nil if empty for _, wallet := range s.am.Wallets() { wallets = append(wallets, rawWallet{ URL: wallet.URL().String(), @@ -282,12 +283,11 @@ func fetchKeystore(am *accounts.Manager) *keystore.KeyStore { // ImportRawKey stores the given hex encoded ECDSA key into the key directory, // encrypting it with the passphrase. func (s *PrivateAccountAPI) ImportRawKey(privkey string, password string) (common.Address, error) { - hexkey, err := hex.DecodeString(privkey) + key, err := crypto.HexToECDSA(privkey) if err != nil { return common.Address{}, err } - - acc, err := fetchKeystore(s.am).ImportECDSA(crypto.ToECDSA(hexkey), password) + acc, err := fetchKeystore(s.am).ImportECDSA(key, password) return acc.Address, err } @@ -317,10 +317,6 @@ func (s *PrivateAccountAPI) LockAccount(addr common.Address) bool { // tries to sign it with the key associated with args.To. If the given passwd isn't // able to decrypt the key it fails. func (s *PrivateAccountAPI) SendTransaction(ctx context.Context, args SendTxArgs, passwd string) (common.Hash, error) { - // Set some sanity defaults and terminate on failure - if err := args.setDefaults(ctx, s.b); err != nil { - return common.Hash{}, err - } // Look up the wallet containing the requested signer account := accounts.Account{Address: args.From} @@ -328,6 +324,18 @@ func (s *PrivateAccountAPI) SendTransaction(ctx context.Context, args SendTxArgs if err != nil { return common.Hash{}, err } + + if args.Nonce == nil { + // Hold the addresse's mutex around signing to prevent concurrent assignment of + // the same nonce to multiple accounts. + s.nonceLock.LockAddr(args.From) + defer s.nonceLock.UnlockAddr(args.From) + } + + // Set some sanity defaults and terminate on failure + if err := args.setDefaults(ctx, s.b); err != nil { + return common.Hash{}, err + } // Assemble the transaction and sign with the wallet tx := args.toTransaction() @@ -887,12 +895,13 @@ func newRPCTransaction(b *types.Block, txHash common.Hash) (*RPCTransaction, err // PublicTransactionPoolAPI exposes methods for the RPC interface type PublicTransactionPoolAPI struct { - b Backend + b Backend + nonceLock *AddrLocker } // NewPublicTransactionPoolAPI creates a new RPC service with methods specific for the transaction pool. -func NewPublicTransactionPoolAPI(b Backend) *PublicTransactionPoolAPI { - return &PublicTransactionPoolAPI{b} +func NewPublicTransactionPoolAPI(b Backend, nonceLock *AddrLocker) *PublicTransactionPoolAPI { + return &PublicTransactionPoolAPI{b, nonceLock} } func getTransaction(chainDb ethdb.Database, b Backend, txHash common.Hash) (*types.Transaction, bool, error) { @@ -1170,10 +1179,7 @@ func submitTransaction(ctx context.Context, b Backend, tx *types.Transaction) (c // SendTransaction creates a transaction for the given argument, sign it and submit it to the // transaction pool. func (s *PublicTransactionPoolAPI) SendTransaction(ctx context.Context, args SendTxArgs) (common.Hash, error) { - // Set some sanity defaults and terminate on failure - if err := args.setDefaults(ctx, s.b); err != nil { - return common.Hash{}, err - } + // Look up the wallet containing the requested signer account := accounts.Account{Address: args.From} @@ -1181,6 +1187,18 @@ func (s *PublicTransactionPoolAPI) SendTransaction(ctx context.Context, args Sen if err != nil { return common.Hash{}, err } + + if args.Nonce == nil { + // Hold the addresse's mutex around signing to prevent concurrent assignment of + // the same nonce to multiple accounts. + s.nonceLock.LockAddr(args.From) + defer s.nonceLock.UnlockAddr(args.From) + } + + // Set some sanity defaults and terminate on failure + if err := args.setDefaults(ctx, s.b); err != nil { + return common.Hash{}, err + } // Assemble the transaction and sign with the wallet tx := args.toTransaction() @@ -1257,6 +1275,12 @@ type SignTransactionResult struct { // The node needs to have the private key of the account corresponding with // the given from address and it needs to be unlocked. func (s *PublicTransactionPoolAPI) SignTransaction(ctx context.Context, args SendTxArgs) (*SignTransactionResult, error) { + if args.Nonce == nil { + // Hold the addresse's mutex around signing to prevent concurrent assignment of + // the same nonce to multiple accounts. + s.nonceLock.LockAddr(args.From) + defer s.nonceLock.UnlockAddr(args.From) + } if err := args.setDefaults(ctx, s.b); err != nil { return nil, err } @@ -1435,11 +1459,11 @@ func (api *PrivateDebugAPI) SetHead(number hexutil.Uint64) { // PublicNetAPI offers network related RPC methods type PublicNetAPI struct { net *p2p.Server - networkVersion int + networkVersion uint64 } // NewPublicNetAPI creates a new net API instance. -func NewPublicNetAPI(net *p2p.Server, networkVersion int) *PublicNetAPI { +func NewPublicNetAPI(net *p2p.Server, networkVersion uint64) *PublicNetAPI { return &PublicNetAPI{net, networkVersion} } diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index 32cdf76563..0553ed7624 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -73,6 +73,7 @@ type State interface { } func GetAPIs(apiBackend Backend) []rpc.API { + nonceLock := new(AddrLocker) return []rpc.API{ { Namespace: "eth", @@ -87,7 +88,7 @@ func GetAPIs(apiBackend Backend) []rpc.API { }, { Namespace: "eth", Version: "1.0", - Service: NewPublicTransactionPoolAPI(apiBackend), + Service: NewPublicTransactionPoolAPI(apiBackend, nonceLock), Public: true, }, { Namespace: "txpool", @@ -111,7 +112,7 @@ func GetAPIs(apiBackend Backend) []rpc.API { }, { Namespace: "personal", Version: "1.0", - Service: NewPrivateAccountAPI(apiBackend), + Service: NewPrivateAccountAPI(apiBackend, nonceLock), Public: false, }, } diff --git a/internal/ethapi/tracer_test.go b/internal/ethapi/tracer_test.go index ca7a308580..4f4519a9f7 100644 --- a/internal/ethapi/tracer_test.go +++ b/internal/ethapi/tracer_test.go @@ -48,7 +48,7 @@ func runTrace(tracer *JavascriptTracer) (interface{}, error) { contract := vm.NewContract(account{}, account{}, big.NewInt(0), 10000) contract.Code = []byte{byte(vm.PUSH1), 0x1, byte(vm.PUSH1), 0x1, 0x0} - _, err := env.Interpreter().Run(contract, []byte{}) + _, err := env.Interpreter().Run(0, contract, []byte{}) if err != nil { return nil, err } diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 63d5288ee6..f71aef9403 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -345,6 +345,11 @@ web3._extend({ call: 'debug_getBadBlocks', params: 0, }), + new web3._extend.Method({ + name: 'storageRangeAt', + call: 'debug_storageRangeAt', + params: 5, + }), ], properties: [] }); @@ -520,7 +525,105 @@ web3._extend({ const Shh_JS = ` web3._extend({ property: 'shh', - methods: [], + methods: [ + new web3._extend.Method({ + name: 'info', + call: 'shh_info' + }), + new web3._extend.Method({ + name: 'setMaxMessageLength', + call: 'shh_setMaxMessageLength', + params: 1 + }), + new web3._extend.Method({ + name: 'setMinimumPoW', + call: 'shh_setMinimumPoW', + params: 1 + }), + new web3._extend.Method({ + name: 'allowP2PMessagesFromPeer', + call: 'shh_allowP2PMessagesFromPeer', + params: 1 + }), + new web3._extend.Method({ + name: 'hasKeyPair', + call: 'shh_hasKeyPair', + params: 1 + }), + new web3._extend.Method({ + name: 'deleteKeyPair', + call: 'shh_deleteKeyPair', + params: 1 + }), + new web3._extend.Method({ + name: 'newKeyPair', + call: 'shh_newKeyPair' + }), + new web3._extend.Method({ + name: 'getPublicKey', + call: 'shh_getPublicKey', + params: 1 + }), + new web3._extend.Method({ + name: 'getPrivateKey', + call: 'shh_getPrivateKey', + params: 1 + }), + new web3._extend.Method({ + name: 'generateSymmetricKey', + call: 'shh_generateSymmetricKey', + }), + new web3._extend.Method({ + name: 'addSymmetricKeyDirect', + call: 'shh_addSymmetricKeyDirect', + params: 1 + }), + new web3._extend.Method({ + name: 'addSymmetricKeyFromPassword', + call: 'shh_addSymmetricKeyFromPassword', + params: 1 + }), + new web3._extend.Method({ + name: 'hasSymmetricKey', + call: 'shh_hasSymmetricKey', + params: 1 + }), + new web3._extend.Method({ + name: 'getSymmetricKey', + call: 'shh_getSymmetricKey', + params: 1 + }), + new web3._extend.Method({ + name: 'deleteSymmetricKey', + call: 'shh_deleteSymmetricKey', + params: 1 + }), + new web3._extend.Method({ + name: 'subscribe', + call: 'shh_subscribe', + params: 1 + }), + new web3._extend.Method({ + name: 'unsubscribe', + call: 'shh_unsubscribe', + params: 1 + }), + new web3._extend.Method({ + name: 'getNewSubscriptionMessages', + call: 'shh_getNewSubscriptionMessages', + params: 1 + }), + new web3._extend.Method({ + name: 'getFloatingMessages', + call: 'shh_getFloatingMessages', + params: 1 + }), + new web3._extend.Method({ + name: 'post', + call: 'shh_post', + params: 1 + }) + ], properties: [ new web3._extend.Property({ @@ -531,6 +634,7 @@ web3._extend({ ] }); ` + const SWARMFS_JS = ` web3._extend({ property: 'swarmfs', diff --git a/les/backend.go b/les/backend.go index acaa8f880a..69ca7b1b76 100644 --- a/les/backend.go +++ b/les/backend.go @@ -61,7 +61,7 @@ type LightEthereum struct { engine consensus.Engine accountManager *accounts.Manager - netVersionId int + networkId uint64 netRPCService *ethapi.PublicNetAPI } @@ -87,7 +87,7 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { accountManager: ctx.AccountManager, engine: eth.CreateConsensusEngine(ctx, config, chainConfig, chainDb), shutdownChan: make(chan bool), - netVersionId: config.NetworkId, + networkId: config.NetworkId, } if eth.blockchain, err = light.NewLightChain(odr, eth.chainConfig, eth.engine, eth.eventMux); err != nil { return nil, err @@ -204,7 +204,7 @@ func (s *LightEthereum) Protocols() []p2p.Protocol { // Ethereum protocol implementation. func (s *LightEthereum) Start(srvr *p2p.Server) error { log.Warn("Light client mode is an experimental feature") - s.netRPCService = ethapi.NewPublicNetAPI(srvr, s.netVersionId) + s.netRPCService = ethapi.NewPublicNetAPI(srvr, s.networkId) s.protocolManager.Start(srvr) return nil } diff --git a/les/execqueue.go b/les/execqueue.go index ac779003bd..614721bf0d 100644 --- a/les/execqueue.go +++ b/les/execqueue.go @@ -16,56 +16,82 @@ package les -import ( - "sync/atomic" -) +import "sync" -// ExecQueue implements a queue that executes function calls in a single thread, +// execQueue implements a queue that executes function calls in a single thread, // in the same order as they have been queued. type execQueue struct { - chn chan func() - cnt, stop, capacity int32 + mu sync.Mutex + cond *sync.Cond + funcs []func() + closeWait chan struct{} } -// NewExecQueue creates a new execution queue. -func newExecQueue(capacity int32) *execQueue { - q := &execQueue{ - chn: make(chan func(), capacity), - capacity: capacity, - } +// newExecQueue creates a new execution queue. +func newExecQueue(capacity int) *execQueue { + q := &execQueue{funcs: make([]func(), 0, capacity)} + q.cond = sync.NewCond(&q.mu) go q.loop() return q } func (q *execQueue) loop() { - for f := range q.chn { - atomic.AddInt32(&q.cnt, -1) - if atomic.LoadInt32(&q.stop) != 0 { - return - } + for f := q.waitNext(false); f != nil; f = q.waitNext(true) { f() } + close(q.closeWait) } -// CanQueue returns true if more function calls can be added to the execution queue. +func (q *execQueue) waitNext(drop bool) (f func()) { + q.mu.Lock() + if drop { + // Remove the function that just executed. We do this here instead of when + // dequeuing so len(q.funcs) includes the function that is running. + q.funcs = append(q.funcs[:0], q.funcs[1:]...) + } + for !q.isClosed() { + if len(q.funcs) > 0 { + f = q.funcs[0] + break + } + q.cond.Wait() + } + q.mu.Unlock() + return f +} + +func (q *execQueue) isClosed() bool { + return q.closeWait != nil +} + +// canQueue returns true if more function calls can be added to the execution queue. func (q *execQueue) canQueue() bool { - return atomic.LoadInt32(&q.stop) == 0 && atomic.LoadInt32(&q.cnt) < q.capacity + q.mu.Lock() + ok := !q.isClosed() && len(q.funcs) < cap(q.funcs) + q.mu.Unlock() + return ok } -// Queue adds a function call to the execution queue. Returns true if successful. +// queue adds a function call to the execution queue. Returns true if successful. func (q *execQueue) queue(f func()) bool { - if atomic.LoadInt32(&q.stop) != 0 { - return false + q.mu.Lock() + ok := !q.isClosed() && len(q.funcs) < cap(q.funcs) + if ok { + q.funcs = append(q.funcs, f) + q.cond.Signal() } - if atomic.AddInt32(&q.cnt, 1) > q.capacity { - atomic.AddInt32(&q.cnt, -1) - return false - } - q.chn <- f - return true + q.mu.Unlock() + return ok } -// Stop stops the exec queue. +// quit stops the exec queue. +// quit waits for the current execution to finish before returning. func (q *execQueue) quit() { - atomic.StoreInt32(&q.stop, 1) + q.mu.Lock() + if !q.isClosed() { + q.closeWait = make(chan struct{}) + q.cond.Signal() + } + q.mu.Unlock() + <-q.closeWait } diff --git a/les/execqueue_test.go b/les/execqueue_test.go new file mode 100644 index 0000000000..cd45b03f22 --- /dev/null +++ b/les/execqueue_test.go @@ -0,0 +1,62 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package les + +import ( + "testing" +) + +func TestExecQueue(t *testing.T) { + var ( + N = 10000 + q = newExecQueue(N) + counter int + execd = make(chan int) + testexit = make(chan struct{}) + ) + defer q.quit() + defer close(testexit) + + check := func(state string, wantOK bool) { + c := counter + counter++ + qf := func() { + select { + case execd <- c: + case <-testexit: + } + } + if q.canQueue() != wantOK { + t.Fatalf("canQueue() == %t for %s", !wantOK, state) + } + if q.queue(qf) != wantOK { + t.Fatalf("canQueue() == %t for %s", !wantOK, state) + } + } + + for i := 0; i < N; i++ { + check("queue below cap", true) + } + check("full queue", false) + for i := 0; i < N; i++ { + if c := <-execd; c != i { + t.Fatal("execution out of order") + } + } + q.quit() + check("closed queue", false) +} diff --git a/les/handler.go b/les/handler.go index 0c3f8592d1..e4d482701c 100644 --- a/les/handler.go +++ b/les/handler.go @@ -95,7 +95,7 @@ type ProtocolManager struct { lightSync bool txpool txPool txrelay *LesTxRelay - networkId int + networkId uint64 chainConfig *params.ChainConfig blockchain BlockChain chainDb ethdb.Database @@ -128,7 +128,7 @@ type ProtocolManager struct { // NewProtocolManager returns a new ethereum sub protocol manager. The Ethereum sub protocol manages peers capable // with the ethereum network. -func NewProtocolManager(chainConfig *params.ChainConfig, lightSync bool, networkId int, mux *event.TypeMux, engine consensus.Engine, blockchain BlockChain, txpool txPool, chainDb ethdb.Database, odr *LesOdr, txrelay *LesTxRelay) (*ProtocolManager, error) { +func NewProtocolManager(chainConfig *params.ChainConfig, lightSync bool, networkId uint64, mux *event.TypeMux, engine consensus.Engine, blockchain BlockChain, txpool txPool, chainDb ethdb.Database, odr *LesOdr, txrelay *LesTxRelay) (*ProtocolManager, error) { // Create the protocol manager with the base fields manager := &ProtocolManager{ lightSync: lightSync, @@ -310,7 +310,7 @@ func (pm *ProtocolManager) Stop() { log.Info("Light Expanse protocol stopped") } -func (pm *ProtocolManager) newPeer(pv, nv int, p *p2p.Peer, rw p2p.MsgReadWriter) *peer { +func (pm *ProtocolManager) newPeer(pv int, nv uint64, p *p2p.Peer, rw p2p.MsgReadWriter) *peer { return newPeer(pv, nv, p, newMeteredMsgWriter(rw)) } diff --git a/les/peer.go b/les/peer.go index 57b3ec81a9..8994c37208 100644 --- a/les/peer.go +++ b/les/peer.go @@ -48,8 +48,8 @@ type peer struct { rw p2p.MsgReadWriter - version int // Protocol version negotiated - network int // Network ID being on + version int // Protocol version negotiated + network uint64 // Network ID being on id string @@ -69,7 +69,7 @@ type peer struct { fcCosts requestCostTable } -func newPeer(version, network int, p *p2p.Peer, rw p2p.MsgReadWriter) *peer { +func newPeer(version int, network uint64, p *p2p.Peer, rw p2p.MsgReadWriter) *peer { id := p.ID() return &peer{ @@ -384,7 +384,7 @@ func (p *peer) Handshake(td *big.Int, head common.Hash, headNum uint64, genesis if rGenesis != genesis { return errResp(ErrGenesisBlockMismatch, "%x (!= %x)", rGenesis[:8], genesis[:8]) } - if int(rNetwork) != p.network { + if rNetwork != p.network { return errResp(ErrNetworkIdMismatch, "%d (!= %d)", rNetwork, p.network) } if int(rVersion) != p.version { diff --git a/light/lightchain.go b/light/lightchain.go index dc7065bc63..57f3a3bc3c 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -377,9 +377,6 @@ func (self *LightChain) InsertHeaderChain(chain []*types.Header, checkFreq int) case core.SideStatTy: log.Debug("Inserted forked header", "number", header.Number, "hash", header.Hash()) events = append(events, core.ChainSideEvent{Block: types.NewBlockWithHeader(header)}) - - case core.SplitStatTy: - events = append(events, core.ChainSplitEvent{Block: types.NewBlockWithHeader(header)}) } return err } diff --git a/light/trie.go b/light/trie.go index 4d37560434..3d95debe0e 100644 --- a/light/trie.go +++ b/light/trie.go @@ -19,6 +19,7 @@ package light import ( "context" + "github.com/expanse-org/go-expanse/crypto" "github.com/expanse-org/go-expanse/ethdb" "github.com/expanse-org/go-expanse/trie" ) @@ -46,26 +47,18 @@ func NewLightTrie(id *TrieID, odr OdrBackend, useFakeMap bool) *LightTrie { // retrieveKey retrieves a single key, returns true and stores nodes in local // database if successful func (t *LightTrie) retrieveKey(ctx context.Context, key []byte) bool { - r := &TrieRequest{Id: t.id, Key: key} + r := &TrieRequest{Id: t.id, Key: crypto.Keccak256(key)} return t.odr.Retrieve(ctx, r) == nil } // do tries and retries to execute a function until it returns with no error or // an error type other than MissingNodeError -func (t *LightTrie) do(ctx context.Context, fallbackKey []byte, fn func() error) error { +func (t *LightTrie) do(ctx context.Context, key []byte, fn func() error) error { err := fn() for err != nil { - mn, ok := err.(*trie.MissingNodeError) - if !ok { + if _, ok := err.(*trie.MissingNodeError); !ok { return err } - - var key []byte - if mn.PrefixLen+mn.SuffixLen > 0 { - key = mn.Key - } else { - key = fallbackKey - } if !t.retrieveKey(ctx, key) { break } diff --git a/miner/miner.go b/miner/miner.go index 517f47ac89..9636591d27 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -19,7 +19,6 @@ package miner import ( "fmt" - "math/big" "sync/atomic" "github.com/expanse-org/go-expanse/accounts" @@ -104,18 +103,6 @@ out: } } -func (m *Miner) GasPrice() *big.Int { - return new(big.Int).Set(m.worker.gasPrice) -} - -func (m *Miner) SetGasPrice(price *big.Int) { - // FIXME block tests set a nil gas price. Quick dirty fix - if price == nil { - return - } - m.worker.setGasPrice(price) -} - func (self *Miner) Start(coinbase common.Address) { atomic.StoreInt32(&self.shouldStart, 1) self.worker.setEtherbase(coinbase) diff --git a/miner/worker.go b/miner/worker.go index 8f8f666eb5..9badc43959 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -59,14 +59,12 @@ type Work struct { config *params.ChainConfig signer types.Signer - state *state.StateDB // apply state changes here - ancestors *set.Set // ancestor set (used for checking uncle parent validity) - family *set.Set // family set (used for checking uncle invalidity) - uncles *set.Set // uncle set - tcount int // tx count in cycle - ownedAccounts *set.Set - lowGasTxs types.Transactions - failedTxs types.Transactions + state *state.StateDB // apply state changes here + ancestors *set.Set // ancestor set (used for checking uncle parent validity) + family *set.Set // family set (used for checking uncle invalidity) + uncles *set.Set // uncle set + tcount int // tx count in cycle + failedTxs types.Transactions Block *types.Block // the new block @@ -103,7 +101,6 @@ type worker struct { chainDb ethdb.Database coinbase common.Address - gasPrice *big.Int extra []byte currentMu sync.Mutex @@ -132,7 +129,6 @@ func newWorker(config *params.ChainConfig, engine consensus.Engine, coinbase com mux: mux, chainDb: eth.ChainDb(), recv: make(chan *Result, resultQueueSize), - gasPrice: new(big.Int), chain: eth.BlockChain(), proc: eth.BlockChain().Validator(), possibleUncles: make(map[common.Hash]*types.Block), @@ -252,7 +248,7 @@ func (self *worker) update() { txs := map[common.Address]types.Transactions{acc: {ev.Tx}} txset := types.NewTransactionsByPriceAndNonce(txs) - self.current.commitTransactions(self.mux, txset, self.gasPrice, self.chain, self.coinbase) + self.current.commitTransactions(self.mux, txset, self.chain, self.coinbase) self.currentMu.Unlock() } } @@ -375,22 +371,10 @@ func (self *worker) makeCurrent(parent *types.Block, header *types.Header) error } // Keep track of transactions which return errors so they can be removed work.tcount = 0 - work.ownedAccounts = accountAddressesSet(accounts) self.current = work return nil } -func (w *worker) setGasPrice(p *big.Int) { - w.mu.Lock() - defer w.mu.Unlock() - - // calculate the minimal gas price the miner accepts when sorting out transactions. - const pct = int64(90) - w.gasPrice = gasprice(p, pct) - - w.mux.Post(core.GasPriceChanged{Price: w.gasPrice}) -} - func (self *worker) commitNewWork() { self.mu.Lock() defer self.mu.Unlock() @@ -460,9 +444,8 @@ func (self *worker) commitNewWork() { return } txs := types.NewTransactionsByPriceAndNonce(pending) - work.commitTransactions(self.mux, txs, self.gasPrice, self.chain, self.coinbase) + work.commitTransactions(self.mux, txs, self.chain, self.coinbase) - self.eth.TxPool().RemoveBatch(work.lowGasTxs) self.eth.TxPool().RemoveBatch(work.failedTxs) // compute uncles for the new block. @@ -515,7 +498,7 @@ func (self *worker) commitUncle(work *Work, uncle *types.Header) error { return nil } -func (env *Work) commitTransactions(mux *event.TypeMux, txs *types.TransactionsByPriceAndNonce, gasPrice *big.Int, bc *core.BlockChain, coinbase common.Address) { +func (env *Work) commitTransactions(mux *event.TypeMux, txs *types.TransactionsByPriceAndNonce, bc *core.BlockChain, coinbase common.Address) { gp := new(core.GasPool).AddGas(env.header.GasLimit) var coalescedLogs []*types.Log @@ -539,19 +522,8 @@ func (env *Work) commitTransactions(mux *event.TypeMux, txs *types.TransactionsB txs.Pop() continue } - - // Ignore any transactions (and accounts subsequently) with low gas limits - if tx.GasPrice().Cmp(gasPrice) < 0 && !env.ownedAccounts.Has(from) { - // Pop the current low-priced transaction without shifting in the next from the account - log.Warn("Transaction below gas price", "sender", from, "hash", tx.Hash(), "have", tx.GasPrice(), "want", gasPrice) - - env.lowGasTxs = append(env.lowGasTxs, tx) - txs.Pop() - - continue - } // Start executing the transaction - env.state.StartRecord(tx.Hash(), common.Hash{}, env.tcount) + env.state.Prepare(tx.Hash(), common.Hash{}, env.tcount) err, logs := env.commitTransaction(tx, bc, coinbase, gp) switch err { @@ -607,25 +579,3 @@ func (env *Work) commitTransaction(tx *types.Transaction, bc *core.BlockChain, c return nil, receipt.Logs } - -// TODO: remove or use -func (self *worker) HashRate() int64 { - return 0 -} - -// gasprice calculates a reduced gas price based on the pct -// XXX Use big.Rat? -func gasprice(price *big.Int, pct int64) *big.Int { - p := new(big.Int).Set(price) - p.Div(p, big.NewInt(100)) - p.Mul(p, big.NewInt(pct)) - return p -} - -func accountAddressesSet(accounts []accounts.Account) *set.Set { - accountSet := set.New() - for _, account := range accounts { - accountSet.Add(account.Address) - } - return accountSet -} diff --git a/mobile/accounts.go b/mobile/accounts.go index 2577bc600a..8ec453f207 100644 --- a/mobile/accounts.go +++ b/mobile/accounts.go @@ -25,6 +25,7 @@ import ( "github.com/expanse-org/go-expanse/accounts" "github.com/expanse-org/go-expanse/accounts/keystore" + "github.com/expanse-org/go-expanse/crypto" ) const ( @@ -115,6 +116,9 @@ func (ks *KeyStore) SignHash(address *Address, hash []byte) (signature []byte, _ // SignTx signs the given transaction with the requested account. func (ks *KeyStore) SignTx(account *Account, tx *Transaction, chainID *BigInt) (*Transaction, error) { + if chainID == nil { // Null passed from mobile app + chainID = new(BigInt) + } signed, err := ks.keystore.SignTx(account.account, tx.tx, chainID.bigint) if err != nil { return nil, err @@ -132,6 +136,9 @@ func (ks *KeyStore) SignHashPassphrase(account *Account, passphrase string, hash // SignTxPassphrase signs the transaction if the private key matching the // given address can be decrypted with the given passphrase. func (ks *KeyStore) SignTxPassphrase(account *Account, passphrase string, tx *Transaction, chainID *BigInt) (*Transaction, error) { + if chainID == nil { // Null passed from mobile app + chainID = new(BigInt) + } signed, err := ks.keystore.SignTxWithPassphrase(account.account, passphrase, tx.tx, chainID.bigint) if err != nil { return nil, err @@ -170,6 +177,11 @@ func (ks *KeyStore) NewAccount(passphrase string) (*Account, error) { return &Account{account}, nil } +// UpdateAccount changes the passphrase of an existing account. +func (ks *KeyStore) UpdateAccount(account *Account, passphrase, newPassphrase string) error { + return ks.keystore.Update(account.account, passphrase, newPassphrase) +} + // ExportKey exports as a JSON key, encrypted with newPassphrase. func (ks *KeyStore) ExportKey(account *Account, passphrase, newPassphrase string) (key []byte, _ error) { return ks.keystore.Export(account.account, passphrase, newPassphrase) @@ -184,9 +196,17 @@ func (ks *KeyStore) ImportKey(keyJSON []byte, passphrase, newPassphrase string) return &Account{acc}, nil } -// UpdateAccount changes the passphrase of an existing account. -func (ks *KeyStore) UpdateAccount(account *Account, passphrase, newPassphrase string) error { - return ks.keystore.Update(account.account, passphrase, newPassphrase) +// ImportECDSAKey stores the given encrypted JSON key into the key directory. +func (ks *KeyStore) ImportECDSAKey(key []byte, passphrase string) (account *Account, _ error) { + privkey, err := crypto.ToECDSA(key) + if err != nil { + return nil, err + } + acc, err := ks.keystore.ImportECDSA(privkey, passphrase) + if err != nil { + return nil, err + } + return &Account{acc}, nil } // ImportPreSaleKey decrypts the given Ethereum presale wallet and stores diff --git a/mobile/common.go b/mobile/common.go index 62caa2a279..cb8326938c 100644 --- a/mobile/common.go +++ b/mobile/common.go @@ -89,6 +89,18 @@ func (h *Hash) GetHex() string { // Hashes represents a slice of hashes. type Hashes struct{ hashes []common.Hash } +// NewHashes creates a slice of uninitialized Hashes. +func NewHashes(size int) *Hashes { + return &Hashes{ + hashes: make([]common.Hash, size), + } +} + +// NewHashesEmpty creates an empty slice of Hashes values. +func NewHashesEmpty() *Hashes { + return NewHashes(0) +} + // Size returns the number of hashes in the slice. func (h *Hashes) Size() int { return len(h.hashes) @@ -102,6 +114,20 @@ func (h *Hashes) Get(index int) (hash *Hash, _ error) { return &Hash{h.hashes[index]}, nil } +// Set sets the Hash at the given index in the slice. +func (h *Hashes) Set(index int, hash *Hash) error { + if index < 0 || index >= len(h.hashes) { + return errors.New("index out of bounds") + } + h.hashes[index] = hash.hash + return nil +} + +// Append adds a new Hash element to the end of the slice. +func (h *Hashes) Append(hash *Hash) { + h.hashes = append(h.hashes, hash.hash) +} + // Address represents the 20 byte address of an Ethereum account. type Address struct { address common.Address @@ -164,6 +190,18 @@ func (a *Address) GetHex() string { // Addresses represents a slice of addresses. type Addresses struct{ addresses []common.Address } +// NewAddresses creates a slice of uninitialized addresses. +func NewAddresses(size int) *Addresses { + return &Addresses{ + addresses: make([]common.Address, size), + } +} + +// NewAddressesEmpty creates an empty slice of Addresses values. +func NewAddressesEmpty() *Addresses { + return NewAddresses(0) +} + // Size returns the number of addresses in the slice. func (a *Addresses) Size() int { return len(a.addresses) @@ -185,3 +223,8 @@ func (a *Addresses) Set(index int, address *Address) error { a.addresses[index] = address.address return nil } + +// Append adds a new address element to the end of the slice. +func (a *Addresses) Append(address *Address) { + a.addresses = append(a.addresses, address.address) +} diff --git a/mobile/ethereum.go b/mobile/ethereum.go index 7174ae728a..ee8d1e5b52 100644 --- a/mobile/ethereum.go +++ b/mobile/ethereum.go @@ -87,6 +87,18 @@ func (p *SyncProgress) GetKnownStates() int64 { return int64(p.progress.KnownS // Topics is a set of topic lists to filter events with. type Topics struct{ topics [][]common.Hash } +// NewTopics creates a slice of uninitialized Topics. +func NewTopics(size int) *Topics { + return &Topics{ + topics: make([][]common.Hash, size), + } +} + +// NewTopicsEmpty creates an empty slice of Topics values. +func NewTopicsEmpty() *Topics { + return NewTopics(0) +} + // Size returns the number of topic lists inside the set func (t *Topics) Size() int { return len(t.topics) @@ -109,6 +121,11 @@ func (t *Topics) Set(index int, topics *Hashes) error { return nil } +// Append adds a new topic list to the end of the slice. +func (t *Topics) Append(topics *Hashes) { + t.topics = append(t.topics, topics.hashes) +} + // FilterQuery contains options for contact log filtering. type FilterQuery struct { query ethereum.FilterQuery @@ -123,3 +140,8 @@ func (fq *FilterQuery) GetFromBlock() *BigInt { return &BigInt{fq.query.FromB func (fq *FilterQuery) GetToBlock() *BigInt { return &BigInt{fq.query.ToBlock} } func (fq *FilterQuery) GetAddresses() *Addresses { return &Addresses{fq.query.Addresses} } func (fq *FilterQuery) GetTopics() *Topics { return &Topics{fq.query.Topics} } + +func (fq *FilterQuery) SetFromBlock(fromBlock *BigInt) { fq.query.FromBlock = fromBlock.bigint } +func (fq *FilterQuery) SetToBlock(toBlock *BigInt) { fq.query.ToBlock = toBlock.bigint } +func (fq *FilterQuery) SetAddresses(addresses *Addresses) { fq.query.Addresses = addresses.addresses } +func (fq *FilterQuery) SetTopics(topics *Topics) { fq.query.Topics = topics.topics } diff --git a/mobile/geth.go b/mobile/geth.go index 014b00e134..6d6e76a692 100644 --- a/mobile/geth.go +++ b/mobile/geth.go @@ -34,7 +34,7 @@ import ( "github.com/expanse-org/go-expanse/p2p" "github.com/expanse-org/go-expanse/p2p/nat" "github.com/expanse-org/go-expanse/params" - whisper "github.com/expanse-org/go-expanse/whisper/whisperv2" + whisper "github.com/expanse-org/go-expanse/whisper/whisperv5" ) // NodeConfig represents the collection of configuration values to fine tune the Gexp @@ -54,7 +54,7 @@ type NodeConfig struct { // EthereumNetworkID is the network identifier used by the Ethereum protocol to // decide if remote peers should be accepted or not. - EthereumNetworkID int + EthereumNetworkID int64 // uint64 in truth, but Java can't handle that... // EthereumGenesis is the genesis JSON to use to seed the blockchain with. An // empty genesis state is equivalent to using the mainnet's state. @@ -148,7 +148,7 @@ func NewNode(datadir string, config *NodeConfig) (stack *Node, _ error) { ethConf := eth.DefaultConfig ethConf.Genesis = genesis ethConf.SyncMode = downloader.LightSync - ethConf.NetworkId = config.EthereumNetworkID + ethConf.NetworkId = uint64(config.EthereumNetworkID) ethConf.DatabaseCache = config.EthereumDatabaseCache if err := rawStack.Register(func(ctx *node.ServiceContext) (node.Service, error) { return les.New(ctx, ðConf) diff --git a/mobile/types.go b/mobile/types.go index b660cbaab5..c2131bbb76 100644 --- a/mobile/types.go +++ b/mobile/types.go @@ -19,10 +19,12 @@ package gexp import ( + "encoding/json" "errors" "fmt" "github.com/expanse-org/go-expanse/core/types" + "github.com/expanse-org/go-expanse/rlp" ) // A Nonce is a 64-bit hash which proves (combined with the mix-hash) that @@ -61,6 +63,45 @@ type Header struct { header *types.Header } +// NewHeaderFromRLP parses a header from an RLP data dump. +func NewHeaderFromRLP(data []byte) (*Header, error) { + h := &Header{ + header: new(types.Header), + } + if err := rlp.DecodeBytes(data, h.header); err != nil { + return nil, err + } + return h, nil +} + +// EncodeRLP encodes a header into an RLP data dump. +func (h *Header) EncodeRLP() ([]byte, error) { + return rlp.EncodeToBytes(h.header) +} + +// NewHeaderFromJSON parses a header from an JSON data dump. +func NewHeaderFromJSON(data string) (*Header, error) { + h := &Header{ + header: new(types.Header), + } + if err := json.Unmarshal([]byte(data), h.header); err != nil { + return nil, err + } + return h, nil +} + +// EncodeJSON encodes a header into an JSON data dump. +func (h *Header) EncodeJSON() (string, error) { + data, err := json.Marshal(h.header) + return string(data), err +} + +// String implements the fmt.Stringer interface to print some semi-meaningful +// data dump of the header for debugging purposes. +func (h *Header) String() string { + return h.header.String() +} + func (h *Header) GetParentHash() *Hash { return &Hash{h.header.ParentHash} } func (h *Header) GetUncleHash() *Hash { return &Hash{h.header.UncleHash} } func (h *Header) GetCoinbase() *Address { return &Address{h.header.Coinbase} } @@ -76,9 +117,7 @@ func (h *Header) GetTime() int64 { return h.header.Time.Int64() } func (h *Header) GetExtra() []byte { return h.header.Extra } func (h *Header) GetMixDigest() *Hash { return &Hash{h.header.MixDigest} } func (h *Header) GetNonce() *Nonce { return &Nonce{h.header.Nonce} } - -func (h *Header) GetHash() *Hash { return &Hash{h.header.Hash()} } -func (h *Header) GetHashNoNonce() *Hash { return &Hash{h.header.HashNoNonce()} } +func (h *Header) GetHash() *Hash { return &Hash{h.header.Hash()} } // Headers represents a slice of headers. type Headers struct{ headers []*types.Header } @@ -101,6 +140,45 @@ type Block struct { block *types.Block } +// NewBlockFromRLP parses a block from an RLP data dump. +func NewBlockFromRLP(data []byte) (*Block, error) { + b := &Block{ + block: new(types.Block), + } + if err := rlp.DecodeBytes(data, b.block); err != nil { + return nil, err + } + return b, nil +} + +// EncodeRLP encodes a block into an RLP data dump. +func (b *Block) EncodeRLP() ([]byte, error) { + return rlp.EncodeToBytes(b.block) +} + +// NewBlockFromJSON parses a block from an JSON data dump. +func NewBlockFromJSON(data string) (*Block, error) { + b := &Block{ + block: new(types.Block), + } + if err := json.Unmarshal([]byte(data), b.block); err != nil { + return nil, err + } + return b, nil +} + +// EncodeJSON encodes a block into an JSON data dump. +func (b *Block) EncodeJSON() (string, error) { + data, err := json.Marshal(b.block) + return string(data), err +} + +// String implements the fmt.Stringer interface to print some semi-meaningful +// data dump of the block for debugging purposes. +func (b *Block) String() string { + return b.block.String() +} + func (b *Block) GetParentHash() *Hash { return &Hash{b.block.ParentHash()} } func (b *Block) GetUncleHash() *Hash { return &Hash{b.block.UncleHash()} } func (b *Block) GetCoinbase() *Address { return &Address{b.block.Coinbase()} } @@ -137,6 +215,45 @@ func NewTransaction(nonce int64, to *Address, amount, gasLimit, gasPrice *BigInt return &Transaction{types.NewTransaction(uint64(nonce), to.address, amount.bigint, gasLimit.bigint, gasPrice.bigint, data)} } +// NewTransactionFromRLP parses a transaction from an RLP data dump. +func NewTransactionFromRLP(data []byte) (*Transaction, error) { + tx := &Transaction{ + tx: new(types.Transaction), + } + if err := rlp.DecodeBytes(data, tx.tx); err != nil { + return nil, err + } + return tx, nil +} + +// EncodeRLP encodes a transaction into an RLP data dump. +func (tx *Transaction) EncodeRLP() ([]byte, error) { + return rlp.EncodeToBytes(tx.tx) +} + +// NewTransactionFromJSON parses a transaction from an JSON data dump. +func NewTransactionFromJSON(data string) (*Transaction, error) { + tx := &Transaction{ + tx: new(types.Transaction), + } + if err := json.Unmarshal([]byte(data), tx.tx); err != nil { + return nil, err + } + return tx, nil +} + +// EncodeJSON encodes a transaction into an JSON data dump. +func (tx *Transaction) EncodeJSON() (string, error) { + data, err := json.Marshal(tx.tx) + return string(data), err +} + +// String implements the fmt.Stringer interface to print some semi-meaningful +// data dump of the transaction for debugging purposes. +func (tx *Transaction) String() string { + return tx.tx.String() +} + func (tx *Transaction) GetData() []byte { return tx.tx.Data() } func (tx *Transaction) GetGas() int64 { return tx.tx.Gas().Int64() } func (tx *Transaction) GetGasPrice() *BigInt { return &BigInt{tx.tx.GasPrice()} } @@ -185,6 +302,45 @@ type Receipt struct { receipt *types.Receipt } +// NewReceiptFromRLP parses a transaction receipt from an RLP data dump. +func NewReceiptFromRLP(data []byte) (*Receipt, error) { + r := &Receipt{ + receipt: new(types.Receipt), + } + if err := rlp.DecodeBytes(data, r.receipt); err != nil { + return nil, err + } + return r, nil +} + +// EncodeRLP encodes a transaction receipt into an RLP data dump. +func (r *Receipt) EncodeRLP() ([]byte, error) { + return rlp.EncodeToBytes(r.receipt) +} + +// NewReceiptFromJSON parses a transaction receipt from an JSON data dump. +func NewReceiptFromJSON(data string) (*Receipt, error) { + r := &Receipt{ + receipt: new(types.Receipt), + } + if err := json.Unmarshal([]byte(data), r.receipt); err != nil { + return nil, err + } + return r, nil +} + +// EncodeJSON encodes a transaction receipt into an JSON data dump. +func (r *Receipt) EncodeJSON() (string, error) { + data, err := rlp.EncodeToBytes(r.receipt) + return string(data), err +} + +// String implements the fmt.Stringer interface to print some semi-meaningful +// data dump of the transaction receipt for debugging purposes. +func (r *Receipt) String() string { + return r.receipt.String() +} + func (r *Receipt) GetPostState() []byte { return r.receipt.PostState } func (r *Receipt) GetCumulativeGasUsed() *BigInt { return &BigInt{r.receipt.CumulativeGasUsed} } func (r *Receipt) GetBloom() *Bloom { return &Bloom{r.receipt.Bloom} } diff --git a/node/config.go b/node/config.go index 48ff6050d3..76a7afb6ef 100644 --- a/node/config.go +++ b/node/config.go @@ -82,6 +82,9 @@ type Config struct { // scrypt KDF at the expense of security. UseLightweightKDF bool `toml:",omitempty"` + // NoUSB disables hardware wallet monitoring and connectivity. + NoUSB bool `toml:",omitempty"` + // IPCPath is the requested location to place the IPC endpoint. If the path is // a simple file name, it is placed inside the data directory (or on the root // pipe path on Windows), whereas if it's a resolvable path name (absolute or @@ -389,10 +392,12 @@ func makeAccountManager(conf *Config) (*accounts.Manager, string, error) { backends := []accounts.Backend{ keystore.NewKeyStore(keydir, scryptN, scryptP), } - if ledgerhub, err := usbwallet.NewLedgerHub(); err != nil { - log.Warn(fmt.Sprintf("Failed to start Ledger hub, disabling: %v", err)) - } else { - backends = append(backends, ledgerhub) + if !conf.NoUSB { + if ledgerhub, err := usbwallet.NewLedgerHub(); err != nil { + log.Warn(fmt.Sprintf("Failed to start Ledger hub, disabling: %v", err)) + } else { + backends = append(backends, ledgerhub) + } } return accounts.NewManager(backends...), ephemeral, nil } diff --git a/node/defaults.go b/node/defaults.go index 78923145ad..90d67fa0d6 100644 --- a/node/defaults.go +++ b/node/defaults.go @@ -41,9 +41,10 @@ var DefaultConfig = Config{ WSPort: DefaultWSPort, WSModules: []string{"net", "web3"}, P2P: p2p.Config{ - ListenAddr: ":42786", - MaxPeers: 50, - NAT: nat.Any(), + ListenAddr: ":42786", + DiscoveryV5Addr: ":42787", + MaxPeers: 50, + NAT: nat.Any(), }, } diff --git a/node/node.go b/node/node.go index f5041cc835..7fcd677d3e 100644 --- a/node/node.go +++ b/node/node.go @@ -536,6 +536,7 @@ func (n *Node) Stop() error { func (n *Node) Wait() { n.lock.RLock() if n.server == nil { + n.lock.RUnlock() return } stop := n.stop diff --git a/p2p/discover/udp.go b/p2p/discover/udp.go index ac06eb845e..d801b8a2a6 100644 --- a/p2p/discover/udp.go +++ b/p2p/discover/udp.go @@ -224,7 +224,7 @@ func ListenUDP(priv *ecdsa.PrivateKey, laddr string, natm nat.Interface, nodeDBP if err != nil { return nil, err } - log.Debug("UDP listener up", "self", tab.self) + log.Info("UDP listener up", "self", tab.self) return tab, nil } diff --git a/params/bootnodes.go b/params/bootnodes.go index d4ce4be943..190f163a69 100644 --- a/params/bootnodes.go +++ b/params/bootnodes.go @@ -36,6 +36,18 @@ var TestnetBootnodes = []string{ "enode://20c9ad97c081d63397d7b685a412227a40e23c8bdc6688c6f37e97cfbc22d2b4d1db1510d8f61e6a8866ad7f0e17c02b14182d37ea7c3c8b9c2683aeb6b733a1@52.169.14.227:30303", // IE } +// RinkebyBootnodes are the enode URLs of the P2P bootstrap nodes running on the +// Rinkeby test network. +var RinkebyBootnodes = []string{ + "enode://a24ac7c5484ef4ed0c5eb2d36620ba4e4aa13b8c84684e1b4aab0cebea2ae45cb4d375b77eab56516d34bfbd3c1a833fc51296ff084b770b94fb9028c4d25ccf@52.169.42.101:30303", // IE +} + +// RinkebyV5Bootnodes are the enode URLs of the P2P bootstrap nodes running on the +// Rinkeby test network for the experimental RLPx v5 topic-discovery network. +var RinkebyV5Bootnodes = []string{ + "enode://a24ac7c5484ef4ed0c5eb2d36620ba4e4aa13b8c84684e1b4aab0cebea2ae45cb4d375b77eab56516d34bfbd3c1a833fc51296ff084b770b94fb9028c4d25ccf@52.169.42.101:30303?discport=30304", // IE +} + // DiscoveryV5Bootnodes are the enode URLs of the P2P bootstrap nodes for the // experimental RLPx v5 topic-discovery network. var DiscoveryV5Bootnodes = []string{ diff --git a/params/config.go b/params/config.go index 44267b1527..dc3c209ef0 100644 --- a/params/config.go +++ b/params/config.go @@ -26,41 +26,63 @@ import ( var ( // MainnetChainConfig is the chain parameters to run a node on the main network. MainnetChainConfig = &ChainConfig{ - ChainId: MainNetChainID, - HomesteadBlock: MainNetHomesteadBlock, - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: MainNetHomesteadGasRepriceBlock, - EIP150Hash: MainNetHomesteadGasRepriceHash, - EIP155Block: MainNetSpuriousDragon, - EIP158Block: MainNetSpuriousDragon, - Ethash: new(EthashConfig), + ChainId: MainNetChainID, + HomesteadBlock: MainNetHomesteadBlock, + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: MainNetHomesteadGasRepriceBlock, + EIP150Hash: MainNetHomesteadGasRepriceHash, + EIP155Block: MainNetSpuriousDragon, + EIP158Block: MainNetSpuriousDragon, + MetropolisBlock: MainNetMetropolisBlock, + + Ethash: new(EthashConfig), } - // TestnetChainConfig contains the chain parameters to run a node on the ropsten test network. + // TestnetChainConfig contains the chain parameters to run a node on the Ropsten test network. TestnetChainConfig = &ChainConfig{ - ChainId: big.NewInt(3), - HomesteadBlock: big.NewInt(0), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(0), - EIP150Hash: common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d"), - EIP155Block: big.NewInt(10), - EIP158Block: big.NewInt(10), - Ethash: new(EthashConfig), + ChainId: big.NewInt(3), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(0), + EIP150Hash: common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d"), + EIP155Block: big.NewInt(10), + EIP158Block: big.NewInt(10), + MetropolisBlock: TestNetMetropolisBlock, + + Ethash: new(EthashConfig), + } + + // RinkebyChainConfig contains the chain parameters to run a node on the Rinkeby test network. + RinkebyChainConfig = &ChainConfig{ + ChainId: big.NewInt(4), + HomesteadBlock: big.NewInt(1), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(2), + EIP150Hash: common.HexToHash("0x9b095b36c15eaf13044373aef8ee0bd3a382a5abb92e402afa44b8249c3a90e9"), + EIP155Block: big.NewInt(3), + EIP158Block: big.NewInt(3), + MetropolisBlock: TestNetMetropolisBlock, + + Clique: &CliqueConfig{ + Period: 15, + Epoch: 30000, + }, } // AllProtocolChanges contains every protocol change (EIPs) // introduced and accepted by the Ethereum core developers. - // TestChainConfig is like AllProtocolChanges but has chain ID 1. // // This configuration is intentionally not using keyed fields. // This configuration must *always* have all forks enabled, which // means that all fields must be set at all times. This forces // anyone adding flags to the config to also have to set these // fields. - AllProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), new(EthashConfig), nil} - TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), new(EthashConfig), nil} + AllProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), new(EthashConfig), nil} + TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil} + TestRules = TestChainConfig.Rules(new(big.Int)) ) // ChainConfig is the core config which determines the blockchain settings. @@ -82,6 +104,8 @@ type ChainConfig struct { EIP155Block *big.Int `json:"eip155Block,omitempty"` // EIP155 HF block EIP158Block *big.Int `json:"eip158Block,omitempty"` // EIP158 HF block + MetropolisBlock *big.Int `json:"metropolisBlock,omitempty"` // Metropolis switch block (nil = no fork, 0 = alraedy on homestead) + // Various consensus engines Ethash *EthashConfig `json:"ethash,omitempty"` Clique *CliqueConfig `json:"clique,omitempty"` @@ -117,7 +141,7 @@ func (c *ChainConfig) String() string { default: engine = "unknown" } - return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Engine: %v}", + return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Metropolis: %v Engine: %v}", c.ChainId, c.HomesteadBlock, c.DAOForkBlock, @@ -125,6 +149,7 @@ func (c *ChainConfig) String() string { c.EIP150Block, c.EIP155Block, c.EIP158Block, + c.MetropolisBlock, engine, ) } @@ -151,6 +176,10 @@ func (c *ChainConfig) IsEIP158(num *big.Int) bool { return isForked(c.EIP158Block, num) } +func (c *ChainConfig) IsMetropolis(num *big.Int) bool { + return isForked(c.MetropolisBlock, num) +} + // GasTable returns the gas table corresponding to the current phase (homestead or homestead reprice). // // The returned GasTable's fields shouldn't, under any circumstances, be changed. @@ -208,6 +237,9 @@ func (c *ChainConfig) checkCompatible(newcfg *ChainConfig, head *big.Int) *Confi if c.IsEIP158(head) && !configNumEqual(c.ChainId, newcfg.ChainId) { return newCompatError("EIP158 chain ID", c.EIP158Block, newcfg.EIP158Block) } + if isForkIncompatible(c.MetropolisBlock, newcfg.MetropolisBlock, head) { + return newCompatError("Metropolis fork block", c.MetropolisBlock, newcfg.MetropolisBlock) + } return nil } @@ -265,3 +297,22 @@ func newCompatError(what string, storedblock, newblock *big.Int) *ConfigCompatEr func (err *ConfigCompatError) Error() string { return fmt.Sprintf("mismatching %s in database (have %d, want %d, rewindto %d)", err.What, err.StoredConfig, err.NewConfig, err.RewindTo) } + +// Rules wraps ChainConfig and is merely syntatic sugar or can be used for functions +// that do not have or require information about the block. +// +// Rules is a one time interface meaning that it shouldn't be used in between transition +// phases. +type Rules struct { + ChainId *big.Int + IsHomestead, IsEIP150, IsEIP155, IsEIP158 bool + IsMetropolis bool +} + +func (c *ChainConfig) Rules(num *big.Int) Rules { + chainId := c.ChainId + if chainId == nil { + chainId = new(big.Int) + } + return Rules{ChainId: new(big.Int).Set(chainId), IsHomestead: c.IsHomestead(num), IsEIP150: c.IsEIP150(num), IsEIP155: c.IsEIP155(num), IsEIP158: c.IsEIP158(num), IsMetropolis: c.IsMetropolis(num)} +} diff --git a/params/util.go b/params/util.go index 1634b2291e..a07a107e93 100644 --- a/params/util.go +++ b/params/util.go @@ -17,6 +17,7 @@ package params import ( + "math" "math/big" "github.com/expanse-org/go-expanse/common" @@ -39,6 +40,9 @@ var ( TestNetSpuriousDragon = big.NewInt(10) MainNetSpuriousDragon = big.NewInt(600000) - TestNetChainID = big.NewInt(3) // Testnet default chain ID - MainNetChainID = big.NewInt(2) // Mainnet default chain ID + TestNetMetropolisBlock = big.NewInt(math.MaxInt64) + MainNetMetropolisBlock = big.NewInt(math.MaxInt64) + + TestNetChainID = big.NewInt(3) // Test net default chain ID + MainNetChainID = big.NewInt(2) // main net default chain ID ) diff --git a/params/version.go b/params/version.go index 7f14e6521f..103b61410b 100644 --- a/params/version.go +++ b/params/version.go @@ -23,7 +23,7 @@ import ( const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 6 // Minor version component of the current release - VersionPatch = 2 // Patch version component of the current release + VersionPatch = 6 // Patch version component of the current release VersionMeta = "beta" // Version metadata to append to the version string ) diff --git a/rpc/client.go b/rpc/client.go index 9cb45413e7..05184f3eb9 100644 --- a/rpc/client.go +++ b/rpc/client.go @@ -27,6 +27,7 @@ import ( "net/url" "reflect" "strconv" + "strings" "sync" "sync/atomic" "time" @@ -373,14 +374,14 @@ func (c *Client) EthSubscribe(ctx context.Context, channel interface{}, args ... return nil, ErrNotificationsUnsupported } - msg, err := c.newMessage(subscribeMethod, args...) + msg, err := c.newMessage("eth"+subscribeMethodSuffix, args...) if err != nil { return nil, err } op := &requestOp{ ids: []json.RawMessage{msg.ID}, resp: make(chan *jsonrpcMessage), - sub: newClientSubscription(c, chanVal), + sub: newClientSubscription(c, "eth", chanVal), } // Send the subscription request. @@ -575,7 +576,7 @@ func (c *Client) closeRequestOps(err error) { } func (c *Client) handleNotification(msg *jsonrpcMessage) { - if msg.Method != notificationMethod { + if !strings.HasSuffix(msg.Method, notificationMethodSuffix) { log.Debug(fmt.Sprint("dropping non-subscription message: ", msg)) return } @@ -653,11 +654,12 @@ func (c *Client) read(conn net.Conn) error { // A ClientSubscription represents a subscription established through EthSubscribe. type ClientSubscription struct { - client *Client - etype reflect.Type - channel reflect.Value - subid string - in chan json.RawMessage + client *Client + etype reflect.Type + channel reflect.Value + namespace string + subid string + in chan json.RawMessage quitOnce sync.Once // ensures quit is closed once quit chan struct{} // quit is closed when the subscription exits @@ -665,14 +667,15 @@ type ClientSubscription struct { err chan error } -func newClientSubscription(c *Client, channel reflect.Value) *ClientSubscription { +func newClientSubscription(c *Client, namespace string, channel reflect.Value) *ClientSubscription { sub := &ClientSubscription{ - client: c, - etype: channel.Type().Elem(), - channel: channel, - quit: make(chan struct{}), - err: make(chan error, 1), - in: make(chan json.RawMessage), + client: c, + namespace: namespace, + etype: channel.Type().Elem(), + channel: channel, + quit: make(chan struct{}), + err: make(chan error, 1), + in: make(chan json.RawMessage), } return sub } @@ -774,5 +777,5 @@ func (sub *ClientSubscription) unmarshal(result json.RawMessage) (interface{}, e func (sub *ClientSubscription) requestUnsubscribe() error { var result interface{} - return sub.client.Call(&result, unsubscribeMethod, sub.subid) + return sub.client.Call(&result, sub.namespace+unsubscribeMethodSuffix, sub.subid) } diff --git a/rpc/http.go b/rpc/http.go index 022f9ce8f3..6bab02ab67 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -162,6 +162,11 @@ func (srv *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { } func newCorsHandler(srv *Server, allowedOrigins []string) http.Handler { + // disable CORS support if user has not specified a custom CORS configuration + if len(allowedOrigins) == 0 { + return srv + } + c := cors.New(cors.Options{ AllowedOrigins: allowedOrigins, AllowedMethods: []string{"POST", "GET"}, diff --git a/rpc/json.go b/rpc/json.go index 6d9ccc7f1b..8170e9fff7 100644 --- a/rpc/json.go +++ b/rpc/json.go @@ -30,11 +30,11 @@ import ( ) const ( - jsonrpcVersion = "2.0" - serviceMethodSeparator = "_" - subscribeMethod = "eth_subscribe" - unsubscribeMethod = "eth_unsubscribe" - notificationMethod = "eth_subscription" + jsonrpcVersion = "2.0" + serviceMethodSeparator = "_" + subscribeMethodSuffix = "_subscribe" + unsubscribeMethodSuffix = "_unsubscribe" + notificationMethodSuffix = "_subscription" ) type jsonRequest struct { @@ -164,7 +164,7 @@ func parseRequest(incomingMsg json.RawMessage) ([]rpcRequest, bool, Error) { } // subscribe are special, they will always use `subscribeMethod` as first param in the payload - if in.Method == subscribeMethod { + if strings.HasSuffix(in.Method, subscribeMethodSuffix) { reqs := []rpcRequest{{id: &in.Id, isPubSub: true}} if len(in.Payload) > 0 { // first param must be subscription name @@ -174,17 +174,16 @@ func parseRequest(incomingMsg json.RawMessage) ([]rpcRequest, bool, Error) { return nil, false, &invalidRequestError{"Unable to parse subscription request"} } - // all subscriptions are made on the eth service - reqs[0].service, reqs[0].method = "eth", subscribeMethod[0] + reqs[0].service, reqs[0].method = strings.TrimSuffix(in.Method, subscribeMethodSuffix), subscribeMethod[0] reqs[0].params = in.Payload return reqs, false, nil } return nil, false, &invalidRequestError{"Unable to parse subscription request"} } - if in.Method == unsubscribeMethod { + if strings.HasSuffix(in.Method, unsubscribeMethodSuffix) { return []rpcRequest{{id: &in.Id, isPubSub: true, - method: unsubscribeMethod, params: in.Payload}}, false, nil + method: in.Method, params: in.Payload}}, false, nil } elems := strings.Split(in.Method, serviceMethodSeparator) @@ -216,8 +215,8 @@ func parseBatchRequest(incomingMsg json.RawMessage) ([]rpcRequest, bool, Error) id := &in[i].Id - // subscribe are special, they will always use `subscribeMethod` as first param in the payload - if r.Method == subscribeMethod { + // subscribe are special, they will always use `subscriptionMethod` as first param in the payload + if strings.HasSuffix(r.Method, subscribeMethodSuffix) { requests[i] = rpcRequest{id: id, isPubSub: true} if len(r.Payload) > 0 { // first param must be subscription name @@ -227,8 +226,7 @@ func parseBatchRequest(incomingMsg json.RawMessage) ([]rpcRequest, bool, Error) return nil, false, &invalidRequestError{"Unable to parse subscription request"} } - // all subscriptions are made on the eth service - requests[i].service, requests[i].method = "eth", subscribeMethod[0] + requests[i].service, requests[i].method = strings.TrimSuffix(r.Method, subscribeMethodSuffix), subscribeMethod[0] requests[i].params = r.Payload continue } @@ -236,8 +234,8 @@ func parseBatchRequest(incomingMsg json.RawMessage) ([]rpcRequest, bool, Error) return nil, true, &invalidRequestError{"Unable to parse (un)subscribe request arguments"} } - if r.Method == unsubscribeMethod { - requests[i] = rpcRequest{id: id, isPubSub: true, method: unsubscribeMethod, params: r.Payload} + if strings.HasSuffix(r.Method, unsubscribeMethodSuffix) { + requests[i] = rpcRequest{id: id, isPubSub: true, method: r.Method, params: r.Payload} continue } @@ -325,13 +323,13 @@ func (c *jsonCodec) CreateErrorResponseWithInfo(id interface{}, err Error, info } // CreateNotification will create a JSON-RPC notification with the given subscription id and event as params. -func (c *jsonCodec) CreateNotification(subid string, event interface{}) interface{} { +func (c *jsonCodec) CreateNotification(subid, namespace string, event interface{}) interface{} { if isHexNum(reflect.TypeOf(event)) { - return &jsonNotification{Version: jsonrpcVersion, Method: notificationMethod, + return &jsonNotification{Version: jsonrpcVersion, Method: namespace + notificationMethodSuffix, Params: jsonSubscription{Subscription: subid, Result: fmt.Sprintf(`%#x`, event)}} } - return &jsonNotification{Version: jsonrpcVersion, Method: notificationMethod, + return &jsonNotification{Version: jsonrpcVersion, Method: namespace + notificationMethodSuffix, Params: jsonSubscription{Subscription: subid, Result: event}} } diff --git a/rpc/server.go b/rpc/server.go index 1330528a3c..70a7b159bd 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -21,6 +21,7 @@ import ( "fmt" "reflect" "runtime" + "strings" "sync" "sync/atomic" @@ -96,32 +97,30 @@ func (s *Server) RegisterName(name string, rcvr interface{}) error { return fmt.Errorf("%s is not exported", reflect.Indirect(rcvrVal).Type().Name()) } + methods, subscriptions := suitableCallbacks(rcvrVal, svc.typ) + // already a previous service register under given sname, merge methods/subscriptions if regsvc, present := s.services[name]; present { - methods, subscriptions := suitableCallbacks(rcvrVal, svc.typ) if len(methods) == 0 && len(subscriptions) == 0 { return fmt.Errorf("Service %T doesn't have any suitable methods/subscriptions to expose", rcvr) } - for _, m := range methods { regsvc.callbacks[formatName(m.method.Name)] = m } for _, s := range subscriptions { regsvc.subscriptions[formatName(s.method.Name)] = s } - return nil } svc.name = name - svc.callbacks, svc.subscriptions = suitableCallbacks(rcvrVal, svc.typ) + svc.callbacks, svc.subscriptions = methods, subscriptions if len(svc.callbacks) == 0 && len(svc.subscriptions) == 0 { return fmt.Errorf("Service %T doesn't have any suitable methods/subscriptions to expose", rcvr) } s.services[svc.name] = svc - return nil } @@ -303,7 +302,7 @@ func (s *Server) handle(ctx context.Context, codec ServerCodec, req *serverReque // active the subscription after the sub id was successfully sent to the client activateSub := func() { notifier, _ := NotifierFromContext(ctx) - notifier.activate(subid) + notifier.activate(subid, req.svcname) } return codec.CreateResponse(req.id, subid), activateSub @@ -383,7 +382,7 @@ func (s *Server) execBatch(ctx context.Context, codec ServerCodec, requests []*s codec.Close() } - // when request holds one of more subscribe requests this allows these subscriptions to be actived + // when request holds one of more subscribe requests this allows these subscriptions to be activated for _, c := range callbacks { c() } @@ -410,7 +409,7 @@ func (s *Server) readRequest(codec ServerCodec) ([]*serverRequest, bool, Error) continue } - if r.isPubSub && r.method == unsubscribeMethod { + if r.isPubSub && strings.HasSuffix(r.method, unsubscribeMethodSuffix) { requests[i] = &serverRequest{id: r.id, isUnsubscribe: true} argTypes := []reflect.Type{reflect.TypeOf("")} // expect subscription id as first arg if args, err := codec.ParseRequestArguments(argTypes, r.params); err == nil { @@ -439,7 +438,7 @@ func (s *Server) readRequest(codec ServerCodec) ([]*serverRequest, bool, Error) } } } else { - requests[i] = &serverRequest{id: r.id, err: &methodNotFoundError{subscribeMethod, r.method}} + requests[i] = &serverRequest{id: r.id, err: &methodNotFoundError{r.method, r.method}} } continue } diff --git a/rpc/subscription.go b/rpc/subscription.go index 9ab6af9e14..720e4dd06b 100644 --- a/rpc/subscription.go +++ b/rpc/subscription.go @@ -35,8 +35,9 @@ type ID string // a Subscription is created by a notifier and tight to that notifier. The client can use // this subscription to wait for an unsubscribe request for the client, see Err(). type Subscription struct { - ID ID - err chan error // closed on unsubscribe + ID ID + namespace string + err chan error // closed on unsubscribe } // Err returns a channel that is closed when the client send an unsubscribe request. @@ -78,7 +79,7 @@ func NotifierFromContext(ctx context.Context) (*Notifier, bool) { // are dropped until the subscription is marked as active. This is done // by the RPC server after the subscription ID is send to the client. func (n *Notifier) CreateSubscription() *Subscription { - s := &Subscription{NewID(), make(chan error)} + s := &Subscription{ID: NewID(), err: make(chan error)} n.subMu.Lock() n.inactive[s.ID] = s n.subMu.Unlock() @@ -91,9 +92,9 @@ func (n *Notifier) Notify(id ID, data interface{}) error { n.subMu.RLock() defer n.subMu.RUnlock() - _, active := n.active[id] + sub, active := n.active[id] if active { - notification := n.codec.CreateNotification(string(id), data) + notification := n.codec.CreateNotification(string(id), sub.namespace, data) if err := n.codec.Write(notification); err != nil { n.codec.Close() return err @@ -124,10 +125,11 @@ func (n *Notifier) unsubscribe(id ID) error { // notifications are dropped. This method is called by the RPC server after // the subscription ID was sent to client. This prevents notifications being // send to the client before the subscription ID is send to the client. -func (n *Notifier) activate(id ID) { +func (n *Notifier) activate(id ID, namespace string) { n.subMu.Lock() defer n.subMu.Unlock() if sub, found := n.inactive[id]; found { + sub.namespace = namespace n.active[id] = sub delete(n.inactive, id) } diff --git a/rpc/subscription_test.go b/rpc/subscription_test.go index 345b4e5f29..0ed15ddfe8 100644 --- a/rpc/subscription_test.go +++ b/rpc/subscription_test.go @@ -19,6 +19,7 @@ package rpc import ( "context" "encoding/json" + "fmt" "net" "sync" "testing" @@ -162,3 +163,162 @@ func TestNotifications(t *testing.T) { t.Error("unsubscribe callback not called after closing connection") } } + +func waitForMessages(t *testing.T, in *json.Decoder, successes chan<- jsonSuccessResponse, + failures chan<- jsonErrResponse, notifications chan<- jsonNotification) { + + // read and parse server messages + for { + var rmsg json.RawMessage + if err := in.Decode(&rmsg); err != nil { + return + } + + var responses []map[string]interface{} + if rmsg[0] == '[' { + if err := json.Unmarshal(rmsg, &responses); err != nil { + t.Fatalf("Received invalid message: %s", rmsg) + } + } else { + var msg map[string]interface{} + if err := json.Unmarshal(rmsg, &msg); err != nil { + t.Fatalf("Received invalid message: %s", rmsg) + } + responses = append(responses, msg) + } + + for _, msg := range responses { + // determine what kind of msg was received and broadcast + // it to over the corresponding channel + if _, found := msg["result"]; found { + successes <- jsonSuccessResponse{ + Version: msg["jsonrpc"].(string), + Id: msg["id"], + Result: msg["result"], + } + continue + } + if _, found := msg["error"]; found { + params := msg["params"].(map[string]interface{}) + failures <- jsonErrResponse{ + Version: msg["jsonrpc"].(string), + Id: msg["id"], + Error: jsonError{int(params["subscription"].(float64)), params["message"].(string), params["data"]}, + } + continue + } + if _, found := msg["params"]; found { + params := msg["params"].(map[string]interface{}) + notifications <- jsonNotification{ + Version: msg["jsonrpc"].(string), + Method: msg["method"].(string), + Params: jsonSubscription{params["subscription"].(string), params["result"]}, + } + continue + } + t.Fatalf("Received invalid message: %s", msg) + } + } +} + +// TestSubscriptionMultipleNamespaces ensures that subscriptions can exists +// for multiple different namespaces. +func TestSubscriptionMultipleNamespaces(t *testing.T) { + var ( + namespaces = []string{"eth", "shh", "bzz"} + server = NewServer() + service = NotificationTestService{} + clientConn, serverConn = net.Pipe() + + out = json.NewEncoder(clientConn) + in = json.NewDecoder(clientConn) + successes = make(chan jsonSuccessResponse) + failures = make(chan jsonErrResponse) + notifications = make(chan jsonNotification) + ) + + // setup and start server + for _, namespace := range namespaces { + if err := server.RegisterName(namespace, &service); err != nil { + t.Fatalf("unable to register test service %v", err) + } + } + + go server.ServeCodec(NewJSONCodec(serverConn), OptionMethodInvocation|OptionSubscriptions) + defer server.Stop() + + // wait for message and write them to the given channels + go waitForMessages(t, in, successes, failures, notifications) + + // create subscriptions one by one + n := 3 + for i, namespace := range namespaces { + request := map[string]interface{}{ + "id": i, + "method": fmt.Sprintf("%s_subscribe", namespace), + "version": "2.0", + "params": []interface{}{"someSubscription", n, i}, + } + + if err := out.Encode(&request); err != nil { + t.Fatalf("Could not create subscription: %v", err) + } + } + + // create all subscriptions in 1 batch + var requests []interface{} + for i, namespace := range namespaces { + requests = append(requests, map[string]interface{}{ + "id": i, + "method": fmt.Sprintf("%s_subscribe", namespace), + "version": "2.0", + "params": []interface{}{"someSubscription", n, i}, + }) + } + + if err := out.Encode(&requests); err != nil { + t.Fatalf("Could not create subscription in batch form: %v", err) + } + + timeout := time.After(30 * time.Second) + subids := make(map[string]string, 2*len(namespaces)) + count := make(map[string]int, 2*len(namespaces)) + + for { + done := true + for id, _ := range count { + if count, found := count[id]; !found || count < (2*n) { + done = false + } + } + + if done && len(count) == len(namespaces) { + break + } + + select { + case suc := <-successes: // subscription created + subids[namespaces[int(suc.Id.(float64))]] = suc.Result.(string) + case failure := <-failures: + t.Errorf("received error: %v", failure.Error) + case notification := <-notifications: + if cnt, found := count[notification.Params.Subscription]; found { + count[notification.Params.Subscription] = cnt + 1 + } else { + count[notification.Params.Subscription] = 1 + } + case <-timeout: + for _, namespace := range namespaces { + subid, found := subids[namespace] + if !found { + t.Errorf("Subscription for '%s' not created", namespace) + continue + } + if count, found := count[subid]; !found || count < n { + t.Errorf("Didn't receive all notifications (%d<%d) in time for namespace '%s'", count, n, namespace) + } + } + return + } + } +} diff --git a/rpc/types.go b/rpc/types.go index 466d56f5af..71e1032ab5 100644 --- a/rpc/types.go +++ b/rpc/types.go @@ -104,17 +104,17 @@ type ServerCodec interface { // Read next request ReadRequestHeaders() ([]rpcRequest, bool, Error) // Parse request argument to the given types - ParseRequestArguments([]reflect.Type, interface{}) ([]reflect.Value, Error) + ParseRequestArguments(argTypes []reflect.Type, params interface{}) ([]reflect.Value, Error) // Assemble success response, expects response id and payload - CreateResponse(interface{}, interface{}) interface{} + CreateResponse(id interface{}, reply interface{}) interface{} // Assemble error response, expects response id and error - CreateErrorResponse(interface{}, Error) interface{} + CreateErrorResponse(id interface{}, err Error) interface{} // Assemble error response with extra information about the error through info CreateErrorResponseWithInfo(id interface{}, err Error, info interface{}) interface{} // Create notification response - CreateNotification(string, interface{}) interface{} + CreateNotification(id, namespace string, event interface{}) interface{} // Write msg to client. - Write(interface{}) error + Write(msg interface{}) error // Close underlying data stream Close() // Closed when underlying connection is closed diff --git a/swarm/api/api.go b/swarm/api/api.go index 9abb84b855..dc76b056ea 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -25,12 +25,14 @@ import ( "sync" "bytes" - "github.com/expanse-org/go-expanse/common" - "github.com/expanse-org/go-expanse/log" - "github.com/expanse-org/go-expanse/swarm/storage" + "mime" "path/filepath" "time" + + "github.com/expanse-org/go-expanse/common" + "github.com/expanse-org/go-expanse/log" + "github.com/expanse-org/go-expanse/swarm/storage" ) var ( @@ -84,26 +86,32 @@ type ErrResolve error func (self *Api) Resolve(uri *URI) (storage.Key, error) { log.Trace(fmt.Sprintf("Resolving : %v", uri.Addr)) - var err error - if !uri.Immutable() { - if self.dns != nil { - resolved, err := self.dns.Resolve(uri.Addr) - if err == nil { - return resolved[:], nil - } - } else { - err = fmt.Errorf("no DNS to resolve name") + // if the URI is immutable, check if the address is a hash + isHash := hashMatcher.MatchString(uri.Addr) + if uri.Immutable() { + if !isHash { + return nil, fmt.Errorf("immutable address not a content hash: %q", uri.Addr) } + return common.Hex2Bytes(uri.Addr), nil } - if hashMatcher.MatchString(uri.Addr) { - return storage.Key(common.Hex2Bytes(uri.Addr)), nil - } - if err != nil { - return nil, fmt.Errorf("'%s' does not resolve: %v but is not a content hash", uri.Addr, err) - } - return nil, fmt.Errorf("'%s' is not a content hash", uri.Addr) -} + // if DNS is not configured, check if the address is a hash + if self.dns == nil { + if !isHash { + return nil, fmt.Errorf("no DNS to resolve name: %q", uri.Addr) + } + return common.Hex2Bytes(uri.Addr), nil + } + + // try and resolve the address + resolved, err := self.dns.Resolve(uri.Addr) + if err == nil { + return resolved[:], nil + } else if !isHash { + return nil, err + } + return common.Hex2Bytes(uri.Addr), nil +} // Put provides singleton manifest creation on top of dpa store func (self *Api) Put(content, contentType string) (storage.Key, error) { diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index 187d0091ba..8827f9f50c 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -17,6 +17,7 @@ package api import ( + "errors" "fmt" "io" "io/ioutil" @@ -117,3 +118,122 @@ func TestApiPut(t *testing.T) { checkResponse(t, resp, exp) }) } + +// testResolver implements the Resolver interface and either returns the given +// hash if it is set, or returns a "name not found" error +type testResolver struct { + hash *common.Hash +} + +func newTestResolver(addr string) *testResolver { + r := &testResolver{} + if addr != "" { + hash := common.HexToHash(addr) + r.hash = &hash + } + return r +} + +func (t *testResolver) Resolve(addr string) (common.Hash, error) { + if t.hash == nil { + return common.Hash{}, fmt.Errorf("DNS name not found: %q", addr) + } + return *t.hash, nil +} + +// TestAPIResolve tests resolving URIs which can either contain content hashes +// or ENS names +func TestAPIResolve(t *testing.T) { + ensAddr := "swarm.eth" + hashAddr := "1111111111111111111111111111111111111111111111111111111111111111" + resolvedAddr := "2222222222222222222222222222222222222222222222222222222222222222" + doesResolve := newTestResolver(resolvedAddr) + doesntResolve := newTestResolver("") + + type test struct { + desc string + dns Resolver + addr string + immutable bool + result string + expectErr error + } + + tests := []*test{ + { + desc: "DNS not configured, hash address, returns hash address", + dns: nil, + addr: hashAddr, + result: hashAddr, + }, + { + desc: "DNS not configured, ENS address, returns error", + dns: nil, + addr: ensAddr, + expectErr: errors.New(`no DNS to resolve name: "swarm.eth"`), + }, + { + desc: "DNS configured, hash address, hash resolves, returns resolved address", + dns: doesResolve, + addr: hashAddr, + result: resolvedAddr, + }, + { + desc: "DNS configured, immutable hash address, hash resolves, returns hash address", + dns: doesResolve, + addr: hashAddr, + immutable: true, + result: hashAddr, + }, + { + desc: "DNS configured, hash address, hash doesn't resolve, returns hash address", + dns: doesntResolve, + addr: hashAddr, + result: hashAddr, + }, + { + desc: "DNS configured, ENS address, name resolves, returns resolved address", + dns: doesResolve, + addr: ensAddr, + result: resolvedAddr, + }, + { + desc: "DNS configured, immutable ENS address, name resolves, returns error", + dns: doesResolve, + addr: ensAddr, + immutable: true, + expectErr: errors.New(`immutable address not a content hash: "swarm.eth"`), + }, + { + desc: "DNS configured, ENS address, name doesn't resolve, returns error", + dns: doesntResolve, + addr: ensAddr, + expectErr: errors.New(`DNS name not found: "swarm.eth"`), + }, + } + for _, x := range tests { + t.Run(x.desc, func(t *testing.T) { + api := &Api{dns: x.dns} + uri := &URI{Addr: x.addr, Scheme: "bzz"} + if x.immutable { + uri.Scheme = "bzzi" + } + res, err := api.Resolve(uri) + if err == nil { + if x.expectErr != nil { + t.Fatalf("expected error %q, got result %q", x.expectErr, res) + } + if res.String() != x.result { + t.Fatalf("expected result %q, got %q", x.result, res) + } + } else { + if x.expectErr == nil { + t.Fatalf("expected no error, got %q", err) + } + if err.Error() != x.expectErr.Error() { + t.Fatalf("expected error %q, got %q", x.expectErr, err) + } + } + }) + } +} diff --git a/swarm/api/config.go b/swarm/api/config.go index 50d0b46d9a..b2602e144b 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -32,7 +32,8 @@ import ( ) const ( - port = "8500" + DefaultHTTPListenAddr = "127.0.0.1" + DefaultHTTPPort = "8500" ) var ( @@ -48,12 +49,13 @@ type Config struct { *network.HiveParams Swap *swap.SwapParams *network.SyncParams - Path string - Port string - PublicKey string - BzzKey string - EnsRoot common.Address - NetworkId uint64 + Path string + ListenAddr string + Port string + PublicKey string + BzzKey string + EnsRoot common.Address + NetworkId uint64 } // config is agnostic to where private key is coming from @@ -76,7 +78,8 @@ func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, n HiveParams: network.NewHiveParams(dirpath), ChunkerParams: storage.NewChunkerParams(), StoreParams: storage.NewStoreParams(dirpath), - Port: port, + ListenAddr: DefaultHTTPListenAddr, + Port: DefaultHTTPPort, Path: dirpath, Swap: swap.DefaultSwapParams(contract, prvKey), PublicKey: pubkeyhex, diff --git a/swarm/api/config_test.go b/swarm/api/config_test.go index 4ca9d06266..76acbc3651 100644 --- a/swarm/api/config_test.go +++ b/swarm/api/config_test.go @@ -80,6 +80,7 @@ var ( false ], "Path": "TMPDIR", + "ListenAddr": "127.0.0.1", "Port": "8500", "PublicKey": "0x045f5cfd26692e48d0017d380349bcf50982488bc11b5145f3ddf88b24924299048450542d43527fbe29a5cb32f38d62755393ac002e6bfdd71b8d7ba725ecd7a3", "BzzKey": "0xe861964402c0b78e2d44098329b8545726f215afa737d803714a4338552fcb81", @@ -95,7 +96,10 @@ func TestConfigWriteRead(t *testing.T) { } defer os.RemoveAll(tmp) - prvkey := crypto.ToECDSA(common.Hex2Bytes(hexprvkey)) + prvkey, err := crypto.HexToECDSA(hexprvkey) + if err != nil { + t.Fatalf("failed to load private key: %v", err) + } orig, err := NewConfig(tmp, common.Address{}, prvkey, 323) if err != nil { t.Fatalf("expected no error, got %v", err) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index ef08b65dd5..aecaac7392 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -69,7 +69,6 @@ func StartHttpServer(api *api.Api, config *ServerConfig) { hdlr := c.Handler(NewServer(api)) go http.ListenAndServe(config.Addr, hdlr) - log.Info(fmt.Sprintf("Swarm HTTP proxy started on localhost:%s", config.Addr)) } func NewServer(api *api.Api) *Server { diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 06bce5c904..017753e5f8 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -106,9 +106,9 @@ func TestBzzrGetPath(t *testing.T) { } nonhashresponses := []string{ - "error resolving name: 'name' does not resolve: no DNS to resolve name but is not a content hash\n", - "error resolving nonhash: 'nonhash' is not a content hash\n", - "error resolving nonhash: 'nonhash' does not resolve: no DNS to resolve name but is not a content hash\n", + "error resolving name: no DNS to resolve name: \"name\"\n", + "error resolving nonhash: immutable address not a content hash: \"nonhash\"\n", + "error resolving nonhash: no DNS to resolve name: \"nonhash\"\n", } for i, url := range nonhashtests { diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index fc1724cef5..bbb194d800 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -237,12 +237,12 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) { } b := byte(entry.Path[0]) - if (self.entries[b] == nil) || (self.entries[b].Path == entry.Path) { + oldentry := self.entries[b] + if (oldentry == nil) || (oldentry.Path == entry.Path && oldentry.ContentType != ManifestType) { self.entries[b] = entry return } - oldentry := self.entries[b] cpl := 0 for (len(entry.Path) > cpl) && (len(oldentry.Path) > cpl) && (entry.Path[cpl] == oldentry.Path[cpl]) { cpl++ diff --git a/swarm/api/manifest_test.go b/swarm/api/manifest_test.go index 133adbd098..b4e49999f3 100644 --- a/swarm/api/manifest_test.go +++ b/swarm/api/manifest_test.go @@ -18,6 +18,8 @@ package api import ( // "encoding/json" + "bytes" + "encoding/json" "fmt" "io" "strings" @@ -78,3 +80,34 @@ func TestGetEntry(t *testing.T) { func TestDeleteEntry(t *testing.T) { } + +// TestAddFileWithManifestPath tests that adding an entry at a path which +// already exists as a manifest just adds the entry to the manifest rather +// than replacing the manifest with the entry +func TestAddFileWithManifestPath(t *testing.T) { + // create a manifest containing "ab" and "ac" + manifest, _ := json.Marshal(&Manifest{ + Entries: []ManifestEntry{ + {Path: "ab", Hash: "ab"}, + {Path: "ac", Hash: "ac"}, + }, + }) + reader := &storage.LazyTestSectionReader{ + SectionReader: io.NewSectionReader(bytes.NewReader(manifest), 0, int64(len(manifest))), + } + trie, err := readManifest(reader, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + checkEntry(t, "ab", "ab", trie) + checkEntry(t, "ac", "ac", trie) + + // now add path "a" and check we can still get "ab" and "ac" + entry := &manifestTrieEntry{} + entry.Path = "a" + entry.Hash = "a" + trie.addEntry(entry, nil) + checkEntry(t, "ab", "ab", trie) + checkEntry(t, "ac", "ac", trie) + checkEntry(t, "a", "a", trie) +} diff --git a/swarm/dev/.dockerignore b/swarm/dev/.dockerignore new file mode 100644 index 0000000000..f9e69b37f3 --- /dev/null +++ b/swarm/dev/.dockerignore @@ -0,0 +1,2 @@ +bin/* +cluster/* diff --git a/swarm/dev/.gitignore b/swarm/dev/.gitignore new file mode 100644 index 0000000000..f9e69b37f3 --- /dev/null +++ b/swarm/dev/.gitignore @@ -0,0 +1,2 @@ +bin/* +cluster/* diff --git a/swarm/dev/Dockerfile b/swarm/dev/Dockerfile new file mode 100644 index 0000000000..728bdab1fb --- /dev/null +++ b/swarm/dev/Dockerfile @@ -0,0 +1,42 @@ +FROM ubuntu:xenial + +# install build + test dependencies +RUN apt-get update && \ + apt-get install --yes --no-install-recommends \ + ca-certificates \ + curl \ + fuse \ + g++ \ + gcc \ + git \ + iproute2 \ + iputils-ping \ + less \ + libc6-dev \ + make \ + pkg-config \ + && \ + apt-get clean + +# install Go +ENV GO_VERSION 1.8.1 +RUN curl -fSLo golang.tar.gz "https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz" && \ + tar -xzf golang.tar.gz -C /usr/local && \ + rm golang.tar.gz +ENV GOPATH /go +ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH + +# install docker CLI +RUN curl -fSLo docker.tar.gz https://get.docker.com/builds/Linux/x86_64/docker-17.04.0-ce.tgz && \ + tar -xzf docker.tar.gz -C /usr/local/bin --strip-components=1 docker/docker && \ + rm docker.tar.gz + +# install jq +RUN curl -fSLo /usr/local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 && \ + chmod +x /usr/local/bin/jq + +# install govendor +RUN go get -u github.com/kardianos/govendor + +# add custom bashrc +ADD bashrc /root/.bashrc diff --git a/swarm/dev/Makefile b/swarm/dev/Makefile new file mode 100644 index 0000000000..44323f8908 --- /dev/null +++ b/swarm/dev/Makefile @@ -0,0 +1,14 @@ +.PHONY: build cluster test + +default: build + +build: + go build -o bin/swarm github.com/expanse-org/go-expanse/cmd/swarm + go build -o bin/geth github.com/expanse-org/go-expanse/cmd/geth + go build -o bin/bootnode github.com/expanse-org/go-expanse/cmd/bootnode + +cluster: build + scripts/boot-cluster.sh + +test: + go test -v github.com/expanse-org/go-expanse/swarm/... diff --git a/swarm/dev/README.md b/swarm/dev/README.md new file mode 100644 index 0000000000..81e3b53585 --- /dev/null +++ b/swarm/dev/README.md @@ -0,0 +1,20 @@ +Swarm development environment +============================= + +The Swarm development environment is a Linux bash shell which can be run in a +Docker container and provides a predictable build and test environment. + +### Start the Docker container + +Run the `run.sh` script to build the Docker image and run it, you will then be +at a bash prompt inside the `swarm/dev` directory. + +### Build binaries + +Run `make` to build the `swarm`, `geth` and `bootnode` binaries into the +`swarm/dev/bin` directory. + +### Boot a cluster + +Run `make cluster` to start a 3 node Swarm cluster, or run +`scripts/boot-cluster.sh --size N` to boot a cluster of size N. diff --git a/swarm/dev/bashrc b/swarm/dev/bashrc new file mode 100644 index 0000000000..e2843776ab --- /dev/null +++ b/swarm/dev/bashrc @@ -0,0 +1,21 @@ +export ROOT="${GOPATH}/src/github.com/expanse-org/go-expanse" +export PATH="${ROOT}/swarm/dev/bin:${PATH}" + +cd "${ROOT}/swarm/dev" + +cat <&2 <&2 + exit 1 + fi + name="$2" + shift 2 + ;; + -d | --docker-args) + if [[ -z "$2" ]]; then + echo "ERROR: --docker-args flag requires an argument" >&2 + exit 1 + fi + docker_args="$2" + shift 2 + ;; + *) + break + ;; + esac + done + + if [[ $# -ne 0 ]]; then + usage + echo "ERROR: invalid arguments" >&2 + exit 1 + fi +} + +build_image() { + docker build --tag "${name}" "${ROOT}/swarm/dev" +} + +run_image() { + exec docker run \ + --privileged \ + --interactive \ + --tty \ + --rm \ + --hostname "${name}" \ + --name "${name}" \ + --volume "${ROOT}:/go/src/github.com/expanse-org/go-expanse" \ + --volume "/var/run/docker.sock:/var/run/docker.sock" \ + ${docker_args} \ + "${name}" \ + /bin/bash +} + +main "$@" diff --git a/swarm/dev/scripts/boot-cluster.sh b/swarm/dev/scripts/boot-cluster.sh new file mode 100755 index 0000000000..073b095ad4 --- /dev/null +++ b/swarm/dev/scripts/boot-cluster.sh @@ -0,0 +1,288 @@ +#!/bin/bash +# +# A script to boot a dev swarm cluster on a Linux host (typically in a Docker +# container started with swarm/dev/run.sh). +# +# The cluster contains a bootnode, a geth node and multiple swarm nodes, with +# each node having its own data directory in a base directory passed with the +# --dir flag (default is swarm/dev/cluster). +# +# To avoid using different ports for each node and to make networking more +# realistic, each node gets its own network namespace with IPs assigned from +# the 192.168.33.0/24 subnet: +# +# bootnode: 192.168.33.2 +# geth: 192.168.33.3 +# swarm: 192.168.33.10{1,2,...,n} + +set -e + +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +source "${ROOT}/swarm/dev/scripts/util.sh" + +# DEFAULT_BASE_DIR is the default base directory to store node data +DEFAULT_BASE_DIR="${ROOT}/swarm/dev/cluster" + +# DEFAULT_CLUSTER_SIZE is the default swarm cluster size +DEFAULT_CLUSTER_SIZE=3 + +# Linux bridge configuration for connecting the node network namespaces +BRIDGE_NAME="swarmbr0" +BRIDGE_IP="192.168.33.1" + +# static bootnode configuration +BOOTNODE_IP="192.168.33.2" +BOOTNODE_PORT="30301" +BOOTNODE_KEY="32078f313bea771848db70745225c52c00981589ad6b5b49163f0f5ee852617d" +BOOTNODE_PUBKEY="760c4460e5336ac9bbd87952a3c7ec4363fc0a97bd31c86430806e287b437fd1b01abc6e1db640cf3106b520344af1d58b00b57823db3e1407cbc433e1b6d04d" +BOOTNODE_URL="enode://${BOOTNODE_PUBKEY}@${BOOTNODE_IP}:${BOOTNODE_PORT}" + +# static geth configuration +GETH_IP="192.168.33.3" +GETH_RPC_PORT="8545" +GETH_RPC_URL="http://${GETH_IP}:${GETH_RPC_PORT}" + +usage() { + cat >&2 < "${key_file}" + + local args=( + --addr "${BOOTNODE_IP}:${BOOTNODE_PORT}" + --nodekey "${key_file}" + --verbosity "6" + ) + + start_node "bootnode" "${BOOTNODE_IP}" "$(which bootnode)" ${args[@]} +} + +# start_geth_node starts a geth node with --datadir pointing at /geth +# and a single, unlocked account with password "geth" +start_geth_node() { + local dir="${base_dir}/geth" + mkdir -p "${dir}" + + local password="geth" + echo "${password}" > "${dir}/password" + + # create an account if necessary + if [[ ! -e "${dir}/keystore" ]]; then + info "creating geth account" + create_account "${dir}" "${password}" + fi + + # get the account address + local address="$(jq --raw-output '.address' ${dir}/keystore/*)" + if [[ -z "${address}" ]]; then + fail "failed to get geth account address" + fi + + local args=( + --datadir "${dir}" + --networkid "321" + --bootnodes "${BOOTNODE_URL}" + --unlock "${address}" + --password "${dir}/password" + --rpc + --rpcaddr "${GETH_IP}" + --rpcport "${GETH_RPC_PORT}" + --verbosity "6" + ) + + start_node "geth" "${GETH_IP}" "$(which geth)" ${args[@]} +} + +start_swarm_nodes() { + for i in $(seq 1 ${cluster_size}); do + start_swarm_node "${i}" + done +} + +# start_swarm_node starts a swarm node with a name like "swarmNN" (where NN is +# a zero-padded integer like "07"), --datadir pointing at / +# (e.g. /swarm07) and a single account with as the password +start_swarm_node() { + local num=$1 + local name="swarm$(printf '%02d' ${num})" + local ip="192.168.33.1$(printf '%02d' ${num})" + + local dir="${base_dir}/${name}" + mkdir -p "${dir}" + + local password="${name}" + echo "${password}" > "${dir}/password" + + # create an account if necessary + if [[ ! -e "${dir}/keystore" ]]; then + info "creating account for ${name}" + create_account "${dir}" "${password}" + fi + + # get the account address + local address="$(jq --raw-output '.address' ${dir}/keystore/*)" + if [[ -z "${address}" ]]; then + fail "failed to get swarm account address" + fi + + local args=( + --bootnodes "${BOOTNODE_URL}" + --datadir "${dir}" + --identity "${name}" + --ethapi "${GETH_RPC_URL}" + --bzznetworkid "321" + --bzzaccount "${address}" + --password "${dir}/password" + --verbosity "6" + ) + + start_node "${name}" "${ip}" "$(which swarm)" ${args[@]} +} + +# start_node runs the node command as a daemon in a network namespace +start_node() { + local name="$1" + local ip="$2" + local path="$3" + local cmd_args=${@:4} + + info "starting ${name} with IP ${ip}" + + create_node_network "${name}" "${ip}" + + # add a marker to the log file + cat >> "${log_dir}/${name}.log" <>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +Starting ${name} node - $(date) +>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + +EOF + + # run the command in the network namespace using start-stop-daemon to + # daemonise the process, sending all output to the log file + local daemon_args=( + --start + --background + --no-close + --make-pidfile + --pidfile "${pid_dir}/${name}.pid" + --exec "${path}" + ) + if ! ip netns exec "${name}" start-stop-daemon ${daemon_args[@]} -- $cmd_args &>> "${log_dir}/${name}.log"; then + fail "could not start ${name}, check ${log_dir}/${name}.log" + fi +} + +# create_node_network creates a network namespace and connects it to the Linux +# bridge using a veth pair +create_node_network() { + local name="$1" + local ip="$2" + + # create the namespace + ip netns add "${name}" + + # create the veth pair + local veth0="veth${name}0" + local veth1="veth${name}1" + ip link add name "${veth0}" type veth peer name "${veth1}" + + # add one end to the bridge + ip link set dev "${veth0}" master "${BRIDGE_NAME}" + ip link set dev "${veth0}" up + + # add the other end to the namespace, rename it eth0 and give it the ip + ip link set dev "${veth1}" netns "${name}" + ip netns exec "${name}" ip link set dev "${veth1}" name "eth0" + ip netns exec "${name}" ip link set dev "eth0" up + ip netns exec "${name}" ip address add "${ip}/24" dev "eth0" +} + +create_account() { + local dir=$1 + local password=$2 + + geth --datadir "${dir}" --password /dev/stdin account new <<< "${password}" +} + +main "$@" diff --git a/swarm/dev/scripts/random-uploads.sh b/swarm/dev/scripts/random-uploads.sh new file mode 100755 index 0000000000..db7887e3c0 --- /dev/null +++ b/swarm/dev/scripts/random-uploads.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# +# A script to upload random data to a swarm cluster. +# +# Example: +# +# random-uploads.sh --addr 192.168.33.101:8500 --size 40k --count 1000 + +set -e + +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +source "${ROOT}/swarm/dev/scripts/util.sh" + +DEFAULT_ADDR="localhost:8500" +DEFAULT_UPLOAD_SIZE="40k" +DEFAULT_UPLOAD_COUNT="1000" + +usage() { + cat >&2 </dev/null +} + +parse_args() { + while true; do + case "$1" in + -h | --help) + usage + exit 0 + ;; + -a | --addr) + if [[ -z "$2" ]]; then + fail "--addr flag requires an argument" + fi + addr="$2" + shift 2 + ;; + -s | --size) + if [[ -z "$2" ]]; then + fail "--size flag requires an argument" + fi + upload_size="$2" + shift 2 + ;; + -c | --count) + if [[ -z "$2" ]]; then + fail "--count flag requires an argument" + fi + upload_count="$2" + shift 2 + ;; + *) + break + ;; + esac + done + + if [[ $# -ne 0 ]]; then + usage + fail "ERROR: invalid arguments: $@" + fi +} + +main "$@" diff --git a/swarm/dev/scripts/stop-cluster.sh b/swarm/dev/scripts/stop-cluster.sh new file mode 100755 index 0000000000..89cb7b0c9a --- /dev/null +++ b/swarm/dev/scripts/stop-cluster.sh @@ -0,0 +1,98 @@ +#!/bin/bash +# +# A script to shutdown a dev swarm cluster. + +set -e + +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +source "${ROOT}/swarm/dev/scripts/util.sh" + +DEFAULT_BASE_DIR="${ROOT}/swarm/dev/cluster" + +usage() { + cat >&2 </dev/null; then + ip link delete dev "veth${name}0" + fi +} + +delete_network() { + if ip link show "swarmbr0" &>/dev/null; then + ip link delete dev "swarmbr0" + fi +} + +main "$@" diff --git a/swarm/dev/scripts/util.sh b/swarm/dev/scripts/util.sh new file mode 100644 index 0000000000..f17a12e420 --- /dev/null +++ b/swarm/dev/scripts/util.sh @@ -0,0 +1,53 @@ +# shared shell functions + +info() { + local msg="$@" + local timestamp="$(date +%H:%M:%S)" + say "===> ${timestamp} ${msg}" "green" +} + +warn() { + local msg="$@" + local timestamp=$(date +%H:%M:%S) + say "===> ${timestamp} WARN: ${msg}" "yellow" >&2 +} + +fail() { + local msg="$@" + say "ERROR: ${msg}" "red" >&2 + exit 1 +} + +# say prints the given message to STDOUT, using the optional color if +# STDOUT is a terminal. +# +# usage: +# +# say "foo" - prints "foo" +# say "bar" "red" - prints "bar" in red +# say "baz" "green" - prints "baz" in green +# say "qux" "red" | tee - prints "qux" with no colour +# +say() { + local msg=$1 + local color=$2 + + if [[ -n "${color}" ]] && [[ -t 1 ]]; then + case "${color}" in + red) + echo -e "\033[1;31m${msg}\033[0m" + ;; + green) + echo -e "\033[1;32m${msg}\033[0m" + ;; + yellow) + echo -e "\033[1;33m${msg}\033[0m" + ;; + *) + echo "${msg}" + ;; + esac + else + echo "${msg}" + fi +} diff --git a/swarm/swarm.go b/swarm/swarm.go index 79130565d8..41cb998b88 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -21,6 +21,7 @@ import ( "context" "crypto/ecdsa" "fmt" + "net" "github.com/expanse-org/go-expanse/accounts/abi/bind" "github.com/expanse-org/go-expanse/common" @@ -166,13 +167,13 @@ Start is called when the stack is started * TODO: start subservices like sword, swear, swarmdns */ // implements the node.Service interface -func (self *Swarm) Start(net *p2p.Server) error { +func (self *Swarm) Start(srv *p2p.Server) error { connectPeer := func(url string) error { node, err := discover.ParseNode(url) if err != nil { return fmt.Errorf("invalid node URL: %v", err) } - net.AddPeer(node) + srv.AddPeer(node) return nil } // set chequebook @@ -189,8 +190,8 @@ func (self *Swarm) Start(net *p2p.Server) error { log.Warn(fmt.Sprintf("Starting Swarm service")) self.hive.Start( - discover.PubkeyID(&net.PrivateKey.PublicKey), - func() string { return net.ListenAddr }, + discover.PubkeyID(&srv.PrivateKey.PublicKey), + func() string { return srv.ListenAddr }, connectPeer, ) log.Info(fmt.Sprintf("Swarm network started on bzz address: %v", self.hive.Addr())) @@ -200,17 +201,16 @@ func (self *Swarm) Start(net *p2p.Server) error { // start swarm http proxy server if self.config.Port != "" { - addr := ":" + self.config.Port + addr := net.JoinHostPort(self.config.ListenAddr, self.config.Port) go httpapi.StartHttpServer(self.api, &httpapi.ServerConfig{ Addr: addr, CorsString: self.corsString, }) - } + log.Info(fmt.Sprintf("Swarm http proxy started on %v", addr)) - log.Debug(fmt.Sprintf("Swarm http proxy started on port: %v", self.config.Port)) - - if self.corsString != "" { - log.Debug(fmt.Sprintf("Swarm http proxy started with corsdomain: %v", self.corsString)) + if self.corsString != "" { + log.Debug(fmt.Sprintf("Swarm http proxy started with corsdomain: %v", self.corsString)) + } } return nil diff --git a/tests/util.go b/tests/util.go index c9dc84c800..b507c3f5f1 100644 --- a/tests/util.go +++ b/tests/util.go @@ -18,7 +18,6 @@ package tests import ( "bytes" - "encoding/hex" "fmt" "math/big" "os" @@ -161,8 +160,8 @@ func NewEVMEnvironment(vmTest bool, chainConfig *params.ChainConfig, statedb *st origin := common.HexToAddress(tx["caller"]) if len(tx["secretKey"]) > 0 { - key, _ := hex.DecodeString(tx["secretKey"]) - origin = crypto.PubkeyToAddress(crypto.ToECDSA(key).PublicKey) + key, _ := crypto.HexToECDSA(tx["secretKey"]) + origin = crypto.PubkeyToAddress(key.PublicKey) } var to *common.Address diff --git a/trie/encoding.go b/trie/encoding.go index 2037118ddf..e96a786e40 100644 --- a/trie/encoding.go +++ b/trie/encoding.go @@ -16,49 +16,54 @@ package trie -func compactEncode(hexSlice []byte) []byte { +// Trie keys are dealt with in three distinct encodings: +// +// KEYBYTES encoding contains the actual key and nothing else. This encoding is the +// input to most API functions. +// +// HEX encoding contains one byte for each nibble of the key and an optional trailing +// 'terminator' byte of value 0x10 which indicates whether or not the node at the key +// contains a value. Hex key encoding is used for nodes loaded in memory because it's +// convenient to access. +// +// COMPACT encoding is defined by the Ethereum Yellow Paper (it's called "hex prefix +// encoding" there) and contains the bytes of the key and a flag. The high nibble of the +// first byte contains the flag; the lowest bit encoding the oddness of the length and +// the second-lowest encoding whether the node at the key is a value node. The low nibble +// of the first byte is zero in the case of an even number of nibbles and the first nibble +// in the case of an odd number. All remaining nibbles (now an even number) fit properly +// into the remaining bytes. Compact encoding is used for nodes stored on disk. + +func hexToCompact(hex []byte) []byte { terminator := byte(0) - if hexSlice[len(hexSlice)-1] == 16 { + if hasTerm(hex) { terminator = 1 - hexSlice = hexSlice[:len(hexSlice)-1] + hex = hex[:len(hex)-1] } - var ( - odd = byte(len(hexSlice) % 2) - buflen = len(hexSlice)/2 + 1 - bi, hi = 0, 0 // indices - hs = byte(0) // shift: flips between 0 and 4 - ) - if odd == 0 { - bi = 1 - hs = 4 - } - buf := make([]byte, buflen) - buf[0] = terminator<<5 | byte(odd)<<4 - for bi < len(buf) && hi < len(hexSlice) { - buf[bi] |= hexSlice[hi] << hs - if hs == 0 { - bi++ - } - hi, hs = hi+1, hs^(1<<2) + buf := make([]byte, len(hex)/2+1) + buf[0] = terminator << 5 // the flag byte + if len(hex)&1 == 1 { + buf[0] |= 1 << 4 // odd flag + buf[0] |= hex[0] // first nibble is contained in the first byte + hex = hex[1:] } + decodeNibbles(hex, buf[1:]) return buf } -func compactDecode(str []byte) []byte { - base := compactHexDecode(str) +func compactToHex(compact []byte) []byte { + base := keybytesToHex(compact) base = base[:len(base)-1] + // apply terminator flag if base[0] >= 2 { base = append(base, 16) } - if base[0]%2 == 1 { - base = base[1:] - } else { - base = base[2:] - } - return base + // apply odd flag + chop := 2 - base[0]&1 + return base[chop:] } -func compactHexDecode(str []byte) []byte { +func keybytesToHex(str []byte) []byte { l := len(str)*2 + 1 var nibbles = make([]byte, l) for i, b := range str { @@ -69,35 +74,24 @@ func compactHexDecode(str []byte) []byte { return nibbles } -// compactHexEncode encodes a series of nibbles into a byte array -func compactHexEncode(nibbles []byte) []byte { - nl := len(nibbles) - if nl == 0 { - return nil +// hexToKeybytes turns hex nibbles into key bytes. +// This can only be used for keys of even length. +func hexToKeybytes(hex []byte) []byte { + if hasTerm(hex) { + hex = hex[:len(hex)-1] } - if nibbles[nl-1] == 16 { - nl-- + if len(hex)&1 != 0 { + panic("can't convert hex key of odd length") } - l := (nl + 1) / 2 - var str = make([]byte, l) - for i := range str { - b := nibbles[i*2] * 16 - if nl > i*2 { - b += nibbles[i*2+1] - } - str[i] = b - } - return str + key := make([]byte, (len(hex)+1)/2) + decodeNibbles(hex, key) + return key } -func decodeCompact(key []byte) []byte { - l := len(key) / 2 - var res = make([]byte, l) - for i := 0; i < l; i++ { - v1, v0 := key[2*i], key[2*i+1] - res[i] = v1*16 + v0 +func decodeNibbles(nibbles []byte, bytes []byte) { + for bi, ni := 0, 0; ni < len(nibbles); bi, ni = bi+1, ni+2 { + bytes[bi] = nibbles[ni]<<4 | nibbles[ni+1] } - return res } // prefixLen returns the length of the common prefix of a and b. @@ -114,15 +108,7 @@ func prefixLen(a, b []byte) int { return i } +// hasTerm returns whether a hex key has the terminator flag. func hasTerm(s []byte) bool { - return s[len(s)-1] == 16 -} - -func remTerm(s []byte) []byte { - if hasTerm(s) { - b := make([]byte, len(s)-1) - copy(b, s) - return b - } - return s + return len(s) > 0 && s[len(s)-1] == 16 } diff --git a/trie/encoding_test.go b/trie/encoding_test.go index 2f125ef2f8..97d8da1361 100644 --- a/trie/encoding_test.go +++ b/trie/encoding_test.go @@ -17,113 +17,88 @@ package trie import ( - "encoding/hex" + "bytes" "testing" - - checker "gopkg.in/check.v1" ) -func TestEncoding(t *testing.T) { checker.TestingT(t) } - -type TrieEncodingSuite struct{} - -var _ = checker.Suite(&TrieEncodingSuite{}) - -func (s *TrieEncodingSuite) TestCompactEncode(c *checker.C) { - // even compact encode - test1 := []byte{1, 2, 3, 4, 5} - res1 := compactEncode(test1) - c.Assert(res1, checker.DeepEquals, []byte("\x11\x23\x45")) - - // odd compact encode - test2 := []byte{0, 1, 2, 3, 4, 5} - res2 := compactEncode(test2) - c.Assert(res2, checker.DeepEquals, []byte("\x00\x01\x23\x45")) - - //odd terminated compact encode - test3 := []byte{0, 15, 1, 12, 11, 8 /*term*/, 16} - res3 := compactEncode(test3) - c.Assert(res3, checker.DeepEquals, []byte("\x20\x0f\x1c\xb8")) - - // even terminated compact encode - test4 := []byte{15, 1, 12, 11, 8 /*term*/, 16} - res4 := compactEncode(test4) - c.Assert(res4, checker.DeepEquals, []byte("\x3f\x1c\xb8")) -} - -func (s *TrieEncodingSuite) TestCompactHexDecode(c *checker.C) { - exp := []byte{7, 6, 6, 5, 7, 2, 6, 2, 16} - res := compactHexDecode([]byte("verb")) - c.Assert(res, checker.DeepEquals, exp) -} - -func (s *TrieEncodingSuite) TestCompactHexEncode(c *checker.C) { - exp := []byte("verb") - res := compactHexEncode([]byte{7, 6, 6, 5, 7, 2, 6, 2, 16}) - c.Assert(res, checker.DeepEquals, exp) -} - -func (s *TrieEncodingSuite) TestCompactDecode(c *checker.C) { - // odd compact decode - exp := []byte{1, 2, 3, 4, 5} - res := compactDecode([]byte("\x11\x23\x45")) - c.Assert(res, checker.DeepEquals, exp) - - // even compact decode - exp = []byte{0, 1, 2, 3, 4, 5} - res = compactDecode([]byte("\x00\x01\x23\x45")) - c.Assert(res, checker.DeepEquals, exp) - - // even terminated compact decode - exp = []byte{0, 15, 1, 12, 11, 8 /*term*/, 16} - res = compactDecode([]byte("\x20\x0f\x1c\xb8")) - c.Assert(res, checker.DeepEquals, exp) - - // even terminated compact decode - exp = []byte{15, 1, 12, 11, 8 /*term*/, 16} - res = compactDecode([]byte("\x3f\x1c\xb8")) - c.Assert(res, checker.DeepEquals, exp) -} - -func (s *TrieEncodingSuite) TestDecodeCompact(c *checker.C) { - exp, _ := hex.DecodeString("012345") - res := decodeCompact([]byte{0, 1, 2, 3, 4, 5}) - c.Assert(res, checker.DeepEquals, exp) - - exp, _ = hex.DecodeString("012345") - res = decodeCompact([]byte{0, 1, 2, 3, 4, 5, 16}) - c.Assert(res, checker.DeepEquals, exp) - - exp, _ = hex.DecodeString("abcdef") - res = decodeCompact([]byte{10, 11, 12, 13, 14, 15}) - c.Assert(res, checker.DeepEquals, exp) -} - -func BenchmarkCompactEncode(b *testing.B) { - - testBytes := []byte{0, 15, 1, 12, 11, 8 /*term*/, 16} - for i := 0; i < b.N; i++ { - compactEncode(testBytes) +func TestHexCompact(t *testing.T) { + tests := []struct{ hex, compact []byte }{ + // empty keys, with and without terminator. + {hex: []byte{}, compact: []byte{0x00}}, + {hex: []byte{16}, compact: []byte{0x20}}, + // odd length, no terminator + {hex: []byte{1, 2, 3, 4, 5}, compact: []byte{0x11, 0x23, 0x45}}, + // even length, no terminator + {hex: []byte{0, 1, 2, 3, 4, 5}, compact: []byte{0x00, 0x01, 0x23, 0x45}}, + // odd length, terminator + {hex: []byte{15, 1, 12, 11, 8, 16 /*term*/}, compact: []byte{0x3f, 0x1c, 0xb8}}, + // even length, terminator + {hex: []byte{0, 15, 1, 12, 11, 8, 16 /*term*/}, compact: []byte{0x20, 0x0f, 0x1c, 0xb8}}, + } + for _, test := range tests { + if c := hexToCompact(test.hex); !bytes.Equal(c, test.compact) { + t.Errorf("hexToCompact(%x) -> %x, want %x", test.hex, c, test.compact) + } + if h := compactToHex(test.compact); !bytes.Equal(h, test.hex) { + t.Errorf("compactToHex(%x) -> %x, want %x", test.compact, h, test.hex) + } } } -func BenchmarkCompactDecode(b *testing.B) { - testBytes := []byte{0, 15, 1, 12, 11, 8 /*term*/, 16} - for i := 0; i < b.N; i++ { - compactDecode(testBytes) +func TestHexKeybytes(t *testing.T) { + tests := []struct{ key, hexIn, hexOut []byte }{ + {key: []byte{}, hexIn: []byte{16}, hexOut: []byte{16}}, + {key: []byte{}, hexIn: []byte{}, hexOut: []byte{16}}, + { + key: []byte{0x12, 0x34, 0x56}, + hexIn: []byte{1, 2, 3, 4, 5, 6, 16}, + hexOut: []byte{1, 2, 3, 4, 5, 6, 16}, + }, + { + key: []byte{0x12, 0x34, 0x5}, + hexIn: []byte{1, 2, 3, 4, 0, 5, 16}, + hexOut: []byte{1, 2, 3, 4, 0, 5, 16}, + }, + { + key: []byte{0x12, 0x34, 0x56}, + hexIn: []byte{1, 2, 3, 4, 5, 6}, + hexOut: []byte{1, 2, 3, 4, 5, 6, 16}, + }, + } + for _, test := range tests { + if h := keybytesToHex(test.key); !bytes.Equal(h, test.hexOut) { + t.Errorf("keybytesToHex(%x) -> %x, want %x", test.key, h, test.hexOut) + } + if k := hexToKeybytes(test.hexIn); !bytes.Equal(k, test.key) { + t.Errorf("hexToKeybytes(%x) -> %x, want %x", test.hexIn, k, test.key) + } } } -func BenchmarkCompactHexDecode(b *testing.B) { +func BenchmarkHexToCompact(b *testing.B) { + testBytes := []byte{0, 15, 1, 12, 11, 8, 16 /*term*/} + for i := 0; i < b.N; i++ { + hexToCompact(testBytes) + } +} + +func BenchmarkCompactToHex(b *testing.B) { + testBytes := []byte{0, 15, 1, 12, 11, 8, 16 /*term*/} + for i := 0; i < b.N; i++ { + compactToHex(testBytes) + } +} + +func BenchmarkKeybytesToHex(b *testing.B) { testBytes := []byte{7, 6, 6, 5, 7, 2, 6, 2, 16} for i := 0; i < b.N; i++ { - compactHexDecode(testBytes) + keybytesToHex(testBytes) } } -func BenchmarkDecodeCompact(b *testing.B) { +func BenchmarkHexToKeybytes(b *testing.B) { testBytes := []byte{7, 6, 6, 5, 7, 2, 6, 2, 16} for i := 0; i < b.N; i++ { - decodeCompact(testBytes) + hexToKeybytes(testBytes) } } diff --git a/trie/errors.go b/trie/errors.go index 857c8def3a..496b0a0e62 100644 --- a/trie/errors.go +++ b/trie/errors.go @@ -30,10 +30,6 @@ import ( // // RootHash is the original root of the trie that contains the node // -// Key is a binary-encoded key that contains the prefix that leads to the first -// missing node and optionally a suffix that hints on which further nodes should -// also be retrieved -// // PrefixLen is the nibble length of the key prefix that leads from the root to // the missing node // @@ -42,7 +38,6 @@ import ( // such hints in the error message) type MissingNodeError struct { RootHash, NodeHash common.Hash - Key []byte PrefixLen, SuffixLen int } diff --git a/trie/hasher.go b/trie/hasher.go index a66fc6143e..9d582b7d0d 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -105,7 +105,7 @@ func (h *hasher) hashChildren(original node, db DatabaseWriter) (node, node, err case *shortNode: // Hash the short node's child, caching the newly hashed subtree collapsed, cached := n.copy(), n.copy() - collapsed.Key = compactEncode(n.Key) + collapsed.Key = hexToCompact(n.Key) cached.Key = common.CopyBytes(n.Key) if _, ok := n.Val.(valueNode); !ok { diff --git a/trie/iterator.go b/trie/iterator.go index 20bf6b7a1c..9298d5a60f 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -19,9 +19,13 @@ package trie import ( "bytes" "container/heap" + "errors" + "github.com/expanse-org/go-expanse/common" ) +var iteratorEnd = errors.New("end of iteration") + // Iterator is a key-value trie iterator that traverses a Trie. type Iterator struct { nodeIt NodeIterator @@ -30,15 +34,8 @@ type Iterator struct { Value []byte // Current data value on which the iterator is positioned on } -// NewIterator creates a new key-value iterator. -func NewIterator(trie *Trie) *Iterator { - return &Iterator{ - nodeIt: NewNodeIterator(trie), - } -} - -// FromNodeIterator creates a new key-value iterator from a node iterator -func NewIteratorFromNodeIterator(it NodeIterator) *Iterator { +// NewIterator creates a new key-value iterator from a node iterator +func NewIterator(it NodeIterator) *Iterator { return &Iterator{ nodeIt: it, } @@ -48,7 +45,7 @@ func NewIteratorFromNodeIterator(it NodeIterator) *Iterator { func (it *Iterator) Next() bool { for it.nodeIt.Next(true) { if it.nodeIt.Leaf() { - it.Key = decodeCompact(it.nodeIt.Path()) + it.Key = hexToKeybytes(it.nodeIt.Path()) it.Value = it.nodeIt.LeafBlob() return true } @@ -85,25 +82,24 @@ type nodeIteratorState struct { hash common.Hash // Hash of the node being iterated (nil if not standalone) node node // Trie node being iterated parent common.Hash // Hash of the first full ancestor node (nil if current is the root) - child int // Child to be processed next + index int // Child to be processed next pathlen int // Length of the path to this node } type nodeIterator struct { trie *Trie // Trie being iterated stack []*nodeIteratorState // Hierarchy of trie nodes persisting the iteration state - - err error // Failure set in case of an internal error in the iterator - - path []byte // Path to the current node + err error // Failure set in case of an internal error in the iterator + path []byte // Path to the current node } -// NewNodeIterator creates an post-order trie iterator. -func NewNodeIterator(trie *Trie) NodeIterator { +func newNodeIterator(trie *Trie, start []byte) NodeIterator { if trie.Hash() == emptyState { return new(nodeIterator) } - return &nodeIterator{trie: trie} + it := &nodeIterator{trie: trie} + it.seek(start) + return it } // Hash returns the hash of the current node @@ -153,6 +149,9 @@ func (it *nodeIterator) Path() []byte { // Error returns the error set in case of an internal error in the iterator func (it *nodeIterator) Error() error { + if it.err == iteratorEnd { + return nil + } return it.err } @@ -161,47 +160,54 @@ func (it *nodeIterator) Error() error { // sets the Error field to the encountered failure. If `descend` is false, // skips iterating over any subnodes of the current node. func (it *nodeIterator) Next(descend bool) bool { - // If the iterator failed previously, don't do anything if it.err != nil { return false } // Otherwise step forward with the iterator and report any errors - if err := it.step(descend); err != nil { + state, parentIndex, path, err := it.peek(descend) + if err != nil { it.err = err return false } - return it.trie != nil + it.push(state, parentIndex, path) + return true } -// step moves the iterator to the next node of the trie. -func (it *nodeIterator) step(descend bool) error { - if it.trie == nil { - // Abort if we reached the end of the iteration - return nil +func (it *nodeIterator) seek(prefix []byte) { + // The path we're looking for is the hex encoded key without terminator. + key := keybytesToHex(prefix) + key = key[:len(key)-1] + // Move forward until we're just before the closest match to key. + for { + state, parentIndex, path, err := it.peek(bytes.HasPrefix(key, it.path)) + if err != nil || bytes.Compare(path, key) >= 0 { + it.err = err + return + } + it.push(state, parentIndex, path) } +} + +// peek creates the next state of the iterator. +func (it *nodeIterator) peek(descend bool) (*nodeIteratorState, *int, []byte, error) { if len(it.stack) == 0 { // Initialize the iterator if we've just started. root := it.trie.Hash() - state := &nodeIteratorState{node: it.trie.root, child: -1} + state := &nodeIteratorState{node: it.trie.root, index: -1} if root != emptyRoot { state.hash = root } - it.stack = append(it.stack, state) - return nil + return state, nil, nil, nil } - if !descend { // If we're skipping children, pop the current node first - it.path = it.path[:it.stack[len(it.stack)-1].pathlen] - it.stack = it.stack[:len(it.stack)-1] + it.pop() } // Continue iteration to the next child -outer: for { if len(it.stack) == 0 { - it.trie = nil - return nil + return nil, nil, nil, iteratorEnd } parent := it.stack[len(it.stack)-1] ancestor := parent.hash @@ -209,63 +215,76 @@ outer: ancestor = parent.parent } if node, ok := parent.node.(*fullNode); ok { - // Full node, iterate over children - for parent.child++; parent.child < len(node.Children); parent.child++ { - child := node.Children[parent.child] + // Full node, move to the first non-nil child. + for i := parent.index + 1; i < len(node.Children); i++ { + child := node.Children[i] if child != nil { hash, _ := child.cache() - it.stack = append(it.stack, &nodeIteratorState{ + state := &nodeIteratorState{ hash: common.BytesToHash(hash), node: child, parent: ancestor, - child: -1, + index: -1, pathlen: len(it.path), - }) - it.path = append(it.path, byte(parent.child)) - break outer + } + path := append(it.path, byte(i)) + parent.index = i - 1 + return state, &parent.index, path, nil } } } else if node, ok := parent.node.(*shortNode); ok { // Short node, return the pointer singleton child - if parent.child < 0 { - parent.child++ + if parent.index < 0 { hash, _ := node.Val.cache() - it.stack = append(it.stack, &nodeIteratorState{ + state := &nodeIteratorState{ hash: common.BytesToHash(hash), node: node.Val, parent: ancestor, - child: -1, + index: -1, pathlen: len(it.path), - }) - if hasTerm(node.Key) { - it.path = append(it.path, node.Key[:len(node.Key)-1]...) - } else { - it.path = append(it.path, node.Key...) } - break + var path []byte + if hasTerm(node.Key) { + path = append(it.path, node.Key[:len(node.Key)-1]...) + } else { + path = append(it.path, node.Key...) + } + return state, &parent.index, path, nil } } else if hash, ok := parent.node.(hashNode); ok { // Hash node, resolve the hash child from the database - if parent.child < 0 { - parent.child++ + if parent.index < 0 { node, err := it.trie.resolveHash(hash, nil, nil) if err != nil { - return err + return it.stack[len(it.stack)-1], &parent.index, it.path, err } - it.stack = append(it.stack, &nodeIteratorState{ + state := &nodeIteratorState{ hash: common.BytesToHash(hash), node: node, parent: ancestor, - child: -1, + index: -1, pathlen: len(it.path), - }) - break + } + return state, &parent.index, it.path, nil } } - it.path = it.path[:parent.pathlen] - it.stack = it.stack[:len(it.stack)-1] + // No more child nodes, move back up. + it.pop() } - return nil +} + +func (it *nodeIterator) push(state *nodeIteratorState, parentIndex *int, path []byte) { + it.path = path + it.stack = append(it.stack, state) + if parentIndex != nil { + *parentIndex += 1 + } +} + +func (it *nodeIterator) pop() { + parent := it.stack[len(it.stack)-1] + it.path = it.path[:parent.pathlen] + it.stack = it.stack[:len(it.stack)-1] } func compareNodes(a, b NodeIterator) int { diff --git a/trie/iterator_test.go b/trie/iterator_test.go index c3139ff953..c4ef7e4aef 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -17,6 +17,8 @@ package trie import ( + "bytes" + "fmt" "testing" "github.com/expanse-org/go-expanse/common" @@ -42,7 +44,7 @@ func TestIterator(t *testing.T) { trie.Commit() found := make(map[string]string) - it := NewIterator(trie) + it := NewIterator(trie.NodeIterator(nil)) for it.Next() { found[string(it.Key)] = string(it.Value) } @@ -72,7 +74,7 @@ func TestIteratorLargeData(t *testing.T) { vals[string(value2.k)] = value2 } - it := NewIterator(trie) + it := NewIterator(trie.NodeIterator(nil)) for it.Next() { vals[string(it.Key)].t = true } @@ -99,7 +101,7 @@ func TestNodeIteratorCoverage(t *testing.T) { // Gather all the node hashes found by the iterator hashes := make(map[common.Hash]struct{}) - for it := NewNodeIterator(trie); it.Next(true); { + for it := trie.NodeIterator(nil); it.Next(true); { if it.Hash() != (common.Hash{}) { hashes[it.Hash()] = struct{}{} } @@ -117,18 +119,20 @@ func TestNodeIteratorCoverage(t *testing.T) { } } -var testdata1 = []struct{ k, v string }{ - {"bar", "b"}, +type kvs struct{ k, v string } + +var testdata1 = []kvs{ {"barb", "ba"}, - {"bars", "bb"}, {"bard", "bc"}, + {"bars", "bb"}, + {"bar", "b"}, {"fab", "z"}, - {"foo", "a"}, {"food", "ab"}, {"foos", "aa"}, + {"foo", "a"}, } -var testdata2 = []struct{ k, v string }{ +var testdata2 = []kvs{ {"aardvark", "c"}, {"bar", "b"}, {"barb", "bd"}, @@ -140,6 +144,47 @@ var testdata2 = []struct{ k, v string }{ {"jars", "d"}, } +func TestIteratorSeek(t *testing.T) { + trie := newEmpty() + for _, val := range testdata1 { + trie.Update([]byte(val.k), []byte(val.v)) + } + + // Seek to the middle. + it := NewIterator(trie.NodeIterator([]byte("fab"))) + if err := checkIteratorOrder(testdata1[4:], it); err != nil { + t.Fatal(err) + } + + // Seek to a non-existent key. + it = NewIterator(trie.NodeIterator([]byte("barc"))) + if err := checkIteratorOrder(testdata1[1:], it); err != nil { + t.Fatal(err) + } + + // Seek beyond the end. + it = NewIterator(trie.NodeIterator([]byte("z"))) + if err := checkIteratorOrder(nil, it); err != nil { + t.Fatal(err) + } +} + +func checkIteratorOrder(want []kvs, it *Iterator) error { + for it.Next() { + if len(want) == 0 { + return fmt.Errorf("didn't expect any more values, got key %q", it.Key) + } + if !bytes.Equal(it.Key, []byte(want[0].k)) { + return fmt.Errorf("wrong key: got %q, want %q", it.Key, want[0].k) + } + want = want[1:] + } + if len(want) > 0 { + return fmt.Errorf("iterator ended early, want key %q", want[0]) + } + return nil +} + func TestDifferenceIterator(t *testing.T) { triea := newEmpty() for _, val := range testdata1 { @@ -154,8 +199,8 @@ func TestDifferenceIterator(t *testing.T) { trieb.Commit() found := make(map[string]string) - di, _ := NewDifferenceIterator(NewNodeIterator(triea), NewNodeIterator(trieb)) - it := NewIteratorFromNodeIterator(di) + di, _ := NewDifferenceIterator(triea.NodeIterator(nil), trieb.NodeIterator(nil)) + it := NewIterator(di) for it.Next() { found[string(it.Key)] = string(it.Value) } @@ -189,8 +234,8 @@ func TestUnionIterator(t *testing.T) { } trieb.Commit() - di, _ := NewUnionIterator([]NodeIterator{NewNodeIterator(triea), NewNodeIterator(trieb)}) - it := NewIteratorFromNodeIterator(di) + di, _ := NewUnionIterator([]NodeIterator{triea.NodeIterator(nil), trieb.NodeIterator(nil)}) + it := NewIterator(di) all := []struct{ k, v string }{ {"aardvark", "c"}, diff --git a/trie/node.go b/trie/node.go index 682469caa4..4be046507b 100644 --- a/trie/node.go +++ b/trie/node.go @@ -139,8 +139,8 @@ func decodeShort(hash, buf, elems []byte, cachegen uint16) (node, error) { return nil, err } flag := nodeFlag{hash: hash, gen: cachegen} - key := compactDecode(kbuf) - if key[len(key)-1] == 16 { + key := compactToHex(kbuf) + if hasTerm(key) { // value node val, _, err := rlp.SplitString(rest) if err != nil { diff --git a/trie/proof.go b/trie/proof.go index dfbd23db67..be8fbc2ae3 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -38,7 +38,7 @@ import ( // absence of the key. func (t *Trie) Prove(key []byte) []rlp.RawValue { // Collect all nodes on the path to key. - key = compactHexDecode(key) + key = keybytesToHex(key) nodes := []node{} tn := t.root for len(key) > 0 && tn != nil { @@ -89,7 +89,7 @@ func (t *Trie) Prove(key []byte) []rlp.RawValue { // returns an error if the proof contains invalid trie nodes or the // wrong value. func VerifyProof(rootHash common.Hash, key []byte, proof []rlp.RawValue) (value []byte, err error) { - key = compactHexDecode(key) + key = keybytesToHex(key) sha := sha3.NewKeccak256() wantHash := rootHash.Bytes() for i, buf := range proof { diff --git a/trie/secure_trie.go b/trie/secure_trie.go index c2742a1188..87f738f896 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -156,12 +156,10 @@ func (t *SecureTrie) Root() []byte { return t.trie.Root() } -func (t *SecureTrie) Iterator() *Iterator { - return t.trie.Iterator() -} - -func (t *SecureTrie) NodeIterator() NodeIterator { - return NewNodeIterator(&t.trie) +// NodeIterator returns an iterator that returns nodes of the underlying trie. Iteration +// starts at the key after the given start key. +func (t *SecureTrie) NodeIterator(start []byte) NodeIterator { + return t.trie.NodeIterator(start) } // CommitTo writes all nodes and the secure hash pre-images to the given database. diff --git a/trie/sync_test.go b/trie/sync_test.go index 23c6888cab..1057f921ae 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -80,7 +80,7 @@ func checkTrieConsistency(db Database, root common.Hash) error { if err != nil { return nil // // Consider a non existent state consistent } - it := NewNodeIterator(trie) + it := trie.NodeIterator(nil) for it.Next(true) { } return it.Error() diff --git a/trie/trie.go b/trie/trie.go index 26f98b615e..a435e25547 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -125,9 +125,10 @@ func New(root common.Hash, db Database) (*Trie, error) { return trie, nil } -// Iterator returns an iterator over all mappings in the trie. -func (t *Trie) Iterator() *Iterator { - return NewIterator(t) +// NodeIterator returns an iterator that returns nodes of the trie. Iteration starts at +// the key after the given start key. +func (t *Trie) NodeIterator(start []byte) NodeIterator { + return newNodeIterator(t, start) } // Get returns the value for key stored in the trie. @@ -144,7 +145,7 @@ func (t *Trie) Get(key []byte) []byte { // The value bytes must not be modified by the caller. // If a node was not found in the database, a MissingNodeError is returned. func (t *Trie) TryGet(key []byte) ([]byte, error) { - key = compactHexDecode(key) + key = keybytesToHex(key) value, newroot, didResolve, err := t.tryGet(t.root, key, 0) if err == nil && didResolve { t.root = newroot @@ -211,7 +212,7 @@ func (t *Trie) Update(key, value []byte) { // // If a node was not found in the database, a MissingNodeError is returned. func (t *Trie) TryUpdate(key, value []byte) error { - k := compactHexDecode(key) + k := keybytesToHex(key) if len(value) != 0 { _, n, err := t.insert(t.root, nil, k, valueNode(value)) if err != nil { @@ -307,7 +308,7 @@ func (t *Trie) Delete(key []byte) { // TryDelete removes any existing value for key from the trie. // If a node was not found in the database, a MissingNodeError is returned. func (t *Trie) TryDelete(key []byte) error { - k := compactHexDecode(key) + k := keybytesToHex(key) _, n, err := t.delete(t.root, nil, k) if err != nil { return err @@ -450,7 +451,6 @@ func (t *Trie) resolveHash(n hashNode, prefix, suffix []byte) (node, error) { return nil, &MissingNodeError{ RootHash: t.originalRoot, NodeHash: common.BytesToHash(n), - Key: compactHexEncode(append(prefix, suffix...)), PrefixLen: len(prefix), SuffixLen: len(suffix), } diff --git a/trie/trie_test.go b/trie/trie_test.go index d07201bb9a..857d342a2b 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -439,7 +439,7 @@ func runRandTest(rt randTest) bool { tr = newtr case opItercheckhash: checktr, _ := New(common.Hash{}, nil) - it := tr.Iterator() + it := NewIterator(tr.NodeIterator(nil)) for it.Next() { checktr.Update(it.Key, it.Value) } diff --git a/vendor/golang.org/x/crypto/curve25519/cswap_amd64.s b/vendor/golang.org/x/crypto/curve25519/cswap_amd64.s index 45484d1b59..cd793a5b5f 100644 --- a/vendor/golang.org/x/crypto/curve25519/cswap_amd64.s +++ b/vendor/golang.org/x/crypto/curve25519/cswap_amd64.s @@ -2,87 +2,64 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// This code was translated into a form compatible with 6a from the public -// domain sources in SUPERCOP: http://bench.cr.yp.to/supercop.html - // +build amd64,!gccgo,!appengine -// func cswap(inout *[5]uint64, v uint64) +// func cswap(inout *[4][5]uint64, v uint64) TEXT ·cswap(SB),7,$0 MOVQ inout+0(FP),DI MOVQ v+8(FP),SI - CMPQ SI,$1 - MOVQ 0(DI),SI - MOVQ 80(DI),DX - MOVQ 8(DI),CX - MOVQ 88(DI),R8 - MOVQ SI,R9 - CMOVQEQ DX,SI - CMOVQEQ R9,DX - MOVQ CX,R9 - CMOVQEQ R8,CX - CMOVQEQ R9,R8 - MOVQ SI,0(DI) - MOVQ DX,80(DI) - MOVQ CX,8(DI) - MOVQ R8,88(DI) - MOVQ 16(DI),SI - MOVQ 96(DI),DX - MOVQ 24(DI),CX - MOVQ 104(DI),R8 - MOVQ SI,R9 - CMOVQEQ DX,SI - CMOVQEQ R9,DX - MOVQ CX,R9 - CMOVQEQ R8,CX - CMOVQEQ R9,R8 - MOVQ SI,16(DI) - MOVQ DX,96(DI) - MOVQ CX,24(DI) - MOVQ R8,104(DI) - MOVQ 32(DI),SI - MOVQ 112(DI),DX - MOVQ 40(DI),CX - MOVQ 120(DI),R8 - MOVQ SI,R9 - CMOVQEQ DX,SI - CMOVQEQ R9,DX - MOVQ CX,R9 - CMOVQEQ R8,CX - CMOVQEQ R9,R8 - MOVQ SI,32(DI) - MOVQ DX,112(DI) - MOVQ CX,40(DI) - MOVQ R8,120(DI) - MOVQ 48(DI),SI - MOVQ 128(DI),DX - MOVQ 56(DI),CX - MOVQ 136(DI),R8 - MOVQ SI,R9 - CMOVQEQ DX,SI - CMOVQEQ R9,DX - MOVQ CX,R9 - CMOVQEQ R8,CX - CMOVQEQ R9,R8 - MOVQ SI,48(DI) - MOVQ DX,128(DI) - MOVQ CX,56(DI) - MOVQ R8,136(DI) - MOVQ 64(DI),SI - MOVQ 144(DI),DX - MOVQ 72(DI),CX - MOVQ 152(DI),R8 - MOVQ SI,R9 - CMOVQEQ DX,SI - CMOVQEQ R9,DX - MOVQ CX,R9 - CMOVQEQ R8,CX - CMOVQEQ R9,R8 - MOVQ SI,64(DI) - MOVQ DX,144(DI) - MOVQ CX,72(DI) - MOVQ R8,152(DI) - MOVQ DI,AX - MOVQ SI,DX + SUBQ $1, SI + NOTQ SI + MOVQ SI, X15 + PSHUFD $0x44, X15, X15 + + MOVOU 0(DI), X0 + MOVOU 16(DI), X2 + MOVOU 32(DI), X4 + MOVOU 48(DI), X6 + MOVOU 64(DI), X8 + MOVOU 80(DI), X1 + MOVOU 96(DI), X3 + MOVOU 112(DI), X5 + MOVOU 128(DI), X7 + MOVOU 144(DI), X9 + + MOVO X1, X10 + MOVO X3, X11 + MOVO X5, X12 + MOVO X7, X13 + MOVO X9, X14 + + PXOR X0, X10 + PXOR X2, X11 + PXOR X4, X12 + PXOR X6, X13 + PXOR X8, X14 + PAND X15, X10 + PAND X15, X11 + PAND X15, X12 + PAND X15, X13 + PAND X15, X14 + PXOR X10, X0 + PXOR X10, X1 + PXOR X11, X2 + PXOR X11, X3 + PXOR X12, X4 + PXOR X12, X5 + PXOR X13, X6 + PXOR X13, X7 + PXOR X14, X8 + PXOR X14, X9 + + MOVOU X0, 0(DI) + MOVOU X2, 16(DI) + MOVOU X4, 32(DI) + MOVOU X6, 48(DI) + MOVOU X8, 64(DI) + MOVOU X1, 80(DI) + MOVOU X3, 96(DI) + MOVOU X5, 112(DI) + MOVOU X7, 128(DI) + MOVOU X9, 144(DI) RET diff --git a/vendor/golang.org/x/crypto/curve25519/curve25519.go b/vendor/golang.org/x/crypto/curve25519/curve25519.go index 6918c47fc2..2d14c2a78a 100644 --- a/vendor/golang.org/x/crypto/curve25519/curve25519.go +++ b/vendor/golang.org/x/crypto/curve25519/curve25519.go @@ -8,6 +8,10 @@ package curve25519 +import ( + "encoding/binary" +) + // This code is a port of the public domain, "ref10" implementation of // curve25519 from SUPERCOP 20130419 by D. J. Bernstein. @@ -50,17 +54,11 @@ func feCopy(dst, src *fieldElement) { // // Preconditions: b in {0,1}. func feCSwap(f, g *fieldElement, b int32) { - var x fieldElement b = -b - for i := range x { - x[i] = b & (f[i] ^ g[i]) - } - for i := range f { - f[i] ^= x[i] - } - for i := range g { - g[i] ^= x[i] + t := b & (f[i] ^ g[i]) + f[i] ^= t + g[i] ^= t } } @@ -75,12 +73,7 @@ func load3(in []byte) int64 { // load4 reads a 32-bit, little-endian value from in. func load4(in []byte) int64 { - var r int64 - r = int64(in[0]) - r |= int64(in[1]) << 8 - r |= int64(in[2]) << 16 - r |= int64(in[3]) << 24 - return r + return int64(binary.LittleEndian.Uint32(in)) } func feFromBytes(dst *fieldElement, src *[32]byte) { diff --git a/vendor/golang.org/x/crypto/ssh/certs.go b/vendor/golang.org/x/crypto/ssh/certs.go index 6331c94d53..67600e2402 100644 --- a/vendor/golang.org/x/crypto/ssh/certs.go +++ b/vendor/golang.org/x/crypto/ssh/certs.go @@ -268,7 +268,7 @@ type CertChecker struct { // HostKeyFallback is called when CertChecker.CheckHostKey encounters a // public key that is not a certificate. It must implement host key // validation or else, if nil, all such keys are rejected. - HostKeyFallback func(addr string, remote net.Addr, key PublicKey) error + HostKeyFallback HostKeyCallback // IsRevoked is called for each certificate so that revocation checking // can be implemented. It should return true if the given certificate diff --git a/vendor/golang.org/x/crypto/ssh/client.go b/vendor/golang.org/x/crypto/ssh/client.go index c97f2978e8..a7e3263bca 100644 --- a/vendor/golang.org/x/crypto/ssh/client.go +++ b/vendor/golang.org/x/crypto/ssh/client.go @@ -5,6 +5,7 @@ package ssh import ( + "bytes" "errors" "fmt" "net" @@ -13,7 +14,7 @@ import ( ) // Client implements a traditional SSH client that supports shells, -// subprocesses, port forwarding and tunneled dialing. +// subprocesses, TCP port/streamlocal forwarding and tunneled dialing. type Client struct { Conn @@ -59,6 +60,7 @@ func NewClient(c Conn, chans <-chan NewChannel, reqs <-chan *Request) *Client { conn.forwards.closeAll() }() go conn.forwards.handleChannels(conn.HandleChannelOpen("forwarded-tcpip")) + go conn.forwards.handleChannels(conn.HandleChannelOpen("forwarded-streamlocal@openssh.com")) return conn } @@ -68,6 +70,11 @@ func NewClient(c Conn, chans <-chan NewChannel, reqs <-chan *Request) *Client { func NewClientConn(c net.Conn, addr string, config *ClientConfig) (Conn, <-chan NewChannel, <-chan *Request, error) { fullConf := *config fullConf.SetDefaults() + if fullConf.HostKeyCallback == nil { + c.Close() + return nil, nil, nil, errors.New("ssh: must specify HostKeyCallback") + } + conn := &connection{ sshConn: sshConn{conn: c}, } @@ -173,6 +180,13 @@ func Dial(network, addr string, config *ClientConfig) (*Client, error) { return NewClient(c, chans, reqs), nil } +// HostKeyCallback is the function type used for verifying server +// keys. A HostKeyCallback must return nil if the host key is OK, or +// an error to reject it. It receives the hostname as passed to Dial +// or NewClientConn. The remote address is the RemoteAddr of the +// net.Conn underlying the the SSH connection. +type HostKeyCallback func(hostname string, remote net.Addr, key PublicKey) error + // A ClientConfig structure is used to configure a Client. It must not be // modified after having been passed to an SSH function. type ClientConfig struct { @@ -188,10 +202,12 @@ type ClientConfig struct { // be used during authentication. Auth []AuthMethod - // HostKeyCallback, if not nil, is called during the cryptographic - // handshake to validate the server's host key. A nil HostKeyCallback - // implies that all host keys are accepted. - HostKeyCallback func(hostname string, remote net.Addr, key PublicKey) error + // HostKeyCallback is called during the cryptographic + // handshake to validate the server's host key. The client + // configuration must supply this callback for the connection + // to succeed. The functions InsecureIgnoreHostKey or + // FixedHostKey can be used for simplistic host key checks. + HostKeyCallback HostKeyCallback // ClientVersion contains the version identification string that will // be used for the connection. If empty, a reasonable default is used. @@ -209,3 +225,33 @@ type ClientConfig struct { // A Timeout of zero means no timeout. Timeout time.Duration } + +// InsecureIgnoreHostKey returns a function that can be used for +// ClientConfig.HostKeyCallback to accept any host key. It should +// not be used for production code. +func InsecureIgnoreHostKey() HostKeyCallback { + return func(hostname string, remote net.Addr, key PublicKey) error { + return nil + } +} + +type fixedHostKey struct { + key PublicKey +} + +func (f *fixedHostKey) check(hostname string, remote net.Addr, key PublicKey) error { + if f.key == nil { + return fmt.Errorf("ssh: required host key was nil") + } + if !bytes.Equal(key.Marshal(), f.key.Marshal()) { + return fmt.Errorf("ssh: host key mismatch") + } + return nil +} + +// FixedHostKey returns a function for use in +// ClientConfig.HostKeyCallback to accept only a specific host key. +func FixedHostKey(key PublicKey) HostKeyCallback { + hk := &fixedHostKey{key} + return hk.check +} diff --git a/vendor/golang.org/x/crypto/ssh/client_auth.go b/vendor/golang.org/x/crypto/ssh/client_auth.go index fd1ec5dda6..b882da0863 100644 --- a/vendor/golang.org/x/crypto/ssh/client_auth.go +++ b/vendor/golang.org/x/crypto/ssh/client_auth.go @@ -179,31 +179,26 @@ func (cb publicKeyCallback) method() string { } func (cb publicKeyCallback) auth(session []byte, user string, c packetConn, rand io.Reader) (bool, []string, error) { - // Authentication is performed in two stages. The first stage sends an - // enquiry to test if each key is acceptable to the remote. The second - // stage attempts to authenticate with the valid keys obtained in the - // first stage. + // Authentication is performed by sending an enquiry to test if a key is + // acceptable to the remote. If the key is acceptable, the client will + // attempt to authenticate with the valid key. If not the client will repeat + // the process with the remaining keys. signers, err := cb() if err != nil { return false, nil, err } - var validKeys []Signer - for _, signer := range signers { - if ok, err := validateKey(signer.PublicKey(), user, c); ok { - validKeys = append(validKeys, signer) - } else { - if err != nil { - return false, nil, err - } - } - } - - // methods that may continue if this auth is not successful. var methods []string - for _, signer := range validKeys { - pub := signer.PublicKey() + for _, signer := range signers { + ok, err := validateKey(signer.PublicKey(), user, c) + if err != nil { + return false, nil, err + } + if !ok { + continue + } + pub := signer.PublicKey() pubKey := pub.Marshal() sign, err := signer.Sign(rand, buildDataSignedForAuth(session, userAuthRequestMsg{ User: user, @@ -236,13 +231,29 @@ func (cb publicKeyCallback) auth(session []byte, user string, c packetConn, rand if err != nil { return false, nil, err } - if success { + + // If authentication succeeds or the list of available methods does not + // contain the "publickey" method, do not attempt to authenticate with any + // other keys. According to RFC 4252 Section 7, the latter can occur when + // additional authentication methods are required. + if success || !containsMethod(methods, cb.method()) { return success, methods, err } } + return false, methods, nil } +func containsMethod(methods []string, method string) bool { + for _, m := range methods { + if m == method { + return true + } + } + + return false +} + // validateKey validates the key provided is acceptable to the server. func validateKey(key PublicKey, user string, c packetConn) (bool, error) { pubKey := key.Marshal() diff --git a/vendor/golang.org/x/crypto/ssh/common.go b/vendor/golang.org/x/crypto/ssh/common.go index 8656d0f85d..dc39e4d231 100644 --- a/vendor/golang.org/x/crypto/ssh/common.go +++ b/vendor/golang.org/x/crypto/ssh/common.go @@ -9,6 +9,7 @@ import ( "crypto/rand" "fmt" "io" + "math" "sync" _ "crypto/sha1" @@ -40,7 +41,7 @@ var supportedKexAlgos = []string{ kexAlgoDH14SHA1, kexAlgoDH1SHA1, } -// supportedKexAlgos specifies the supported host-key algorithms (i.e. methods +// supportedHostKeyAlgos specifies the supported host-key algorithms (i.e. methods // of authenticating servers) in preference order. var supportedHostKeyAlgos = []string{ CertAlgoRSAv01, CertAlgoDSAv01, CertAlgoECDSA256v01, @@ -186,7 +187,7 @@ type Config struct { // The maximum number of bytes sent or received after which a // new key is negotiated. It must be at least 256. If - // unspecified, 1 gigabyte is used. + // unspecified, a size suitable for the chosen cipher is used. RekeyThreshold uint64 // The allowed key exchanges algorithms. If unspecified then a @@ -230,11 +231,12 @@ func (c *Config) SetDefaults() { } if c.RekeyThreshold == 0 { - // RFC 4253, section 9 suggests rekeying after 1G. - c.RekeyThreshold = 1 << 30 - } - if c.RekeyThreshold < minRekeyThreshold { + // cipher specific default + } else if c.RekeyThreshold < minRekeyThreshold { c.RekeyThreshold = minRekeyThreshold + } else if c.RekeyThreshold >= math.MaxInt64 { + // Avoid weirdness if somebody uses -1 as a threshold. + c.RekeyThreshold = math.MaxInt64 } } diff --git a/vendor/golang.org/x/crypto/ssh/doc.go b/vendor/golang.org/x/crypto/ssh/doc.go index d6be894662..67b7322c05 100644 --- a/vendor/golang.org/x/crypto/ssh/doc.go +++ b/vendor/golang.org/x/crypto/ssh/doc.go @@ -14,5 +14,8 @@ others. References: [PROTOCOL.certkeys]: http://cvsweb.openbsd.org/cgi-bin/cvsweb/src/usr.bin/ssh/PROTOCOL.certkeys?rev=HEAD [SSH-PARAMETERS]: http://www.iana.org/assignments/ssh-parameters/ssh-parameters.xml#ssh-parameters-1 + +This package does not fall under the stability promise of the Go language itself, +so its API may be changed when pressing needs arise. */ package ssh // import "golang.org/x/crypto/ssh" diff --git a/vendor/golang.org/x/crypto/ssh/handshake.go b/vendor/golang.org/x/crypto/ssh/handshake.go index 8de650644a..932ce8393e 100644 --- a/vendor/golang.org/x/crypto/ssh/handshake.go +++ b/vendor/golang.org/x/crypto/ssh/handshake.go @@ -74,7 +74,7 @@ type handshakeTransport struct { startKex chan *pendingKex // data for host key checking - hostKeyCallback func(hostname string, remote net.Addr, key PublicKey) error + hostKeyCallback HostKeyCallback dialAddress string remoteAddr net.Addr @@ -107,6 +107,8 @@ func newHandshakeTransport(conn keyingTransport, config *Config, clientVersion, config: config, } + t.resetReadThresholds() + t.resetWriteThresholds() // We always start with a mandatory key exchange. t.requestKex <- struct{}{} @@ -237,6 +239,17 @@ func (t *handshakeTransport) requestKeyExchange() { } } +func (t *handshakeTransport) resetWriteThresholds() { + t.writePacketsLeft = packetRekeyThreshold + if t.config.RekeyThreshold > 0 { + t.writeBytesLeft = int64(t.config.RekeyThreshold) + } else if t.algorithms != nil { + t.writeBytesLeft = t.algorithms.w.rekeyBytes() + } else { + t.writeBytesLeft = 1 << 30 + } +} + func (t *handshakeTransport) kexLoop() { write: @@ -285,12 +298,8 @@ write: t.writeError = err t.sentInitPacket = nil t.sentInitMsg = nil - t.writePacketsLeft = packetRekeyThreshold - if t.config.RekeyThreshold > 0 { - t.writeBytesLeft = int64(t.config.RekeyThreshold) - } else if t.algorithms != nil { - t.writeBytesLeft = t.algorithms.w.rekeyBytes() - } + + t.resetWriteThresholds() // we have completed the key exchange. Since the // reader is still blocked, it is safe to clear out @@ -344,6 +353,17 @@ write: // key exchange itself. const packetRekeyThreshold = (1 << 31) +func (t *handshakeTransport) resetReadThresholds() { + t.readPacketsLeft = packetRekeyThreshold + if t.config.RekeyThreshold > 0 { + t.readBytesLeft = int64(t.config.RekeyThreshold) + } else if t.algorithms != nil { + t.readBytesLeft = t.algorithms.r.rekeyBytes() + } else { + t.readBytesLeft = 1 << 30 + } +} + func (t *handshakeTransport) readOnePacket(first bool) ([]byte, error) { p, err := t.conn.readPacket() if err != nil { @@ -391,12 +411,7 @@ func (t *handshakeTransport) readOnePacket(first bool) ([]byte, error) { return nil, err } - t.readPacketsLeft = packetRekeyThreshold - if t.config.RekeyThreshold > 0 { - t.readBytesLeft = int64(t.config.RekeyThreshold) - } else { - t.readBytesLeft = t.algorithms.r.rekeyBytes() - } + t.resetReadThresholds() // By default, a key exchange is hidden from higher layers by // translating it into msgIgnore. @@ -574,7 +589,9 @@ func (t *handshakeTransport) enterKeyExchange(otherInitPacket []byte) error { } result.SessionID = t.sessionID - t.conn.prepareKeyChange(t.algorithms, result) + if err := t.conn.prepareKeyChange(t.algorithms, result); err != nil { + return err + } if err = t.conn.writePacket([]byte{msgNewKeys}); err != nil { return err } @@ -614,11 +631,9 @@ func (t *handshakeTransport) client(kex kexAlgorithm, algs *algorithms, magics * return nil, err } - if t.hostKeyCallback != nil { - err = t.hostKeyCallback(t.dialAddress, t.remoteAddr, hostKey) - if err != nil { - return nil, err - } + err = t.hostKeyCallback(t.dialAddress, t.remoteAddr, hostKey) + if err != nil { + return nil, err } return result, nil diff --git a/vendor/golang.org/x/crypto/ssh/keys.go b/vendor/golang.org/x/crypto/ssh/keys.go index f38de9898c..cf6853232b 100644 --- a/vendor/golang.org/x/crypto/ssh/keys.go +++ b/vendor/golang.org/x/crypto/ssh/keys.go @@ -824,7 +824,7 @@ func ParseDSAPrivateKey(der []byte) (*dsa.PrivateKey, error) { // Implemented based on the documentation at // https://github.com/openssh/openssh-portable/blob/master/PROTOCOL.key -func parseOpenSSHPrivateKey(key []byte) (*ed25519.PrivateKey, error) { +func parseOpenSSHPrivateKey(key []byte) (crypto.PrivateKey, error) { magic := append([]byte("openssh-key-v1"), 0) if !bytes.Equal(magic, key[0:len(magic)]) { return nil, errors.New("ssh: invalid openssh private key format") @@ -844,14 +844,15 @@ func parseOpenSSHPrivateKey(key []byte) (*ed25519.PrivateKey, error) { return nil, err } + if w.KdfName != "none" || w.CipherName != "none" { + return nil, errors.New("ssh: cannot decode encrypted private keys") + } + pk1 := struct { Check1 uint32 Check2 uint32 Keytype string - Pub []byte - Priv []byte - Comment string - Pad []byte `ssh:"rest"` + Rest []byte `ssh:"rest"` }{} if err := Unmarshal(w.PrivKeyBlock, &pk1); err != nil { @@ -862,24 +863,75 @@ func parseOpenSSHPrivateKey(key []byte) (*ed25519.PrivateKey, error) { return nil, errors.New("ssh: checkint mismatch") } - // we only handle ed25519 keys currently - if pk1.Keytype != KeyAlgoED25519 { + // we only handle ed25519 and rsa keys currently + switch pk1.Keytype { + case KeyAlgoRSA: + // https://github.com/openssh/openssh-portable/blob/master/sshkey.c#L2760-L2773 + key := struct { + N *big.Int + E *big.Int + D *big.Int + Iqmp *big.Int + P *big.Int + Q *big.Int + Comment string + Pad []byte `ssh:"rest"` + }{} + + if err := Unmarshal(pk1.Rest, &key); err != nil { + return nil, err + } + + for i, b := range key.Pad { + if int(b) != i+1 { + return nil, errors.New("ssh: padding not as expected") + } + } + + pk := &rsa.PrivateKey{ + PublicKey: rsa.PublicKey{ + N: key.N, + E: int(key.E.Int64()), + }, + D: key.D, + Primes: []*big.Int{key.P, key.Q}, + } + + if err := pk.Validate(); err != nil { + return nil, err + } + + pk.Precompute() + + return pk, nil + case KeyAlgoED25519: + key := struct { + Pub []byte + Priv []byte + Comment string + Pad []byte `ssh:"rest"` + }{} + + if err := Unmarshal(pk1.Rest, &key); err != nil { + return nil, err + } + + if len(key.Priv) != ed25519.PrivateKeySize { + return nil, errors.New("ssh: private key unexpected length") + } + + for i, b := range key.Pad { + if int(b) != i+1 { + return nil, errors.New("ssh: padding not as expected") + } + } + + pk := ed25519.PrivateKey(make([]byte, ed25519.PrivateKeySize)) + copy(pk, key.Priv) + return &pk, nil + default: return nil, errors.New("ssh: unhandled key type") } - - for i, b := range pk1.Pad { - if int(b) != i+1 { - return nil, errors.New("ssh: padding not as expected") - } - } - - if len(pk1.Priv) != ed25519.PrivateKeySize { - return nil, errors.New("ssh: private key unexpected length") - } - - pk := ed25519.PrivateKey(make([]byte, ed25519.PrivateKeySize)) - copy(pk, pk1.Priv) - return &pk, nil } // FingerprintLegacyMD5 returns the user presentation of the key's diff --git a/vendor/golang.org/x/crypto/ssh/server.go b/vendor/golang.org/x/crypto/ssh/server.go index 77c84d165c..8e95acc6ac 100644 --- a/vendor/golang.org/x/crypto/ssh/server.go +++ b/vendor/golang.org/x/crypto/ssh/server.go @@ -45,6 +45,12 @@ type ServerConfig struct { // authenticating. NoClientAuth bool + // MaxAuthTries specifies the maximum number of authentication attempts + // permitted per connection. If set to a negative number, the number of + // attempts are unlimited. If set to zero, the number of attempts are limited + // to 6. + MaxAuthTries int + // PasswordCallback, if non-nil, is called when a user // attempts to authenticate using a password. PasswordCallback func(conn ConnMetadata, password []byte) (*Permissions, error) @@ -141,6 +147,10 @@ type ServerConn struct { // Request and NewChannel channels must be serviced, or the connection // will hang. func NewServerConn(c net.Conn, config *ServerConfig) (*ServerConn, <-chan NewChannel, <-chan *Request, error) { + if config.MaxAuthTries == 0 { + config.MaxAuthTries = 6 + } + fullConf := *config fullConf.SetDefaults() s := &connection{ @@ -267,8 +277,23 @@ func (s *connection) serverAuthenticate(config *ServerConfig) (*Permissions, err var cache pubKeyCache var perms *Permissions + authFailures := 0 + userAuthLoop: for { + if authFailures >= config.MaxAuthTries && config.MaxAuthTries > 0 { + discMsg := &disconnectMsg{ + Reason: 2, + Message: "too many authentication failures", + } + + if err := s.transport.writePacket(Marshal(discMsg)); err != nil { + return nil, err + } + + return nil, discMsg + } + var userAuthReq userAuthRequestMsg if packet, err := s.transport.readPacket(); err != nil { return nil, err @@ -289,6 +314,11 @@ userAuthLoop: if config.NoClientAuth { authErr = nil } + + // allow initial attempt of 'none' without penalty + if authFailures == 0 { + authFailures-- + } case "password": if config.PasswordCallback == nil { authErr = errors.New("ssh: password auth not configured") @@ -360,6 +390,7 @@ userAuthLoop: if isQuery { // The client can query if the given public key // would be okay. + if len(payload) > 0 { return nil, parseError(msgUserAuthRequest) } @@ -409,6 +440,8 @@ userAuthLoop: break userAuthLoop } + authFailures++ + var failureMsg userAuthFailureMsg if config.PasswordCallback != nil { failureMsg.Methods = append(failureMsg.Methods, "password") diff --git a/vendor/golang.org/x/crypto/ssh/streamlocal.go b/vendor/golang.org/x/crypto/ssh/streamlocal.go new file mode 100644 index 0000000000..a2dccc64c7 --- /dev/null +++ b/vendor/golang.org/x/crypto/ssh/streamlocal.go @@ -0,0 +1,115 @@ +package ssh + +import ( + "errors" + "io" + "net" +) + +// streamLocalChannelOpenDirectMsg is a struct used for SSH_MSG_CHANNEL_OPEN message +// with "direct-streamlocal@openssh.com" string. +// +// See openssh-portable/PROTOCOL, section 2.4. connection: Unix domain socket forwarding +// https://github.com/openssh/openssh-portable/blob/master/PROTOCOL#L235 +type streamLocalChannelOpenDirectMsg struct { + socketPath string + reserved0 string + reserved1 uint32 +} + +// forwardedStreamLocalPayload is a struct used for SSH_MSG_CHANNEL_OPEN message +// with "forwarded-streamlocal@openssh.com" string. +type forwardedStreamLocalPayload struct { + SocketPath string + Reserved0 string +} + +// streamLocalChannelForwardMsg is a struct used for SSH2_MSG_GLOBAL_REQUEST message +// with "streamlocal-forward@openssh.com"/"cancel-streamlocal-forward@openssh.com" string. +type streamLocalChannelForwardMsg struct { + socketPath string +} + +// ListenUnix is similar to ListenTCP but uses a Unix domain socket. +func (c *Client) ListenUnix(socketPath string) (net.Listener, error) { + m := streamLocalChannelForwardMsg{ + socketPath, + } + // send message + ok, _, err := c.SendRequest("streamlocal-forward@openssh.com", true, Marshal(&m)) + if err != nil { + return nil, err + } + if !ok { + return nil, errors.New("ssh: streamlocal-forward@openssh.com request denied by peer") + } + ch := c.forwards.add(&net.UnixAddr{Name: socketPath, Net: "unix"}) + + return &unixListener{socketPath, c, ch}, nil +} + +func (c *Client) dialStreamLocal(socketPath string) (Channel, error) { + msg := streamLocalChannelOpenDirectMsg{ + socketPath: socketPath, + } + ch, in, err := c.OpenChannel("direct-streamlocal@openssh.com", Marshal(&msg)) + if err != nil { + return nil, err + } + go DiscardRequests(in) + return ch, err +} + +type unixListener struct { + socketPath string + + conn *Client + in <-chan forward +} + +// Accept waits for and returns the next connection to the listener. +func (l *unixListener) Accept() (net.Conn, error) { + s, ok := <-l.in + if !ok { + return nil, io.EOF + } + ch, incoming, err := s.newCh.Accept() + if err != nil { + return nil, err + } + go DiscardRequests(incoming) + + return &chanConn{ + Channel: ch, + laddr: &net.UnixAddr{ + Name: l.socketPath, + Net: "unix", + }, + raddr: &net.UnixAddr{ + Name: "@", + Net: "unix", + }, + }, nil +} + +// Close closes the listener. +func (l *unixListener) Close() error { + // this also closes the listener. + l.conn.forwards.remove(&net.UnixAddr{Name: l.socketPath, Net: "unix"}) + m := streamLocalChannelForwardMsg{ + l.socketPath, + } + ok, _, err := l.conn.SendRequest("cancel-streamlocal-forward@openssh.com", true, Marshal(&m)) + if err == nil && !ok { + err = errors.New("ssh: cancel-streamlocal-forward@openssh.com failed") + } + return err +} + +// Addr returns the listener's network address. +func (l *unixListener) Addr() net.Addr { + return &net.UnixAddr{ + Name: l.socketPath, + Net: "unix", + } +} diff --git a/vendor/golang.org/x/crypto/ssh/tcpip.go b/vendor/golang.org/x/crypto/ssh/tcpip.go index 6151241ff0..acf17175df 100644 --- a/vendor/golang.org/x/crypto/ssh/tcpip.go +++ b/vendor/golang.org/x/crypto/ssh/tcpip.go @@ -20,12 +20,20 @@ import ( // addr. Incoming connections will be available by calling Accept on // the returned net.Listener. The listener must be serviced, or the // SSH connection may hang. +// N must be "tcp", "tcp4", "tcp6", or "unix". func (c *Client) Listen(n, addr string) (net.Listener, error) { - laddr, err := net.ResolveTCPAddr(n, addr) - if err != nil { - return nil, err + switch n { + case "tcp", "tcp4", "tcp6": + laddr, err := net.ResolveTCPAddr(n, addr) + if err != nil { + return nil, err + } + return c.ListenTCP(laddr) + case "unix": + return c.ListenUnix(addr) + default: + return nil, fmt.Errorf("ssh: unsupported protocol: %s", n) } - return c.ListenTCP(laddr) } // Automatic port allocation is broken with OpenSSH before 6.0. See @@ -116,7 +124,7 @@ func (c *Client) ListenTCP(laddr *net.TCPAddr) (net.Listener, error) { } // Register this forward, using the port number we obtained. - ch := c.forwards.add(*laddr) + ch := c.forwards.add(laddr) return &tcpListener{laddr, c, ch}, nil } @@ -131,7 +139,7 @@ type forwardList struct { // forwardEntry represents an established mapping of a laddr on a // remote ssh server to a channel connected to a tcpListener. type forwardEntry struct { - laddr net.TCPAddr + laddr net.Addr c chan forward } @@ -139,16 +147,16 @@ type forwardEntry struct { // arguments to add/remove/lookup should be address as specified in // the original forward-request. type forward struct { - newCh NewChannel // the ssh client channel underlying this forward - raddr *net.TCPAddr // the raddr of the incoming connection + newCh NewChannel // the ssh client channel underlying this forward + raddr net.Addr // the raddr of the incoming connection } -func (l *forwardList) add(addr net.TCPAddr) chan forward { +func (l *forwardList) add(addr net.Addr) chan forward { l.Lock() defer l.Unlock() f := forwardEntry{ - addr, - make(chan forward, 1), + laddr: addr, + c: make(chan forward, 1), } l.entries = append(l.entries, f) return f.c @@ -176,44 +184,69 @@ func parseTCPAddr(addr string, port uint32) (*net.TCPAddr, error) { func (l *forwardList) handleChannels(in <-chan NewChannel) { for ch := range in { - var payload forwardedTCPPayload - if err := Unmarshal(ch.ExtraData(), &payload); err != nil { - ch.Reject(ConnectionFailed, "could not parse forwarded-tcpip payload: "+err.Error()) - continue - } + var ( + laddr net.Addr + raddr net.Addr + err error + ) + switch channelType := ch.ChannelType(); channelType { + case "forwarded-tcpip": + var payload forwardedTCPPayload + if err = Unmarshal(ch.ExtraData(), &payload); err != nil { + ch.Reject(ConnectionFailed, "could not parse forwarded-tcpip payload: "+err.Error()) + continue + } - // RFC 4254 section 7.2 specifies that incoming - // addresses should list the address, in string - // format. It is implied that this should be an IP - // address, as it would be impossible to connect to it - // otherwise. - laddr, err := parseTCPAddr(payload.Addr, payload.Port) - if err != nil { - ch.Reject(ConnectionFailed, err.Error()) - continue - } - raddr, err := parseTCPAddr(payload.OriginAddr, payload.OriginPort) - if err != nil { - ch.Reject(ConnectionFailed, err.Error()) - continue - } + // RFC 4254 section 7.2 specifies that incoming + // addresses should list the address, in string + // format. It is implied that this should be an IP + // address, as it would be impossible to connect to it + // otherwise. + laddr, err = parseTCPAddr(payload.Addr, payload.Port) + if err != nil { + ch.Reject(ConnectionFailed, err.Error()) + continue + } + raddr, err = parseTCPAddr(payload.OriginAddr, payload.OriginPort) + if err != nil { + ch.Reject(ConnectionFailed, err.Error()) + continue + } - if ok := l.forward(*laddr, *raddr, ch); !ok { + case "forwarded-streamlocal@openssh.com": + var payload forwardedStreamLocalPayload + if err = Unmarshal(ch.ExtraData(), &payload); err != nil { + ch.Reject(ConnectionFailed, "could not parse forwarded-streamlocal@openssh.com payload: "+err.Error()) + continue + } + laddr = &net.UnixAddr{ + Name: payload.SocketPath, + Net: "unix", + } + raddr = &net.UnixAddr{ + Name: "@", + Net: "unix", + } + default: + panic(fmt.Errorf("ssh: unknown channel type %s", channelType)) + } + if ok := l.forward(laddr, raddr, ch); !ok { // Section 7.2, implementations MUST reject spurious incoming // connections. ch.Reject(Prohibited, "no forward for address") continue } + } } // remove removes the forward entry, and the channel feeding its // listener. -func (l *forwardList) remove(addr net.TCPAddr) { +func (l *forwardList) remove(addr net.Addr) { l.Lock() defer l.Unlock() for i, f := range l.entries { - if addr.IP.Equal(f.laddr.IP) && addr.Port == f.laddr.Port { + if addr.Network() == f.laddr.Network() && addr.String() == f.laddr.String() { l.entries = append(l.entries[:i], l.entries[i+1:]...) close(f.c) return @@ -231,12 +264,12 @@ func (l *forwardList) closeAll() { l.entries = nil } -func (l *forwardList) forward(laddr, raddr net.TCPAddr, ch NewChannel) bool { +func (l *forwardList) forward(laddr, raddr net.Addr, ch NewChannel) bool { l.Lock() defer l.Unlock() for _, f := range l.entries { - if laddr.IP.Equal(f.laddr.IP) && laddr.Port == f.laddr.Port { - f.c <- forward{ch, &raddr} + if laddr.Network() == f.laddr.Network() && laddr.String() == f.laddr.String() { + f.c <- forward{newCh: ch, raddr: raddr} return true } } @@ -262,7 +295,7 @@ func (l *tcpListener) Accept() (net.Conn, error) { } go DiscardRequests(incoming) - return &tcpChanConn{ + return &chanConn{ Channel: ch, laddr: l.laddr, raddr: s.raddr, @@ -277,7 +310,7 @@ func (l *tcpListener) Close() error { } // this also closes the listener. - l.conn.forwards.remove(*l.laddr) + l.conn.forwards.remove(l.laddr) ok, _, err := l.conn.SendRequest("cancel-tcpip-forward", true, Marshal(&m)) if err == nil && !ok { err = errors.New("ssh: cancel-tcpip-forward failed") @@ -293,29 +326,52 @@ func (l *tcpListener) Addr() net.Addr { // Dial initiates a connection to the addr from the remote host. // The resulting connection has a zero LocalAddr() and RemoteAddr(). func (c *Client) Dial(n, addr string) (net.Conn, error) { - // Parse the address into host and numeric port. - host, portString, err := net.SplitHostPort(addr) - if err != nil { - return nil, err + var ch Channel + switch n { + case "tcp", "tcp4", "tcp6": + // Parse the address into host and numeric port. + host, portString, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + port, err := strconv.ParseUint(portString, 10, 16) + if err != nil { + return nil, err + } + ch, err = c.dial(net.IPv4zero.String(), 0, host, int(port)) + if err != nil { + return nil, err + } + // Use a zero address for local and remote address. + zeroAddr := &net.TCPAddr{ + IP: net.IPv4zero, + Port: 0, + } + return &chanConn{ + Channel: ch, + laddr: zeroAddr, + raddr: zeroAddr, + }, nil + case "unix": + var err error + ch, err = c.dialStreamLocal(addr) + if err != nil { + return nil, err + } + return &chanConn{ + Channel: ch, + laddr: &net.UnixAddr{ + Name: "@", + Net: "unix", + }, + raddr: &net.UnixAddr{ + Name: addr, + Net: "unix", + }, + }, nil + default: + return nil, fmt.Errorf("ssh: unsupported protocol: %s", n) } - port, err := strconv.ParseUint(portString, 10, 16) - if err != nil { - return nil, err - } - // Use a zero address for local and remote address. - zeroAddr := &net.TCPAddr{ - IP: net.IPv4zero, - Port: 0, - } - ch, err := c.dial(net.IPv4zero.String(), 0, host, int(port)) - if err != nil { - return nil, err - } - return &tcpChanConn{ - Channel: ch, - laddr: zeroAddr, - raddr: zeroAddr, - }, nil } // DialTCP connects to the remote address raddr on the network net, @@ -332,7 +388,7 @@ func (c *Client) DialTCP(n string, laddr, raddr *net.TCPAddr) (net.Conn, error) if err != nil { return nil, err } - return &tcpChanConn{ + return &chanConn{ Channel: ch, laddr: laddr, raddr: raddr, @@ -366,26 +422,26 @@ type tcpChan struct { Channel // the backing channel } -// tcpChanConn fulfills the net.Conn interface without +// chanConn fulfills the net.Conn interface without // the tcpChan having to hold laddr or raddr directly. -type tcpChanConn struct { +type chanConn struct { Channel laddr, raddr net.Addr } // LocalAddr returns the local network address. -func (t *tcpChanConn) LocalAddr() net.Addr { +func (t *chanConn) LocalAddr() net.Addr { return t.laddr } // RemoteAddr returns the remote network address. -func (t *tcpChanConn) RemoteAddr() net.Addr { +func (t *chanConn) RemoteAddr() net.Addr { return t.raddr } // SetDeadline sets the read and write deadlines associated // with the connection. -func (t *tcpChanConn) SetDeadline(deadline time.Time) error { +func (t *chanConn) SetDeadline(deadline time.Time) error { if err := t.SetReadDeadline(deadline); err != nil { return err } @@ -396,12 +452,14 @@ func (t *tcpChanConn) SetDeadline(deadline time.Time) error { // A zero value for t means Read will not time out. // After the deadline, the error from Read will implement net.Error // with Timeout() == true. -func (t *tcpChanConn) SetReadDeadline(deadline time.Time) error { +func (t *chanConn) SetReadDeadline(deadline time.Time) error { + // for compatibility with previous version, + // the error message contains "tcpChan" return errors.New("ssh: tcpChan: deadline not supported") } // SetWriteDeadline exists to satisfy the net.Conn interface // but is not implemented by this type. It always returns an error. -func (t *tcpChanConn) SetWriteDeadline(deadline time.Time) error { +func (t *chanConn) SetWriteDeadline(deadline time.Time) error { return errors.New("ssh: tcpChan: deadline not supported") } diff --git a/vendor/golang.org/x/crypto/ssh/terminal/util_solaris.go b/vendor/golang.org/x/crypto/ssh/terminal/util_solaris.go index 07eb5edd77..a2e1b57dc1 100644 --- a/vendor/golang.org/x/crypto/ssh/terminal/util_solaris.go +++ b/vendor/golang.org/x/crypto/ssh/terminal/util_solaris.go @@ -14,14 +14,12 @@ import ( // State contains the state of a terminal. type State struct { - termios syscall.Termios + state *unix.Termios } // IsTerminal returns true if the given file descriptor is a terminal. func IsTerminal(fd int) bool { - // see: http://src.illumos.org/source/xref/illumos-gate/usr/src/lib/libbc/libc/gen/common/isatty.c - var termio unix.Termio - err := unix.IoctlSetTermio(fd, unix.TCGETA, &termio) + _, err := unix.IoctlGetTermio(fd, unix.TCGETA) return err == nil } @@ -71,3 +69,60 @@ func ReadPassword(fd int) ([]byte, error) { return ret, nil } + +// MakeRaw puts the terminal connected to the given file descriptor into raw +// mode and returns the previous state of the terminal so that it can be +// restored. +// see http://cr.illumos.org/~webrev/andy_js/1060/ +func MakeRaw(fd int) (*State, error) { + oldTermiosPtr, err := unix.IoctlGetTermios(fd, unix.TCGETS) + if err != nil { + return nil, err + } + oldTermios := *oldTermiosPtr + + newTermios := oldTermios + newTermios.Iflag &^= syscall.IGNBRK | syscall.BRKINT | syscall.PARMRK | syscall.ISTRIP | syscall.INLCR | syscall.IGNCR | syscall.ICRNL | syscall.IXON + newTermios.Oflag &^= syscall.OPOST + newTermios.Lflag &^= syscall.ECHO | syscall.ECHONL | syscall.ICANON | syscall.ISIG | syscall.IEXTEN + newTermios.Cflag &^= syscall.CSIZE | syscall.PARENB + newTermios.Cflag |= syscall.CS8 + newTermios.Cc[unix.VMIN] = 1 + newTermios.Cc[unix.VTIME] = 0 + + if err := unix.IoctlSetTermios(fd, unix.TCSETS, &newTermios); err != nil { + return nil, err + } + + return &State{ + state: oldTermiosPtr, + }, nil +} + +// Restore restores the terminal connected to the given file descriptor to a +// previous state. +func Restore(fd int, oldState *State) error { + return unix.IoctlSetTermios(fd, unix.TCSETS, oldState.state) +} + +// GetState returns the current state of a terminal which may be useful to +// restore the terminal after a signal. +func GetState(fd int) (*State, error) { + oldTermiosPtr, err := unix.IoctlGetTermios(fd, unix.TCGETS) + if err != nil { + return nil, err + } + + return &State{ + state: oldTermiosPtr, + }, nil +} + +// GetSize returns the dimensions of the given terminal. +func GetSize(fd int) (width, height int, err error) { + ws, err := unix.IoctlGetWinsize(fd, unix.TIOCGWINSZ) + if err != nil { + return 0, 0, err + } + return int(ws.Col), int(ws.Row), nil +} diff --git a/vendor/vendor.json b/vendor/vendor.json index b2c8dde0a3..5da4093068 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -384,92 +384,92 @@ { "checksumSHA1": "TT1rac6kpQp2vz24m5yDGUNQ/QQ=", "path": "golang.org/x/crypto/cast5", - "revision": "453249f01cfeb54c3d549ddb75ff152ca243f9d8", - "revisionTime": "2017-02-08T20:51:15Z" + "revision": "c7af5bf2638a1164f2eb5467c39c6cffbd13a02e", + "revisionTime": "2017-04-25T18:31:00Z" }, { - "checksumSHA1": "C1KKOxFoW7/W/NFNpiXK+boguNo=", + "checksumSHA1": "nAu0XmCeC6WnUySyI8R7w4cxAqU=", "path": "golang.org/x/crypto/curve25519", - "revision": "459e26527287adbc2adcc5d0d49abff9a5f315a7", - "revisionTime": "2017-03-17T13:29:17Z" + "revision": "c7af5bf2638a1164f2eb5467c39c6cffbd13a02e", + "revisionTime": "2017-04-25T18:31:00Z" }, { "checksumSHA1": "wGb//LjBPNxYHqk+dcLo7BjPXK8=", "path": "golang.org/x/crypto/ed25519", - "revision": "459e26527287adbc2adcc5d0d49abff9a5f315a7", - "revisionTime": "2017-03-17T13:29:17Z" + "revision": "c7af5bf2638a1164f2eb5467c39c6cffbd13a02e", + "revisionTime": "2017-04-25T18:31:00Z" }, { "checksumSHA1": "LXFcVx8I587SnWmKycSDEq9yvK8=", "path": "golang.org/x/crypto/ed25519/internal/edwards25519", - "revision": "459e26527287adbc2adcc5d0d49abff9a5f315a7", - "revisionTime": "2017-03-17T13:29:17Z" + "revision": "c7af5bf2638a1164f2eb5467c39c6cffbd13a02e", + "revisionTime": "2017-04-25T18:31:00Z" }, { "checksumSHA1": "IIhFTrLlmlc6lEFSitqi4aw2lw0=", "path": "golang.org/x/crypto/openpgp", - "revision": "453249f01cfeb54c3d549ddb75ff152ca243f9d8", - "revisionTime": "2017-02-08T20:51:15Z" + "revision": "c7af5bf2638a1164f2eb5467c39c6cffbd13a02e", + "revisionTime": "2017-04-25T18:31:00Z" }, { "checksumSHA1": "olOKkhrdkYQHZ0lf1orrFQPQrv4=", "path": "golang.org/x/crypto/openpgp/armor", - "revision": "453249f01cfeb54c3d549ddb75ff152ca243f9d8", - "revisionTime": "2017-02-08T20:51:15Z" + "revision": "c7af5bf2638a1164f2eb5467c39c6cffbd13a02e", + "revisionTime": "2017-04-25T18:31:00Z" }, { "checksumSHA1": "eo/KtdjieJQXH7Qy+faXFcF70ME=", "path": "golang.org/x/crypto/openpgp/elgamal", - "revision": "453249f01cfeb54c3d549ddb75ff152ca243f9d8", - "revisionTime": "2017-02-08T20:51:15Z" + "revision": "c7af5bf2638a1164f2eb5467c39c6cffbd13a02e", + "revisionTime": "2017-04-25T18:31:00Z" }, { "checksumSHA1": "rlxVSaGgqdAgwblsErxTxIfuGfg=", "path": "golang.org/x/crypto/openpgp/errors", - "revision": "453249f01cfeb54c3d549ddb75ff152ca243f9d8", - "revisionTime": "2017-02-08T20:51:15Z" + "revision": "c7af5bf2638a1164f2eb5467c39c6cffbd13a02e", + "revisionTime": "2017-04-25T18:31:00Z" }, { "checksumSHA1": "LWdaR8Q9yn6eBCcnGl0HvJRDUBE=", "path": "golang.org/x/crypto/openpgp/packet", - "revision": "453249f01cfeb54c3d549ddb75ff152ca243f9d8", - "revisionTime": "2017-02-08T20:51:15Z" + "revision": "c7af5bf2638a1164f2eb5467c39c6cffbd13a02e", + "revisionTime": "2017-04-25T18:31:00Z" }, { "checksumSHA1": "s2qT4UwvzBSkzXuiuMkowif1Olw=", "path": "golang.org/x/crypto/openpgp/s2k", - "revision": "453249f01cfeb54c3d549ddb75ff152ca243f9d8", - "revisionTime": "2017-02-08T20:51:15Z" + "revision": "c7af5bf2638a1164f2eb5467c39c6cffbd13a02e", + "revisionTime": "2017-04-25T18:31:00Z" }, { "checksumSHA1": "1MGpGDQqnUoRpv7VEcQrXOBydXE=", "path": "golang.org/x/crypto/pbkdf2", - "revision": "453249f01cfeb54c3d549ddb75ff152ca243f9d8", - "revisionTime": "2017-02-08T20:51:15Z" + "revision": "c7af5bf2638a1164f2eb5467c39c6cffbd13a02e", + "revisionTime": "2017-04-25T18:31:00Z" }, { "checksumSHA1": "y/oIaxq2d3WPizRZfVjo8RCRYTU=", "path": "golang.org/x/crypto/ripemd160", - "revision": "453249f01cfeb54c3d549ddb75ff152ca243f9d8", - "revisionTime": "2017-02-08T20:51:15Z" + "revision": "c7af5bf2638a1164f2eb5467c39c6cffbd13a02e", + "revisionTime": "2017-04-25T18:31:00Z" }, { "checksumSHA1": "E8pDMGySfy5Mw+jzXOkOxo35bww=", "path": "golang.org/x/crypto/scrypt", - "revision": "453249f01cfeb54c3d549ddb75ff152ca243f9d8", - "revisionTime": "2017-02-08T20:51:15Z" + "revision": "c7af5bf2638a1164f2eb5467c39c6cffbd13a02e", + "revisionTime": "2017-04-25T18:31:00Z" }, { - "checksumSHA1": "fsrFs762jlaILyqqQImS1GfvIvw=", + "checksumSHA1": "8sVsMTphul+B0sI0qAv4TE1ZxUk=", "path": "golang.org/x/crypto/ssh", - "revision": "459e26527287adbc2adcc5d0d49abff9a5f315a7", - "revisionTime": "2017-03-17T13:29:17Z" + "revision": "c7af5bf2638a1164f2eb5467c39c6cffbd13a02e", + "revisionTime": "2017-04-25T18:31:00Z" }, { - "checksumSHA1": "xiderUuvye8Kpn7yX3niiJg32bE=", + "checksumSHA1": "ZaU56svwLgiJD0y8JOB3+/mpYBA=", "path": "golang.org/x/crypto/ssh/terminal", - "revision": "459e26527287adbc2adcc5d0d49abff9a5f315a7", - "revisionTime": "2017-03-17T13:29:17Z" + "revision": "c7af5bf2638a1164f2eb5467c39c6cffbd13a02e", + "revisionTime": "2017-04-25T18:31:00Z" }, { "checksumSHA1": "Y+HGqEkYM15ir+J93MEaHdyFy0c=", diff --git a/whisper/mailserver/server_test.go b/whisper/mailserver/server_test.go index 77a5c6cd73..a4407847ed 100644 --- a/whisper/mailserver/server_test.go +++ b/whisper/mailserver/server_test.go @@ -58,15 +58,19 @@ func TestDBKey(t *testing.T) { } func generateEnvelope(t *testing.T) *whisper.Envelope { + h := crypto.Keccak256Hash([]byte("test sample data")) params := &whisper.MessageParams{ - KeySym: []byte("test key"), + KeySym: h[:], Topic: whisper.TopicType{}, Payload: []byte("test payload"), PoW: powRequirement, WorkTime: 2, } - msg := whisper.NewSentMessage(params) + msg, err := whisper.NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } env, err := msg.Wrap(params) if err != nil { t.Fatalf("failed to wrap with seed %d: %s.", seed, err) @@ -188,7 +192,10 @@ func createRequest(t *testing.T, p *ServerTestParams) *whisper.Envelope { Src: p.key, } - msg := whisper.NewSentMessage(params) + msg, err := whisper.NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } env, err := msg.Wrap(params) if err != nil { t.Fatalf("failed to wrap with seed %d: %s.", seed, err) diff --git a/whisper/whisperv5/api.go b/whisper/whisperv5/api.go index fae8e8806d..73718937d7 100644 --- a/whisper/whisperv5/api.go +++ b/whisper/whisperv5/api.go @@ -214,7 +214,6 @@ func (api *PublicWhisperAPI) Subscribe(args WhisperFilterArgs) (string, error) { } filter := Filter{ - Src: crypto.ToECDSAPub(common.FromHex(args.SignedWith)), PoW: args.MinPoW, Messages: make(map[common.Hash]*ReceivedMessage), AllowP2P: args.AllowP2P, @@ -232,9 +231,14 @@ func (api *PublicWhisperAPI) Subscribe(args WhisperFilterArgs) (string, error) { return "", errors.New("subscribe: " + err.Error()) } - if len(args.SignedWith) > 0 { + if len(args.Sig) > 0 { + sb := common.FromHex(args.Sig) + if sb == nil { + return "", errors.New("subscribe: sig parameter is invalid") + } + filter.Src = crypto.ToECDSAPub(sb) if !ValidatePublicKey(filter.Src) { - return "", errors.New("subscribe: invalid 'SignedWith' field") + return "", errors.New("subscribe: invalid 'sig' field") } } @@ -269,9 +273,10 @@ func (api *PublicWhisperAPI) Unsubscribe(id string) { api.whisper.Unsubscribe(id) } -// GetSubscriptionMessages retrieves all the new messages matched by a filter since the last retrieval. -func (api *PublicWhisperAPI) GetSubscriptionMessages(filterId string) []*WhisperMessage { - f := api.whisper.GetFilter(filterId) +// GetSubscriptionMessages retrieves all the new messages matched by the corresponding +// subscription filter since the last retrieval. +func (api *PublicWhisperAPI) GetNewSubscriptionMessages(id string) []*WhisperMessage { + f := api.whisper.GetFilter(id) if f != nil { newMail := f.Retrieve() return toWhisperMessages(newMail) @@ -279,10 +284,10 @@ func (api *PublicWhisperAPI) GetSubscriptionMessages(filterId string) []*Whisper return toWhisperMessages(nil) } -// GetMessages retrieves all the floating messages that match a specific filter. +// GetMessages retrieves all the floating messages that match a specific subscription filter. // It is likely to be called once per session, right after Subscribe call. -func (api *PublicWhisperAPI) GetMessages(filterId string) []*WhisperMessage { - all := api.whisper.Messages(filterId) +func (api *PublicWhisperAPI) GetFloatingMessages(id string) []*WhisperMessage { + all := api.whisper.Messages(id) return toWhisperMessages(all) } @@ -314,8 +319,8 @@ func (api *PublicWhisperAPI) Post(args PostArgs) error { return errors.New("post: key is missing") } - if len(args.SignWith) > 0 { - params.Src, err = api.whisper.GetPrivateKey(args.SignWith) + if len(args.Sig) > 0 { + params.Src, err = api.whisper.GetPrivateKey(args.Sig) if err != nil { return err } @@ -345,7 +350,11 @@ func (api *PublicWhisperAPI) Post(args PostArgs) error { return errors.New("post: topic is missing for symmetric encryption") } } else if args.Type == "asym" { - params.Dst = crypto.ToECDSAPub(common.FromHex(args.Key)) + kb := common.FromHex(args.Key) + if kb == nil { + return errors.New("post: public key for asymmetric encryption is invalid") + } + params.Dst = crypto.ToECDSAPub(kb) if !ValidatePublicKey(params.Dst) { return errors.New("post: public key for asymmetric encryption is invalid") } @@ -354,9 +363,9 @@ func (api *PublicWhisperAPI) Post(args PostArgs) error { } // encrypt and send - message := NewSentMessage(¶ms) - if message == nil { - return errors.New("post: failed create new message, probably due to failed rand function (OS level)") + message, err := NewSentMessage(¶ms) + if err != nil { + return err } envelope, err := message.Wrap(¶ms) if err != nil { @@ -382,8 +391,8 @@ func (api *PublicWhisperAPI) Post(args PostArgs) error { type PostArgs struct { Type string `json:"type"` // "sym"/"asym" (symmetric or asymmetric) TTL uint32 `json:"ttl"` // time-to-live in seconds - SignWith string `json:"signWith"` // id of the signing key - Key string `json:"key"` // id of encryption key + Sig string `json:"sig"` // id of the signing key + Key string `json:"key"` // key id (in case of sym) or public key (in case of asym) Topic hexutil.Bytes `json:"topic"` // topic (4 bytes) Padding hexutil.Bytes `json:"padding"` // optional padding bytes Payload hexutil.Bytes `json:"payload"` // payload to be encrypted @@ -393,12 +402,12 @@ type PostArgs struct { } type WhisperFilterArgs struct { - Symmetric bool // encryption type - Key string // id of the key to be used for decryption - SignedWith string // public key of the sender to be verified - MinPoW float64 // minimal PoW requirement - Topics [][]byte // list of topics (up to 4 bytes each) to match - AllowP2P bool // indicates wheather direct p2p messages are allowed for this filter + Symmetric bool // encryption type + Key string // id of the key to be used for decryption + Sig string // public key of the sender to be verified + MinPoW float64 // minimal PoW requirement + Topics [][]byte // list of topics (up to 4 bytes each) to match + AllowP2P bool // indicates wheather direct p2p messages are allowed for this filter } // UnmarshalJSON implements the json.Unmarshaler interface, invoked to convert a @@ -406,12 +415,12 @@ type WhisperFilterArgs struct { func (args *WhisperFilterArgs) UnmarshalJSON(b []byte) (err error) { // Unmarshal the JSON message and sanity check var obj struct { - Type string `json:"type"` - Key string `json:"key"` - SignedWith string `json:"signedWith"` - MinPoW float64 `json:"minPoW"` - Topics []interface{} `json:"topics"` - AllowP2P bool `json:"allowP2P"` + Type string `json:"type"` + Key string `json:"key"` + Sig string `json:"sig"` + MinPoW float64 `json:"minPoW"` + Topics []interface{} `json:"topics"` + AllowP2P bool `json:"allowP2P"` } if err := json.Unmarshal(b, &obj); err != nil { return err @@ -427,7 +436,7 @@ func (args *WhisperFilterArgs) UnmarshalJSON(b []byte) (err error) { } args.Key = obj.Key - args.SignedWith = obj.SignedWith + args.Sig = obj.Sig args.MinPoW = obj.MinPoW args.AllowP2P = obj.AllowP2P @@ -463,7 +472,7 @@ type WhisperMessage struct { Topic string `json:"topic"` Payload string `json:"payload"` Padding string `json:"padding"` - Src string `json:"signedWith"` + Src string `json:"sig"` Dst string `json:"recipientPublicKey"` Timestamp uint32 `json:"timestamp"` TTL uint32 `json:"ttl"` @@ -474,7 +483,6 @@ type WhisperMessage struct { // NewWhisperMessage converts an internal message into an API version. func NewWhisperMessage(message *ReceivedMessage) *WhisperMessage { msg := WhisperMessage{ - Topic: common.ToHex(message.Topic[:]), Payload: common.ToHex(message.Payload), Padding: common.ToHex(message.Padding), Timestamp: message.Sent, @@ -483,11 +491,20 @@ func NewWhisperMessage(message *ReceivedMessage) *WhisperMessage { Hash: common.ToHex(message.EnvelopeHash.Bytes()), } + if len(message.Topic) == TopicLength { + msg.Topic = common.ToHex(message.Topic[:]) + } if message.Dst != nil { - msg.Dst = common.ToHex(crypto.FromECDSAPub(message.Dst)) + b := crypto.FromECDSAPub(message.Dst) + if b != nil { + msg.Dst = common.ToHex(b) + } } if isMessageSigned(message.Raw[0]) { - msg.Src = common.ToHex(crypto.FromECDSAPub(message.SigToPubKey())) + b := crypto.FromECDSAPub(message.SigToPubKey()) + if b != nil { + msg.Src = common.ToHex(b) + } } return &msg } diff --git a/whisper/whisperv5/api_test.go b/whisper/whisperv5/api_test.go index 570b3dbadf..18910866b7 100644 --- a/whisper/whisperv5/api_test.go +++ b/whisper/whisperv5/api_test.go @@ -43,7 +43,7 @@ func TestBasic(t *testing.T) { t.Fatalf("wrong version: %d.", ver) } - mail := api.GetSubscriptionMessages("non-existent-id") + mail := api.GetNewSubscriptionMessages("non-existent-id") if len(mail) != 0 { t.Fatalf("failed GetFilterChanges: premature result") } @@ -173,7 +173,7 @@ func TestUnmarshalFilterArgs(t *testing.T) { s := []byte(`{ "type":"sym", "key":"0x70c87d191324e6712a591f304b4eedef6ad9bb9d", - "signedWith":"0x9b2055d370f73ec7d8a03e965129118dc8f5bf83", + "sig":"0x9b2055d370f73ec7d8a03e965129118dc8f5bf83", "minPoW":2.34, "topics":["0x00000000", "0x007f80ff", "0xff807f00", "0xf26e7779"], "allowP2P":true @@ -191,8 +191,8 @@ func TestUnmarshalFilterArgs(t *testing.T) { if f.Key != "0x70c87d191324e6712a591f304b4eedef6ad9bb9d" { t.Fatalf("wrong key: %s.", f.Key) } - if f.SignedWith != "0x9b2055d370f73ec7d8a03e965129118dc8f5bf83" { - t.Fatalf("wrong SignedWith: %s.", f.SignedWith) + if f.Sig != "0x9b2055d370f73ec7d8a03e965129118dc8f5bf83" { + t.Fatalf("wrong sig: %s.", f.Sig) } if f.MinPoW != 2.34 { t.Fatalf("wrong MinPoW: %f.", f.MinPoW) @@ -229,7 +229,7 @@ func TestUnmarshalPostArgs(t *testing.T) { s := []byte(`{ "type":"sym", "ttl":12345, - "signWith":"0x70c87d191324e6712a591f304b4eedef6ad9bb9d", + "sig":"0x70c87d191324e6712a591f304b4eedef6ad9bb9d", "key":"0x9b2055d370f73ec7d8a03e965129118dc8f5bf83", "topic":"0xf26e7779", "padding":"0x74686973206973206D79207465737420737472696E67", @@ -251,8 +251,8 @@ func TestUnmarshalPostArgs(t *testing.T) { if a.TTL != 12345 { t.Fatalf("wrong ttl: %d.", a.TTL) } - if a.SignWith != "0x70c87d191324e6712a591f304b4eedef6ad9bb9d" { - t.Fatalf("wrong From: %s.", a.SignWith) + if a.Sig != "0x70c87d191324e6712a591f304b4eedef6ad9bb9d" { + t.Fatalf("wrong From: %s.", a.Sig) } if a.Key != "0x9b2055d370f73ec7d8a03e965129118dc8f5bf83" { t.Fatalf("wrong Key: %s.", a.Key) @@ -282,7 +282,7 @@ func waitForMessages(api *PublicWhisperAPI, id string, target int) []*WhisperMes // timeout: 2 seconds result := make([]*WhisperMessage, 0, target) for i := 0; i < 100; i++ { - mail := api.GetSubscriptionMessages(id) + mail := api.GetNewSubscriptionMessages(id) if len(mail) > 0 { for _, m := range mail { result = append(result, m) @@ -347,7 +347,7 @@ func TestIntegrationAsym(t *testing.T) { var f WhisperFilterArgs f.Symmetric = false f.Key = key - f.SignedWith = sigPubKey.String() + f.Sig = sigPubKey.String() f.Topics = make([][]byte, 2) f.Topics[0] = topics[0][:] f.Topics[1] = topics[1][:] @@ -362,7 +362,7 @@ func TestIntegrationAsym(t *testing.T) { var p PostArgs p.Type = "asym" p.TTL = 2 - p.SignWith = sig + p.Sig = sig p.Key = dstPubKey.String() p.Padding = []byte("test string") p.Payload = []byte("extended test string") @@ -448,8 +448,8 @@ func TestIntegrationSym(t *testing.T) { f.Topics = make([][]byte, 2) f.Topics[0] = topics[0][:] f.Topics[1] = topics[1][:] - f.MinPoW = 0.324 - f.SignedWith = sigPubKey.String() + f.MinPoW = DefaultMinimumPoW / 2 + f.Sig = sigPubKey.String() f.AllowP2P = false id, err := api.Subscribe(f) @@ -461,7 +461,7 @@ func TestIntegrationSym(t *testing.T) { p.Type = "sym" p.TTL = 1 p.Key = symKeyID - p.SignWith = sig + p.Sig = sig p.Padding = []byte("test string") p.Payload = []byte("extended test string") p.PowTarget = DefaultMinimumPoW @@ -546,8 +546,8 @@ func TestIntegrationSymWithFilter(t *testing.T) { f.Topics = make([][]byte, 2) f.Topics[0] = topics[0][:] f.Topics[1] = topics[1][:] - f.MinPoW = 0.324 - f.SignedWith = sigPubKey.String() + f.MinPoW = DefaultMinimumPoW / 2 + f.Sig = sigPubKey.String() f.AllowP2P = false id, err := api.Subscribe(f) @@ -559,7 +559,7 @@ func TestIntegrationSymWithFilter(t *testing.T) { p.Type = "sym" p.TTL = 1 p.Key = symKeyID - p.SignWith = sigKeyID + p.Sig = sigKeyID p.Padding = []byte("test string") p.Payload = []byte("extended test string") p.PowTarget = DefaultMinimumPoW diff --git a/whisper/whisperv5/benchmarks_test.go b/whisper/whisperv5/benchmarks_test.go index c987f33bbc..09fcb77940 100644 --- a/whisper/whisperv5/benchmarks_test.go +++ b/whisper/whisperv5/benchmarks_test.go @@ -28,12 +28,6 @@ func BenchmarkDeriveKeyMaterial(b *testing.B) { } } -func BenchmarkDeriveOneTimeKey(b *testing.B) { - for i := 0; i < b.N; i++ { - DeriveOneTimeKey([]byte("test value 1"), []byte("test value 2"), 0) - } -} - func BenchmarkEncryptionSym(b *testing.B) { InitSingleTest() @@ -43,7 +37,7 @@ func BenchmarkEncryptionSym(b *testing.B) { } for i := 0; i < b.N; i++ { - msg := NewSentMessage(params) + msg, _ := NewSentMessage(params) _, err := msg.Wrap(params) if err != nil { b.Errorf("failed Wrap with seed %d: %s.", seed, err) @@ -68,7 +62,7 @@ func BenchmarkEncryptionAsym(b *testing.B) { params.Dst = &key.PublicKey for i := 0; i < b.N; i++ { - msg := NewSentMessage(params) + msg, _ := NewSentMessage(params) _, err := msg.Wrap(params) if err != nil { b.Fatalf("failed Wrap with seed %d: %s.", seed, err) @@ -83,7 +77,7 @@ func BenchmarkDecryptionSymValid(b *testing.B) { if err != nil { b.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - msg := NewSentMessage(params) + msg, _ := NewSentMessage(params) env, err := msg.Wrap(params) if err != nil { b.Fatalf("failed Wrap with seed %d: %s.", seed, err) @@ -105,7 +99,7 @@ func BenchmarkDecryptionSymInvalid(b *testing.B) { if err != nil { b.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - msg := NewSentMessage(params) + msg, _ := NewSentMessage(params) env, err := msg.Wrap(params) if err != nil { b.Fatalf("failed Wrap with seed %d: %s.", seed, err) @@ -134,7 +128,7 @@ func BenchmarkDecryptionAsymValid(b *testing.B) { f := Filter{KeyAsym: key} params.KeySym = nil params.Dst = &key.PublicKey - msg := NewSentMessage(params) + msg, _ := NewSentMessage(params) env, err := msg.Wrap(params) if err != nil { b.Fatalf("failed Wrap with seed %d: %s.", seed, err) @@ -161,7 +155,7 @@ func BenchmarkDecryptionAsymInvalid(b *testing.B) { } params.KeySym = nil params.Dst = &key.PublicKey - msg := NewSentMessage(params) + msg, _ := NewSentMessage(params) env, err := msg.Wrap(params) if err != nil { b.Fatalf("failed Wrap with seed %d: %s.", seed, err) @@ -203,7 +197,7 @@ func BenchmarkPoW(b *testing.B) { for i := 0; i < b.N; i++ { increment(params.Payload) - msg := NewSentMessage(params) + msg, _ := NewSentMessage(params) _, err := msg.Wrap(params) if err != nil { b.Fatalf("failed Wrap with seed %d: %s.", seed, err) diff --git a/whisper/whisperv5/doc.go b/whisper/whisperv5/doc.go index d60868f670..768291a161 100644 --- a/whisper/whisperv5/doc.go +++ b/whisper/whisperv5/doc.go @@ -49,18 +49,16 @@ const ( paddingMask = byte(3) signatureFlag = byte(4) - TopicLength = 4 - signatureLength = 65 - aesKeyLength = 32 - saltLength = 12 - AESNonceMaxLength = 12 - keyIdSize = 32 + TopicLength = 4 + signatureLength = 65 + aesKeyLength = 32 + AESNonceLength = 12 + keyIdSize = 32 DefaultMaxMessageLength = 1024 * 1024 - DefaultMinimumPoW = 1.0 // todo: review after testing. + DefaultMinimumPoW = 0.2 - padSizeLimitLower = 128 // it can not be less - we don't want to reveal the absence of signature - padSizeLimitUpper = 256 // just an arbitrary number, could be changed without losing compatibility + padSizeLimit = 256 // just an arbitrary number, could be changed without breaking the protocol (must not exceed 2^24) messageQueueLimit = 1024 expirationCycle = time.Second diff --git a/whisper/whisperv5/envelope.go b/whisper/whisperv5/envelope.go index 040faf1509..edaf83baf9 100644 --- a/whisper/whisperv5/envelope.go +++ b/whisper/whisperv5/envelope.go @@ -40,7 +40,6 @@ type Envelope struct { Expiry uint32 TTL uint32 Topic TopicType - Salt []byte AESNonce []byte Data []byte EnvNonce uint64 @@ -50,15 +49,25 @@ type Envelope struct { // Don't access hash directly, use Hash() function instead. } +// size returns the size of envelope as it is sent (i.e. public fields only) +func (e *Envelope) size() int { + return 20 + len(e.Version) + len(e.AESNonce) + len(e.Data) +} + +// rlpWithoutNonce returns the RLP encoded envelope contents, except the nonce. +func (e *Envelope) rlpWithoutNonce() []byte { + res, _ := rlp.EncodeToBytes([]interface{}{e.Version, e.Expiry, e.TTL, e.Topic, e.AESNonce, e.Data}) + return res +} + // NewEnvelope wraps a Whisper message with expiration and destination data // included into an envelope for network forwarding. -func NewEnvelope(ttl uint32, topic TopicType, salt []byte, aesNonce []byte, msg *SentMessage) *Envelope { +func NewEnvelope(ttl uint32, topic TopicType, aesNonce []byte, msg *SentMessage) *Envelope { env := Envelope{ Version: make([]byte, 1), Expiry: uint32(time.Now().Add(time.Second * time.Duration(ttl)).Unix()), TTL: ttl, Topic: topic, - Salt: salt, AESNonce: aesNonce, Data: msg.Raw, EnvNonce: 0, @@ -126,10 +135,6 @@ func (e *Envelope) Seal(options *MessageParams) error { return nil } -func (e *Envelope) size() int { - return len(e.Data) + len(e.Version) + len(e.AESNonce) + len(e.Salt) + 20 -} - func (e *Envelope) PoW() float64 { if e.pow == 0 { e.calculatePoW(0) @@ -159,12 +164,6 @@ func (e *Envelope) powToFirstBit(pow float64) int { return int(bits) } -// rlpWithoutNonce returns the RLP encoded envelope contents, except the nonce. -func (e *Envelope) rlpWithoutNonce() []byte { - res, _ := rlp.EncodeToBytes([]interface{}{e.Expiry, e.TTL, e.Topic, e.Salt, e.AESNonce, e.Data}) - return res -} - // Hash returns the SHA3 hash of the envelope, calculating it if not yet done. func (e *Envelope) Hash() common.Hash { if (e.hash == common.Hash{}) { @@ -210,7 +209,7 @@ func (e *Envelope) OpenAsymmetric(key *ecdsa.PrivateKey) (*ReceivedMessage, erro // OpenSymmetric tries to decrypt an envelope, potentially encrypted with a particular key. func (e *Envelope) OpenSymmetric(key []byte) (msg *ReceivedMessage, err error) { msg = &ReceivedMessage{Raw: e.Data} - err = msg.decryptSymmetric(key, e.Salt, e.AESNonce) + err = msg.decryptSymmetric(key, e.AESNonce) if err != nil { msg = nil } diff --git a/whisper/whisperv5/filter_test.go b/whisper/whisperv5/filter_test.go index 98287e3226..7acb39066b 100644 --- a/whisper/whisperv5/filter_test.go +++ b/whisper/whisperv5/filter_test.go @@ -68,7 +68,7 @@ func generateFilter(t *testing.T, symmetric bool) (*Filter, error) { f.Src = &key.PublicKey if symmetric { - f.KeySym = make([]byte, 12) + f.KeySym = make([]byte, aesKeyLength) mrand.Read(f.KeySym) f.SymKeyHash = crypto.Keccak256Hash(f.KeySym) } else { @@ -179,7 +179,10 @@ func TestMatchEnvelope(t *testing.T) { params.Topic[0] = 0xFF // ensure mismatch // mismatch with pseudo-random data - msg := NewSentMessage(params) + msg, err := NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } env, err := msg.Wrap(params) if err != nil { t.Fatalf("failed Wrap with seed %d: %s.", seed, err) @@ -197,7 +200,10 @@ func TestMatchEnvelope(t *testing.T) { i := mrand.Int() % 4 fsym.Topics[i] = params.Topic[:] fasym.Topics[i] = params.Topic[:] - msg = NewSentMessage(params) + msg, err = NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } env, err = msg.Wrap(params) if err != nil { t.Fatalf("failed Wrap() with seed %d: %s.", seed, err) @@ -245,7 +251,10 @@ func TestMatchEnvelope(t *testing.T) { } params.KeySym = nil params.Dst = &key.PublicKey - msg = NewSentMessage(params) + msg, err = NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } env, err = msg.Wrap(params) if err != nil { t.Fatalf("failed Wrap() with seed %d: %s.", seed, err) @@ -323,12 +332,14 @@ func TestMatchMessageSym(t *testing.T) { params.KeySym = f.KeySym params.Topic = BytesToTopic(f.Topics[index]) - sentMessage := NewSentMessage(params) + sentMessage, err := NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } env, err := sentMessage.Wrap(params) if err != nil { t.Fatalf("failed Wrap with seed %d: %s.", seed, err) } - msg := env.Open(f) if msg == nil { t.Fatalf("failed Open with seed %d.", seed) @@ -419,12 +430,14 @@ func TestMatchMessageAsym(t *testing.T) { keySymOrig := params.KeySym params.KeySym = nil - sentMessage := NewSentMessage(params) + sentMessage, err := NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } env, err := sentMessage.Wrap(params) if err != nil { t.Fatalf("failed Wrap with seed %d: %s.", seed, err) } - msg := env.Open(f) if msg == nil { t.Fatalf("failed to open with seed %d.", seed) @@ -506,7 +519,10 @@ func generateCompatibeEnvelope(t *testing.T, f *Filter) *Envelope { params.KeySym = f.KeySym params.Topic = BytesToTopic(f.Topics[2]) - sentMessage := NewSentMessage(params) + sentMessage, err := NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } env, err := sentMessage.Wrap(params) if err != nil { t.Fatalf("failed Wrap with seed %d: %s.", seed, err) @@ -678,7 +694,10 @@ func TestVariableTopics(t *testing.T) { if err != nil { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - msg := NewSentMessage(params) + msg, err := NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } env, err := msg.Wrap(params) if err != nil { t.Fatalf("failed Wrap with seed %d: %s.", seed, err) diff --git a/whisper/whisperv5/message.go b/whisper/whisperv5/message.go index 7fa3b9435d..f12b048ecf 100644 --- a/whisper/whisperv5/message.go +++ b/whisper/whisperv5/message.go @@ -23,14 +23,14 @@ import ( "crypto/cipher" "crypto/ecdsa" crand "crypto/rand" - "crypto/sha256" + "encoding/binary" "errors" + "strconv" "github.com/expanse-org/go-expanse/common" "github.com/expanse-org/go-expanse/crypto" "github.com/expanse-org/go-expanse/crypto/ecies" "github.com/expanse-org/go-expanse/log" - "golang.org/x/crypto/pbkdf2" ) // Options specifies the exact way a message should be wrapped into an Envelope. @@ -86,58 +86,76 @@ func (msg *ReceivedMessage) isAsymmetricEncryption() bool { return msg.Dst != nil } -func DeriveOneTimeKey(key []byte, salt []byte, version uint64) ([]byte, error) { - if version == 0 { - derivedKey := pbkdf2.Key(key, salt, 8, aesKeyLength, sha256.New) - return derivedKey, nil - } else { - return nil, unknownVersionError(version) - } -} - // NewMessage creates and initializes a non-signed, non-encrypted Whisper message. -func NewSentMessage(params *MessageParams) *SentMessage { +func NewSentMessage(params *MessageParams) (*SentMessage, error) { msg := SentMessage{} - msg.Raw = make([]byte, 1, len(params.Payload)+len(params.Payload)+signatureLength+padSizeLimitUpper) + msg.Raw = make([]byte, 1, len(params.Payload)+len(params.Padding)+signatureLength+padSizeLimit) msg.Raw[0] = 0 // set all the flags to zero err := msg.appendPadding(params) if err != nil { - log.Error("failed to create NewSentMessage", "err", err) - return nil + return nil, err } msg.Raw = append(msg.Raw, params.Payload...) - return &msg + return &msg, nil +} + +// getSizeOfLength returns the number of bytes necessary to encode the entire size padding (including these bytes) +func getSizeOfLength(b []byte) (sz int, err error) { + sz = intSize(len(b)) // first iteration + sz = intSize(len(b) + sz) // second iteration + if sz > 3 { + err = errors.New("oversized padding parameter") + } + return sz, err +} + +// sizeOfIntSize returns minimal number of bytes necessary to encode an integer value +func intSize(i int) (s int) { + for s = 1; i >= 256; s++ { + i /= 256 + } + return s } // appendPadding appends the pseudorandom padding bytes and sets the padding flag. // The last byte contains the size of padding (thus, its size must not exceed 256). func (msg *SentMessage) appendPadding(params *MessageParams) error { - total := len(params.Payload) + 1 + rawSize := len(params.Payload) + 1 if params.Src != nil { - total += signatureLength + rawSize += signatureLength } - padChunk := padSizeLimitUpper - if total <= padSizeLimitLower { - padChunk = padSizeLimitLower - } - odd := total % padChunk - if odd > 0 { - padSize := padChunk - odd - if padSize > 255 { - // this algorithm is only valid if padSizeLimitUpper <= 256. - // if padSizeLimitUpper will ever change, please fix the algorithm - // (for more information see ReceivedMessage.extractPadding() function). + odd := rawSize % padSizeLimit + + if len(params.Padding) != 0 { + padSize := len(params.Padding) + padLengthSize, err := getSizeOfLength(params.Padding) + if err != nil { + return err + } + totalPadSize := padSize + padLengthSize + buf := make([]byte, 8) + binary.LittleEndian.PutUint32(buf, uint32(totalPadSize)) + buf = buf[:padLengthSize] + msg.Raw = append(msg.Raw, buf...) + msg.Raw = append(msg.Raw, params.Padding...) + msg.Raw[0] |= byte(padLengthSize) // number of bytes indicating the padding size + } else if odd != 0 { + totalPadSize := padSizeLimit - odd + if totalPadSize > 255 { + // this algorithm is only valid if padSizeLimit < 256. + // if padSizeLimit will ever change, please fix the algorithm + // (please see also ReceivedMessage.extractPadding() function). panic("please fix the padding algorithm before releasing new version") } - buf := make([]byte, padSize) + buf := make([]byte, totalPadSize) _, err := crand.Read(buf[1:]) if err != nil { return err } - buf[0] = byte(padSize) - if params.Padding != nil { - copy(buf[1:], params.Padding) + if totalPadSize > 6 && !validateSymmetricKey(buf) { + return errors.New("failed to generate random padding of size " + strconv.Itoa(totalPadSize)) } + buf[0] = byte(totalPadSize) msg.Raw = append(msg.Raw, buf...) msg.Raw[0] |= byte(0x1) // number of bytes indicating the padding size } @@ -178,46 +196,31 @@ func (msg *SentMessage) encryptAsymmetric(key *ecdsa.PublicKey) error { // encryptSymmetric encrypts a message with a topic key, using AES-GCM-256. // nonce size should be 12 bytes (see cipher.gcmStandardNonceSize). -func (msg *SentMessage) encryptSymmetric(key []byte) (salt []byte, nonce []byte, err error) { +func (msg *SentMessage) encryptSymmetric(key []byte) (nonce []byte, err error) { if !validateSymmetricKey(key) { - return nil, nil, errors.New("invalid key provided for symmetric encryption") + return nil, errors.New("invalid key provided for symmetric encryption") } - salt = make([]byte, saltLength) - _, err = crand.Read(salt) + block, err := aes.NewCipher(key) if err != nil { - return nil, nil, err - } else if !validateSymmetricKey(salt) { - return nil, nil, errors.New("crypto/rand failed to generate salt") - } - - derivedKey, err := DeriveOneTimeKey(key, salt, EnvelopeVersion) - if err != nil { - return nil, nil, err - } - if !validateSymmetricKey(derivedKey) { - return nil, nil, errors.New("failed to derive one-time key") - } - block, err := aes.NewCipher(derivedKey) - if err != nil { - return nil, nil, err + return nil, err } aesgcm, err := cipher.NewGCM(block) if err != nil { - return nil, nil, err + return nil, err } // never use more than 2^32 random nonces with a given key nonce = make([]byte, aesgcm.NonceSize()) _, err = crand.Read(nonce) if err != nil { - return nil, nil, err + return nil, err } else if !validateSymmetricKey(nonce) { - return nil, nil, errors.New("crypto/rand failed to generate nonce") + return nil, errors.New("crypto/rand failed to generate nonce") } msg.Raw = aesgcm.Seal(nil, nonce, msg.Raw, nil) - return salt, nonce, nil + return nonce, nil } // Wrap bundles the message into an Envelope to transmit over the network. @@ -231,11 +234,11 @@ func (msg *SentMessage) Wrap(options *MessageParams) (envelope *Envelope, err er return nil, err } } - var salt, nonce []byte + var nonce []byte if options.Dst != nil { err = msg.encryptAsymmetric(options.Dst) } else if options.KeySym != nil { - salt, nonce, err = msg.encryptSymmetric(options.KeySym) + nonce, err = msg.encryptSymmetric(options.KeySym) } else { err = errors.New("unable to encrypt the message: neither symmetric nor assymmetric key provided") } @@ -244,7 +247,7 @@ func (msg *SentMessage) Wrap(options *MessageParams) (envelope *Envelope, err er return nil, err } - envelope = NewEnvelope(options.TTL, options.Topic, salt, nonce, msg) + envelope = NewEnvelope(options.TTL, options.Topic, nonce, msg) err = envelope.Seal(options) if err != nil { return nil, err @@ -254,13 +257,8 @@ func (msg *SentMessage) Wrap(options *MessageParams) (envelope *Envelope, err er // decryptSymmetric decrypts a message with a topic key, using AES-GCM-256. // nonce size should be 12 bytes (see cipher.gcmStandardNonceSize). -func (msg *ReceivedMessage) decryptSymmetric(key []byte, salt []byte, nonce []byte) error { - derivedKey, err := DeriveOneTimeKey(key, salt, msg.EnvelopeVersion) - if err != nil { - return err - } - - block, err := aes.NewCipher(derivedKey) +func (msg *ReceivedMessage) decryptSymmetric(key []byte, nonce []byte) error { + block, err := aes.NewCipher(key) if err != nil { return err } @@ -323,7 +321,8 @@ func (msg *ReceivedMessage) Validate() bool { // can be successfully decrypted. func (msg *ReceivedMessage) extractPadding(end int) (int, bool) { paddingSize := 0 - sz := int(msg.Raw[0] & paddingMask) // number of bytes containing the entire size of padding, could be zero + sz := int(msg.Raw[0] & paddingMask) // number of bytes indicating the entire size of padding (including these bytes) + // could be zero -- it means no padding if sz != 0 { paddingSize = int(bytesToUintLittleEndian(msg.Raw[1 : 1+sz])) if paddingSize < sz || paddingSize+1 > end { diff --git a/whisper/whisperv5/message_test.go b/whisper/whisperv5/message_test.go index 9e04cb49ca..7a7e84fe9e 100644 --- a/whisper/whisperv5/message_test.go +++ b/whisper/whisperv5/message_test.go @@ -31,9 +31,9 @@ func copyFromBuf(dst []byte, src []byte, beg int) int { } func generateMessageParams() (*MessageParams, error) { - // set all the parameters except p.Dst + // set all the parameters except p.Dst and p.Padding - buf := make([]byte, 1024) + buf := make([]byte, 4) mrand.Read(buf) sz := mrand.Intn(400) @@ -42,14 +42,10 @@ func generateMessageParams() (*MessageParams, error) { p.WorkTime = 1 p.TTL = uint32(mrand.Intn(1024)) p.Payload = make([]byte, sz) - p.Padding = make([]byte, padSizeLimitUpper) p.KeySym = make([]byte, aesKeyLength) - - var b int - b = copyFromBuf(p.Payload, buf, b) - b = copyFromBuf(p.Padding, buf, b) - b = copyFromBuf(p.KeySym, buf, b) - p.Topic = BytesToTopic(buf[b:]) + mrand.Read(p.Payload) + mrand.Read(p.KeySym) + p.Topic = BytesToTopic(buf) var err error p.Src, err = crypto.GenerateKey() @@ -77,11 +73,12 @@ func singleMessageTest(t *testing.T, symmetric bool) { } text := make([]byte, 0, 512) - steg := make([]byte, 0, 512) text = append(text, params.Payload...) - steg = append(steg, params.Padding...) - msg := NewSentMessage(params) + msg, err := NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } env, err := msg.Wrap(params) if err != nil { t.Fatalf("failed Wrap with seed %d: %s.", seed, err) @@ -102,10 +99,6 @@ func singleMessageTest(t *testing.T, symmetric bool) { t.Fatalf("failed to validate with seed %d.", seed) } - padsz := len(decrypted.Padding) - if !bytes.Equal(steg[:padsz], decrypted.Padding) { - t.Fatalf("failed with seed %d: compare padding.", seed) - } if !bytes.Equal(text, decrypted.Payload) { t.Fatalf("failed with seed %d: compare payload.", seed) } @@ -140,7 +133,10 @@ func TestMessageWrap(t *testing.T) { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - msg := NewSentMessage(params) + msg, err := NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } params.TTL = 1 params.WorkTime = 12 params.PoW = target @@ -155,7 +151,10 @@ func TestMessageWrap(t *testing.T) { } // set PoW target too high, expect error - msg2 := NewSentMessage(params) + msg2, err := NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } params.TTL = 1000000 params.WorkTime = 1 params.PoW = 10000000.0 @@ -175,14 +174,15 @@ func TestMessageSeal(t *testing.T) { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - msg := NewSentMessage(params) + msg, err := NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } params.TTL = 1 aesnonce := make([]byte, 12) - salt := make([]byte, 12) mrand.Read(aesnonce) - mrand.Read(salt) - env := NewEnvelope(params.TTL, params.Topic, salt, aesnonce, msg) + env := NewEnvelope(params.TTL, params.Topic, aesnonce, msg) if err != nil { t.Fatalf("failed Wrap with seed %d: %s.", seed, err) } @@ -236,11 +236,12 @@ func singleEnvelopeOpenTest(t *testing.T, symmetric bool) { } text := make([]byte, 0, 512) - steg := make([]byte, 0, 512) text = append(text, params.Payload...) - steg = append(steg, params.Padding...) - msg := NewSentMessage(params) + msg, err := NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } env, err := msg.Wrap(params) if err != nil { t.Fatalf("failed Wrap with seed %d: %s.", seed, err) @@ -252,10 +253,6 @@ func singleEnvelopeOpenTest(t *testing.T, symmetric bool) { t.Fatalf("failed to open with seed %d.", seed) } - padsz := len(decrypted.Padding) - if !bytes.Equal(steg[:padsz], decrypted.Padding) { - t.Fatalf("failed with seed %d: compare padding.", seed) - } if !bytes.Equal(text, decrypted.Payload) { t.Fatalf("failed with seed %d: compare payload.", seed) } @@ -291,21 +288,38 @@ func TestEncryptWithZeroKey(t *testing.T) { if err != nil { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - - msg := NewSentMessage(params) - + msg, err := NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } params.KeySym = make([]byte, aesKeyLength) _, err = msg.Wrap(params) if err == nil { t.Fatalf("wrapped with zero key, seed: %d.", seed) } + params, err = generateMessageParams() + if err != nil { + t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) + } + msg, err = NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } params.KeySym = make([]byte, 0) _, err = msg.Wrap(params) if err == nil { t.Fatalf("wrapped with empty key, seed: %d.", seed) } + params, err = generateMessageParams() + if err != nil { + t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) + } + msg, err = NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } params.KeySym = nil _, err = msg.Wrap(params) if err == nil { @@ -320,7 +334,10 @@ func TestRlpEncode(t *testing.T) { if err != nil { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - msg := NewSentMessage(params) + msg, err := NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } env, err := msg.Wrap(params) if err != nil { t.Fatalf("wrapped with zero key, seed: %d.", seed) @@ -344,3 +361,60 @@ func TestRlpEncode(t *testing.T) { t.Fatalf("Hashes are not equal: %x vs. %x", he, hd) } } + +func singlePaddingTest(t *testing.T, padSize int) { + params, err := generateMessageParams() + if err != nil { + t.Fatalf("failed generateMessageParams with seed %d and sz=%d: %s.", seed, padSize, err) + } + params.Padding = make([]byte, padSize) + params.PoW = 0.0000000001 + pad := make([]byte, padSize) + _, err = mrand.Read(pad) + if err != nil { + t.Fatalf("padding is not generated (seed %d): %s", seed, err) + } + n := copy(params.Padding, pad) + if n != padSize { + t.Fatalf("padding is not copied (seed %d): %s", seed, err) + } + msg, err := NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } + env, err := msg.Wrap(params) + if err != nil { + t.Fatalf("failed to wrap, seed: %d and sz=%d.", seed, padSize) + } + f := Filter{KeySym: params.KeySym} + decrypted := env.Open(&f) + if decrypted == nil { + t.Fatalf("failed to open, seed and sz=%d: %d.", seed, padSize) + } + if !bytes.Equal(pad, decrypted.Padding) { + t.Fatalf("padding is not retireved as expected with seed %d and sz=%d:\n[%x]\n[%x].", seed, padSize, pad, decrypted.Padding) + } +} + +func TestPadding(t *testing.T) { + InitSingleTest() + + for i := 1; i < 260; i++ { + singlePaddingTest(t, i) + } + + lim := 256 * 256 + for i := lim - 5; i < lim+2; i++ { + singlePaddingTest(t, i) + } + + for i := 0; i < 256; i++ { + n := mrand.Intn(256*254) + 256 + singlePaddingTest(t, n) + } + + for i := 0; i < 256; i++ { + n := mrand.Intn(256*1024) + 256*256 + singlePaddingTest(t, n) + } +} diff --git a/whisper/whisperv5/peer.go b/whisper/whisperv5/peer.go index 5c3c744ce3..5c0eacf620 100644 --- a/whisper/whisperv5/peer.go +++ b/whisper/whisperv5/peer.go @@ -149,23 +149,22 @@ func (peer *Peer) expire() { // broadcast iterates over the collection of envelopes and transmits yet unknown // ones over the network. func (p *Peer) broadcast() error { - // Fetch the envelopes and collect the unknown ones + var cnt int envelopes := p.host.Envelopes() - transmit := make([]*Envelope, 0, len(envelopes)) for _, envelope := range envelopes { if !p.marked(envelope) { - transmit = append(transmit, envelope) - p.mark(envelope) + err := p2p.Send(p.ws, messagesCode, envelope) + if err != nil { + return err + } else { + p.mark(envelope) + cnt++ + } } } - if len(transmit) == 0 { - return nil + if cnt > 0 { + log.Trace("broadcast", "num. messages", cnt) } - // Transmit the unknown batch (potentially empty) - if err := p2p.Send(p.ws, messagesCode, transmit); err != nil { - return err - } - log.Trace("broadcast", "num. messages", len(transmit)) return nil } diff --git a/whisper/whisperv5/peer_test.go b/whisper/whisperv5/peer_test.go index 0987064b82..ca27b3f15d 100644 --- a/whisper/whisperv5/peer_test.go +++ b/whisper/whisperv5/peer_test.go @@ -265,7 +265,10 @@ func sendMsg(t *testing.T, expected bool, id int) { opt.Payload = opt.Payload[1:] } - msg := NewSentMessage(&opt) + msg, err := NewSentMessage(&opt) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } envelope, err := msg.Wrap(&opt) if err != nil { t.Fatalf("failed to seal message: %s", err) @@ -286,7 +289,10 @@ func TestPeerBasic(t *testing.T) { } params.PoW = 0.001 - msg := NewSentMessage(params) + msg, err := NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } env, err := msg.Wrap(params) if err != nil { t.Fatalf("failed Wrap with seed %d.", seed) diff --git a/whisper/whisperv5/whisper.go b/whisper/whisperv5/whisper.go index 78abed778b..5fdc1d0132 100644 --- a/whisper/whisperv5/whisper.go +++ b/whisper/whisperv5/whisper.go @@ -263,24 +263,14 @@ func (w *Whisper) GetPrivateKey(id string) (*ecdsa.PrivateKey, error) { // GenerateSymKey generates a random symmetric key and stores it under id, // which is then returned. Will be used in the future for session key exchange. func (w *Whisper) GenerateSymKey() (string, error) { - const size = aesKeyLength * 2 - buf := make([]byte, size) - _, err := crand.Read(buf) + key := make([]byte, aesKeyLength) + _, err := crand.Read(key) if err != nil { return "", err - } else if !validateSymmetricKey(buf) { + } else if !validateSymmetricKey(key) { return "", fmt.Errorf("error in GenerateSymKey: crypto/rand failed to generate random data") } - key := buf[:aesKeyLength] - salt := buf[aesKeyLength:] - derived, err := DeriveOneTimeKey(key, salt, EnvelopeVersion) - if err != nil { - return "", err - } else if !validateSymmetricKey(derived) { - return "", fmt.Errorf("failed to derive valid key") - } - id, err := GenerateRandomID() if err != nil { return "", fmt.Errorf("failed to generate ID: %s", err) @@ -292,7 +282,7 @@ func (w *Whisper) GenerateSymKey() (string, error) { if w.symKeys[id] != nil { return "", fmt.Errorf("failed to generate unique ID") } - w.symKeys[id] = derived + w.symKeys[id] = key return id, nil } @@ -396,6 +386,9 @@ func (w *Whisper) Unsubscribe(id string) error { // network in the coming cycles. func (w *Whisper) Send(envelope *Envelope) error { ok, err := w.add(envelope) + if err != nil { + return err + } if !ok { return fmt.Errorf("failed to add envelope") } @@ -470,21 +463,18 @@ func (wh *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error { log.Warn("unxepected status message received", "peer", p.peer.ID()) case messagesCode: // decode the contained envelopes - var envelopes []*Envelope - if err := packet.Decode(&envelopes); err != nil { + var envelope Envelope + if err := packet.Decode(&envelope); err != nil { log.Warn("failed to decode envelope, peer will be disconnected", "peer", p.peer.ID(), "err", err) return errors.New("invalid envelope") } - // inject all envelopes into the internal pool - for _, envelope := range envelopes { - cached, err := wh.add(envelope) - if err != nil { - log.Warn("bad envelope received, peer will be disconnected", "peer", p.peer.ID(), "err", err) - return errors.New("invalid envelope") - } - if cached { - p.mark(envelope) - } + cached, err := wh.add(&envelope) + if err != nil { + log.Warn("bad envelope received, peer will be disconnected", "peer", p.peer.ID(), "err", err) + return errors.New("invalid envelope") + } + if cached { + p.mark(&envelope) } case p2pCode: // peer-to-peer message, sent directly to peer bypassing PoW checks, etc. @@ -551,14 +541,11 @@ func (wh *Whisper) add(envelope *Envelope) (bool, error) { return false, fmt.Errorf("oversized version [%x]", envelope.Hash()) } - if len(envelope.AESNonce) > AESNonceMaxLength { - // the standard AES GSM nonce size is 12, - // but const gcmStandardNonceSize cannot be accessed directly - return false, fmt.Errorf("oversized AESNonce [%x]", envelope.Hash()) - } - - if len(envelope.Salt) > saltLength { - return false, fmt.Errorf("oversized salt [%x]", envelope.Hash()) + aesNonceSize := len(envelope.AESNonce) + if aesNonceSize != 0 && aesNonceSize != AESNonceLength { + // the standard AES GCM nonce size is 12 bytes, + // but constant gcmStandardNonceSize cannot be accessed (not exported) + return false, fmt.Errorf("wrong size of AESNonce: %d bytes [env: %x]", aesNonceSize, envelope.Hash()) } if envelope.PoW() < wh.minPoW { diff --git a/whisper/whisperv5/whisper_test.go b/whisper/whisperv5/whisper_test.go index d5668259e5..225728c42e 100644 --- a/whisper/whisperv5/whisper_test.go +++ b/whisper/whisperv5/whisper_test.go @@ -455,7 +455,10 @@ func TestExpiry(t *testing.T) { } params.TTL = 1 - msg := NewSentMessage(params) + msg, err := NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } env, err := msg.Wrap(params) if err != nil { t.Fatalf("failed Wrap with seed %d: %s.", seed, err) @@ -515,7 +518,10 @@ func TestCustomization(t *testing.T) { params.Topic = BytesToTopic(f.Topics[2]) params.PoW = smallPoW params.TTL = 3600 * 24 // one day - msg := NewSentMessage(params) + msg, err := NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } env, err := msg.Wrap(params) if err != nil { t.Fatalf("failed Wrap with seed %d: %s.", seed, err) @@ -533,7 +539,10 @@ func TestCustomization(t *testing.T) { } params.TTL++ - msg = NewSentMessage(params) + msg, err = NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } env, err = msg.Wrap(params) if err != nil { t.Fatalf("failed Wrap with seed %d: %s.", seed, err)