forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-20 10:52:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge pull request #5 from ethereum/master

Browse source 
update
This commit is contained in:
jkcomment 2019-02-25 14:49:55 +09:00 committed by GitHub
commit c3ced0bef1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
559 changed files with 101545 additions and 20114 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
.travis.yml
View file

@ -53,7 +53,7 @@ matrix:
- go run build/ci.go lint
# This builder does the Ubuntu PPA upload
- if: type = push
- if: repo = ethereum/go-ethereum AND type = push
os: linux
dist: trusty
go: 1.11.x
@ -68,11 +68,14 @@ matrix:
- debhelper
- dput
- fakeroot
- python-bzrlib
- python-paramiko
script:
- go run build/ci.go debsrc -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>" -upload ppa:ethereum/ethereum
- echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
- go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
# This builder does the Linux Azure uploads
- if: type = push
- if: repo = ethereum/go-ethereum AND type = push
os: linux
dist: trusty
sudo: required
@ -106,7 +109,7 @@ matrix:
- go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
# This builder does the Linux Azure MIPS xgo uploads
- if: type = push
- if: repo = ethereum/go-ethereum AND type = push
os: linux
dist: trusty
services:
@ -134,7 +137,7 @@ matrix:
- go run build/ci.go archive -arch mips64le -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
# This builder does the Android Maven and Azure uploads
- if: type = push
- if: repo = ethereum/go-ethereum AND type = push
os: linux
dist: trusty
addons:
@ -168,11 +171,11 @@ matrix:
- export ANDROID_NDK=$HOME/android-ndk-r17b
- mkdir -p $GOPATH/src/github.com/ethereum
- ln -s `pwd` $GOPATH/src/github.com/ethereum
- ln -s `pwd` $GOPATH/src/github.com/ethereum/go-ethereum
- go run build/ci.go aar -signer ANDROID_SIGNING_KEY -deploy https://oss.sonatype.org -upload gethstore/builds
# This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads
- if: type = push
- if: repo = ethereum/go-ethereum AND type = push
os: osx
go: 1.11.x
env:
@ -201,7 +204,7 @@ matrix:
- go run build/ci.go xcode -signer IOS_SIGNING_KEY -deploy trunk -upload gethstore/builds
# This builder does the Azure archive purges to avoid accumulating junk
- if: type = cron
- if: repo = ethereum/go-ethereum AND type = cron
os: linux
dist: trusty
go: 1.11.x
@ -211,3 +214,12 @@ matrix:
submodules: false # avoid cloning ethereum/tests
script:
- go run build/ci.go purge -store gethstore/builds -days 14
- name: Race Detector for Swarm
if: repo = ethersphere/go-ethereum
os: linux
dist: trusty
go: 1.11.x
git:
submodules: false # avoid cloning ethereum/tests
script: ./build/travis_keepalive.sh go test -v -timeout 20m -race ./swarm...

26
README.md
View file

@ -1,6 +1,6 @@
## Go Ethereum
Official golang implementation of the Ethereum protocol.
Official Golang implementation of the Ethereum protocol.
[![API Reference](
https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667
@ -35,11 +35,11 @@ The go-ethereum project comes with several wrappers/executables found in the `cm
| Command | Description |
|:----------:|-------------|
| **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default), archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI Wiki page](https://github.com/ethereum/go-ethereum/wiki/Command-Line-Options) for command line options. |
| `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI) with expanded functionality if the contract bytecode is also available. However it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://github.com/ethereum/go-ethereum/wiki/Native-DApps:-Go-bindings-to-Ethereum-contracts) wiki page for details. |
| `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI) with expanded functionality if the contract bytecode is also available. However, it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://github.com/ethereum/go-ethereum/wiki/Native-DApps:-Go-bindings-to-Ethereum-contracts) wiki page for details. |
| `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks. |
| `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug`). |
| `gethrpctest` | Developer utility tool to support our [ethereum/rpc-test](https://github.com/ethereum/rpc-tests) test suite which validates baseline conformity to the [Ethereum JSON RPC](https://github.com/ethereum/wiki/wiki/JSON-RPC) specs. Please see the [test suite's readme](https://github.com/ethereum/rpc-tests/blob/master/README.md) for details. |
| `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://github.com/ethereum/wiki/wiki/RLP)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). |
| `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://github.com/ethereum/wiki/wiki/RLP)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). |
| `swarm` | Swarm daemon and tools. This is the entry point for the Swarm network. `swarm --help` for command line options and subcommands. See [Swarm README](https://github.com/ethereum/go-ethereum/tree/master/swarm) for more information. |
| `puppeth` | a CLI wizard that aids in creating a new Ethereum network. |
@ -72,7 +72,7 @@ This command will:
This tool is optional and if you leave it out you can always attach to an already running Geth instance
with `geth attach`.
### Full node on the Ethereum test network
### A Full node on the Ethereum test network
Transitioning towards developers, if you'd like to play around with creating Ethereum contracts, you
almost certainly would like to do that without any real money involved until you get the hang of the
@ -83,10 +83,10 @@ network with your node, which is fully equivalent to the main network, but with
$ geth --testnet console
```
The `console` subcommand have the exact same meaning as above and they are equally useful on the
testnet too. Please see above for their explanations if you've skipped to here.
The `console` subcommand has the exact same meaning as above and they are equally useful on the
testnet too. Please see above for their explanations if you've skipped here.
Specifying the `--testnet` flag however will reconfigure your Geth instance a bit:
Specifying the `--testnet` flag, however, will reconfigure your Geth instance a bit:
* Instead of using the default data directory (`~/.ethereum` on Linux for example), Geth will nest
itself one level deeper into a `testnet` subfolder (`~/.ethereum/testnet` on Linux). Note, on OSX
@ -103,7 +103,7 @@ separate the two networks and will not make any accounts available between them.
### Full node on the Rinkeby test network
The above test network is a cross client one based on the ethash proof-of-work consensus algorithm. As such, it has certain extra overhead and is more susceptible to reorganization attacks due to the network's low difficulty / security. Go Ethereum also supports connecting to a proof-of-authority based test network called [*Rinkeby*](https://www.rinkeby.io) (operated by members of the community). This network is lighter, more secure, but is only supported by go-ethereum.
The above test network is a cross-client one based on the ethash proof-of-work consensus algorithm. As such, it has certain extra overhead and is more susceptible to reorganization attacks due to the network's low difficulty/security. Go Ethereum also supports connecting to a proof-of-authority based test network called [*Rinkeby*](https://www.rinkeby.io) (operated by members of the community). This network is lighter, more secure, but is only supported by go-ethereum.
```
$ geth --rinkeby console
@ -139,13 +139,13 @@ This will start geth in fast-sync mode with a DB memory allowance of 1GB just as
Do not forget `--rpcaddr 0.0.0.0`, if you want to access RPC from other containers and/or hosts. By default, `geth` binds to the local interface and RPC endpoints is not accessible from the outside.
### Programatically interfacing Geth nodes
### Programmatically interfacing Geth nodes
As a developer, sooner rather than later you'll want to start interacting with Geth and the Ethereum
network via your own programs and not manually through the console. To aid this, Geth has built-in
support for a JSON-RPC based APIs ([standard APIs](https://github.com/ethereum/wiki/wiki/JSON-RPC) and
[Geth specific APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs)). These can be
exposed via HTTP, WebSockets and IPC (unix sockets on unix based platforms, and named pipes on Windows).
exposed via HTTP, WebSockets and IPC (UNIX sockets on UNIX based platforms, and named pipes on Windows).
The IPC interface is enabled by default and exposes all the APIs supported by Geth, whereas the HTTP
and WS interfaces need to manually be enabled and only expose a subset of APIs due to security reasons.
@ -241,7 +241,7 @@ that other nodes can use to connect to it and exchange peer information. Make su
displayed IP address information (most probably `[::]`) with your externally accessible IP to get the
actual `enode` URL.
*Note: You could also use a full fledged Geth node as a bootnode, but it's the less recommended way.*
*Note: You could also use a full-fledged Geth node as a bootnode, but it's the less recommended way.*
#### Starting up your member nodes
@ -264,7 +264,7 @@ an OpenCL or CUDA enabled `ethminer` instance. For information on such a setup,
[EtherMining subreddit](https://www.reddit.com/r/EtherMining/) and the [Genoil miner](https://github.com/Genoil/cpp-ethereum)
repository.
In a private network setting however, a single CPU miner instance is more than enough for practical
In a private network setting, however a single CPU miner instance is more than enough for practical
purposes as it can produce a stable stream of blocks at the correct intervals without needing heavy
resources (consider running on a single thread, no need for multiple ones either). To start a Geth
instance for mining, run it with all your usual flags, extended by:
@ -298,7 +298,7 @@ Please make sure your contributions adhere to our coding guidelines:
* E.g. "eth, rpc: make trace configs optional"
Please see the [Developers' Guide](https://github.com/ethereum/go-ethereum/wiki/Developers'-Guide)
for more details on configuring your environment, managing project dependencies and testing procedures.
for more details on configuring your environment, managing project dependencies, and testing procedures.
## License

2
accounts/abi/abi.go
View file

@ -136,7 +136,7 @@ func (abi *ABI) UnmarshalJSON(data []byte) error {
// returns nil if none found
func (abi *ABI) MethodById(sigdata []byte) (*Method, error) {
if len(sigdata) < 4 {
return nil, fmt.Errorf("data too short (% bytes) for abi method lookup", len(sigdata))
return nil, fmt.Errorf("data too short (%d bytes) for abi method lookup", len(sigdata))
}
for _, method := range abi.Methods {
if bytes.Equal(method.Id(), sigdata[:4]) {

19
accounts/abi/bind/backends/simulated.go
View file

@ -164,6 +164,25 @@ func (b *SimulatedBackend) TransactionReceipt(ctx context.Context, txHash common
return receipt, nil
}
// TransactionByHash checks the pool of pending transactions in addition to the
// blockchain. The isPending return value indicates whether the transaction has been
// mined yet. Note that the transaction may not be part of the canonical chain even if
// it's not pending.
func (b *SimulatedBackend) TransactionByHash(ctx context.Context, txHash common.Hash) (*types.Transaction, bool, error) {
b.mu.Lock()
defer b.mu.Unlock()
tx := b.pendingBlock.Transaction(txHash)
if tx != nil {
return tx, true, nil
}
tx, _, _, _ = rawdb.ReadTransaction(b.database, txHash)
if tx != nil {
return tx, false, nil
}
return nil, false, ethereum.NotFound
}
// PendingCodeAt returns the code associated with an account in the pending state.
func (b *SimulatedBackend) PendingCodeAt(ctx context.Context, contract common.Address) ([]byte, error) {
b.mu.Lock()

82
accounts/abi/bind/backends/simulated_test.go Normal file
View file

@ -0,0 +1,82 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package backends_test
import (
"context"
"math/big"
"testing"
ethereum "github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/accounts/abi/bind"
"github.com/ethereum/go-ethereum/accounts/abi/bind/backends"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
)
func TestSimulatedBackend(t *testing.T) {
var gasLimit uint64 = 8000029
key, _ := crypto.GenerateKey() // nolint: gosec
auth := bind.NewKeyedTransactor(key)
genAlloc := make(core.GenesisAlloc)
genAlloc[auth.From] = core.GenesisAccount{Balance: big.NewInt(9223372036854775807)}
sim := backends.NewSimulatedBackend(genAlloc, gasLimit)
// should return an error if the tx is not found
txHash := common.HexToHash("2")
_, isPending, err := sim.TransactionByHash(context.Background(), txHash)
if isPending {
t.Fatal("transaction should not be pending")
}
if err != ethereum.NotFound {
t.Fatalf("err should be `ethereum.NotFound` but received %v", err)
}
// generate a transaction and confirm you can retrieve it
code := `6060604052600a8060106000396000f360606040526008565b00`
var gas uint64 = 3000000
tx := types.NewContractCreation(0, big.NewInt(0), gas, big.NewInt(1), common.FromHex(code))
tx, _ = types.SignTx(tx, types.HomesteadSigner{}, key)
err = sim.SendTransaction(context.Background(), tx)
if err != nil {
t.Fatal("error sending transaction")
}
txHash = tx.Hash()
_, isPending, err = sim.TransactionByHash(context.Background(), txHash)
if err != nil {
t.Fatalf("error getting transaction with hash: %v", txHash.String())
}
if !isPending {
t.Fatal("transaction should have pending status")
}
sim.Commit()
tx, isPending, err = sim.TransactionByHash(context.Background(), txHash)
if err != nil {
t.Fatalf("error getting transaction with hash: %v", txHash.String())
}
if isPending {
t.Fatal("transaction should not have pending status")
}
}

16
accounts/abi/bind/base_test.go
View file

@ -1,3 +1,19 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package bind_test
import (

73
accounts/accounts.go
View file

@ -18,12 +18,14 @@
package accounts
import (
"fmt"
"math/big"
ethereum "github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/event"
"golang.org/x/crypto/sha3"
)
// Account represents an Ethereum account located at a specific location defined
@ -33,6 +35,13 @@ type Account struct {
URL URL `json:"url"` // Optional resource locator within a backend
}
const (
MimetypeTextWithValidator = "text/validator"
MimetypeTypedData = "data/typed"
MimetypeClique = "application/x-clique-header"
MimetypeTextPlain = "text/plain"
)
// Wallet represents a software or hardware wallet that might contain one or more
// accounts (derived from the same seed).
type Wallet interface {
@ -87,8 +96,26 @@ type Wallet interface {
// chain state reader.
SelfDerive(base DerivationPath, chain ethereum.ChainStateReader)
// SignHash requests the wallet to sign the given hash.
// SignData requests the wallet to sign the hash of the given data
// It looks up the account specified either solely via its address contained within,
// or optionally with the aid of any location metadata from the embedded URL field.
//
// If the wallet requires additional authentication to sign the request (e.g.
// a password to decrypt the account, or a PIN code o verify the transaction),
// an AuthNeededError instance will be returned, containing infos for the user
// about which fields or actions are needed. The user may retry by providing
// the needed details via SignDataWithPassphrase, or by other means (e.g. unlock
// the account in a keystore).
SignData(account Account, mimeType string, data []byte) ([]byte, error)
// SignDataWithPassphrase is identical to SignData, but also takes a password
// NOTE: there's an chance that an erroneous call might mistake the two strings, and
// supply password in the mimetype field, or vice versa. Thus, an implementation
// should never echo the mimetype or return the mimetype in the error-response
SignDataWithPassphrase(account Account, passphrase, mimeType string, data []byte) ([]byte, error)
// SignText requests the wallet to sign the hash of a given piece of data, prefixed
// by the Ethereum prefix scheme
// It looks up the account specified either solely via its address contained within,
// or optionally with the aid of any location metadata from the embedded URL field.
//
@ -98,7 +125,10 @@ type Wallet interface {
// about which fields or actions are needed. The user may retry by providing
// the needed details via SignHashWithPassphrase, or by other means (e.g. unlock
// the account in a keystore).
SignHash(account Account, hash []byte) ([]byte, error)
SignText(account Account, text []byte) ([]byte, error)
// SignTextWithPassphrase is identical to Signtext, but also takes a password
SignTextWithPassphrase(account Account, passphrase string, hash []byte) ([]byte, error)
// SignTx requests the wallet to sign the given transaction.
//
@ -113,18 +143,7 @@ type Wallet interface {
// the account in a keystore).
SignTx(account Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error)
// SignHashWithPassphrase requests the wallet to sign the given hash with the
// given passphrase as extra authentication information.
//
// It looks up the account specified either solely via its address contained within,
// or optionally with the aid of any location metadata from the embedded URL field.
SignHashWithPassphrase(account Account, passphrase string, hash []byte) ([]byte, error)
// SignTxWithPassphrase requests the wallet to sign the given transaction, with the
// given passphrase as extra authentication information.
//
// It looks up the account specified either solely via its address contained within,
// or optionally with the aid of any location metadata from the embedded URL field.
// SignTxWithPassphrase is identical to SignTx, but also takes a password
SignTxWithPassphrase(account Account, passphrase string, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error)
}
@ -148,6 +167,32 @@ type Backend interface {
Subscribe(sink chan<- WalletEvent) event.Subscription
}
// TextHash is a helper function that calculates a hash for the given message that can be
// safely used to calculate a signature from.
//
// The hash is calulcated as
// keccak256("\x19Ethereum Signed Message:\n"${message length}${message}).
//
// This gives context to the signed message and prevents signing of transactions.
func TextHash(data []byte) []byte {
hash, _ := TextAndHash(data)
return hash
}
// TextAndHash is a helper function that calculates a hash for the given message that can be
// safely used to calculate a signature from.
//
// The hash is calulcated as
// keccak256("\x19Ethereum Signed Message:\n"${message length}${message}).
//
// This gives context to the signed message and prevents signing of transactions.
func TextAndHash(data []byte) ([]byte, string) {
msg := fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", len(data), string(data))
hasher := sha3.NewLegacyKeccak256()
hasher.Write([]byte(msg))
return hasher.Sum(nil), msg
}
// WalletEventType represents the different event types that can be fired by
// the wallet subscription subsystem.
type WalletEventType int

31
rpc/utils_test.go → accounts/accounts_test.go
View file

@ -1,4 +1,4 @@
// Copyright 2016 The go-ethereum Authors
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
@ -14,30 +14,19 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package rpc
package accounts
import (
"strings"
"bytes"
"testing"
"github.com/ethereum/go-ethereum/common/hexutil"
)
func TestNewID(t *testing.T) {
hexchars := "0123456789ABCDEFabcdef"
for i := 0; i < 100; i++ {
id := string(NewID())
if !strings.HasPrefix(id, "0x") {
t.Fatalf("invalid ID prefix, want '0x...', got %s", id)
}
id = id[2:]
if len(id) == 0 || len(id) > 32 {
t.Fatalf("invalid ID length, want len(id) > 0 && len(id) <= 32), got %d", len(id))
}
for i := 0; i < len(id); i++ {
if strings.IndexByte(hexchars, id[i]) == -1 {
t.Fatalf("unexpected byte, want any valid hex char, got %c", id[i])
}
}
func TestTextHash(t *testing.T) {
hash := TextHash([]byte("Hello Joe"))
want := hexutil.MustDecode("0xa080337ae51c4e064c189e113edd0ba391df9206e2f49db658bb32cf2911730b")
if !bytes.Equal(hash, want) {
t.Fatalf("wrong hash: %x", hash)
}
}

228
accounts/external/backend.go vendored Normal file
View file

@ -0,0 +1,228 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package external
import (
"fmt"
"math/big"
"sync"
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/accounts"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/internal/ethapi"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/signer/core"
)
type ExternalBackend struct {
signers []accounts.Wallet
}
func (eb *ExternalBackend) Wallets() []accounts.Wallet {
return eb.signers
}
func NewExternalBackend(endpoint string) (*ExternalBackend, error) {
signer, err := NewExternalSigner(endpoint)
if err != nil {
return nil, err
}
return &ExternalBackend{
signers: []accounts.Wallet{signer},
}, nil
}
func (eb *ExternalBackend) Subscribe(sink chan<- accounts.WalletEvent) event.Subscription {
return event.NewSubscription(func(quit <-chan struct{}) error {
<-quit
return nil
})
}
// ExternalSigner provides an API to interact with an external signer (clef)
// It proxies request to the external signer while forwarding relevant
// request headers
type ExternalSigner struct {
client *rpc.Client
endpoint string
status string
cacheMu sync.RWMutex
cache []accounts.Account
}
func NewExternalSigner(endpoint string) (*ExternalSigner, error) {
client, err := rpc.Dial(endpoint)
if err != nil {
return nil, err
}
extsigner := &ExternalSigner{
client: client,
endpoint: endpoint,
}
// Check if reachable
version, err := extsigner.pingVersion()
if err != nil {
return nil, err
}
extsigner.status = fmt.Sprintf("ok [version=%v]", version)
return extsigner, nil
}
func (api *ExternalSigner) URL() accounts.URL {
return accounts.URL{
Scheme: "extapi",
Path: api.endpoint,
}
}
func (api *ExternalSigner) Status() (string, error) {
return api.status, nil
}
func (api *ExternalSigner) Open(passphrase string) error {
return fmt.Errorf("operation not supported on external signers")
}
func (api *ExternalSigner) Close() error {
return fmt.Errorf("operation not supported on external signers")
}
func (api *ExternalSigner) Accounts() []accounts.Account {
var accnts []accounts.Account
res, err := api.listAccounts()
if err != nil {
log.Error("account listing failed", "error", err)
return accnts
}
for _, addr := range res {
accnts = append(accnts, accounts.Account{
URL: accounts.URL{
Scheme: "extapi",
Path: api.endpoint,
},
Address: addr,
})
}
api.cacheMu.Lock()
api.cache = accnts
api.cacheMu.Unlock()
return accnts
}
func (api *ExternalSigner) Contains(account accounts.Account) bool {
api.cacheMu.RLock()
defer api.cacheMu.RUnlock()
for _, a := range api.cache {
if a.Address == account.Address && (account.URL == (accounts.URL{}) || account.URL == api.URL()) {
return true
}
}
return false
}
func (api *ExternalSigner) Derive(path accounts.DerivationPath, pin bool) (accounts.Account, error) {
return accounts.Account{}, fmt.Errorf("operation not supported on external signers")
}
func (api *ExternalSigner) SelfDerive(base accounts.DerivationPath, chain ethereum.ChainStateReader) {
log.Error("operation SelfDerive not supported on external signers")
}
func (api *ExternalSigner) signHash(account accounts.Account, hash []byte) ([]byte, error) {
return []byte{}, fmt.Errorf("operation not supported on external signers")
}
// SignData signs keccak256(data). The mimetype parameter describes the type of data being signed
func (api *ExternalSigner) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) {
var res hexutil.Bytes
var signAddress = common.NewMixedcaseAddress(account.Address)
if err := api.client.Call(&res, "account_signData",
mimeType,
&signAddress, // Need to use the pointer here, because of how MarshalJSON is defined
hexutil.Encode(data)); err != nil {
return nil, err
}
// If V is on 27/28-form, convert to to 0/1 for Clique
if mimeType == accounts.MimetypeClique && (res[64] == 27 || res[64] == 28) {
res[64] -= 27 // Transform V from 27/28 to 0/1 for Clique use
}
return res, nil
}
func (api *ExternalSigner) SignText(account accounts.Account, text []byte) ([]byte, error) {
var res hexutil.Bytes
var signAddress = common.NewMixedcaseAddress(account.Address)
if err := api.client.Call(&res, "account_signData",
accounts.MimetypeTextPlain,
&signAddress, // Need to use the pointer here, because of how MarshalJSON is defined
hexutil.Encode(text)); err != nil {
return nil, err
}
return res, nil
}
func (api *ExternalSigner) SignTx(account accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
res := ethapi.SignTransactionResult{}
to := common.NewMixedcaseAddress(*tx.To())
data := hexutil.Bytes(tx.Data())
args := &core.SendTxArgs{
Data: &data,
Nonce: hexutil.Uint64(tx.Nonce()),
Value: hexutil.Big(*tx.Value()),
Gas: hexutil.Uint64(tx.Gas()),
GasPrice: hexutil.Big(*tx.GasPrice()),
To: &to,
From: common.NewMixedcaseAddress(account.Address),
}
if err := api.client.Call(&res, "account_signTransaction", args); err != nil {
return nil, err
}
return res.Tx, nil
}
func (api *ExternalSigner) SignTextWithPassphrase(account accounts.Account, passphrase string, text []byte) ([]byte, error) {
return []byte{}, fmt.Errorf("passphrase-operations not supported on external signers")
}
func (api *ExternalSigner) SignTxWithPassphrase(account accounts.Account, passphrase string, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
return nil, fmt.Errorf("passphrase-operations not supported on external signers")
}
func (api *ExternalSigner) SignDataWithPassphrase(account accounts.Account, passphrase, mimeType string, data []byte) ([]byte, error) {
return nil, fmt.Errorf("passphrase-operations not supported on external signers")
}
func (api *ExternalSigner) listAccounts() ([]common.Address, error) {
var res []common.Address
if err := api.client.Call(&res, "account_list"); err != nil {
return nil, err
}
return res, nil
}
func (api *ExternalSigner) pingVersion() (string, error) {
var v string
if err := api.client.Call(&v, "account_version"); err != nil {
return "", err
}
return v, nil
}

46
accounts/keystore/wallet.go
View file

@ -22,6 +22,7 @@ import (
ethereum "github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/accounts"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
)
// keystoreWallet implements the accounts.Wallet interface for the original
@ -78,11 +79,11 @@ func (w *keystoreWallet) Derive(path accounts.DerivationPath, pin bool) (account
// there is no notion of hierarchical account derivation for plain keystore accounts.
func (w *keystoreWallet) SelfDerive(base accounts.DerivationPath, chain ethereum.ChainStateReader) {}
// SignHash implements accounts.Wallet, attempting to sign the given hash with
// signHash attempts to sign the given hash with
// the given account. If the wallet does not wrap this particular account, an
// error is returned to avoid account leakage (even though in theory we may be
// able to sign via our shared keystore backend).
func (w *keystoreWallet) SignHash(account accounts.Account, hash []byte) ([]byte, error) {
func (w *keystoreWallet) signHash(account accounts.Account, hash []byte) ([]byte, error) {
// Make sure the requested account is contained within
if !w.Contains(account) {
return nil, accounts.ErrUnknownAccount
@ -91,6 +92,36 @@ func (w *keystoreWallet) SignHash(account accounts.Account, hash []byte) ([]byte
return w.keystore.SignHash(account, hash)
}
// SignData signs keccak256(data). The mimetype parameter describes the type of data being signed
func (w *keystoreWallet) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) {
return w.signHash(account, crypto.Keccak256(data))
}
// SignDataWithPassphrase signs keccak256(data). The mimetype parameter describes the type of data being signed
func (w *keystoreWallet) SignDataWithPassphrase(account accounts.Account, passphrase, mimeType string, data []byte) ([]byte, error) {
// Make sure the requested account is contained within
if !w.Contains(account) {
return nil, accounts.ErrUnknownAccount
}
// Account seems valid, request the keystore to sign
return w.keystore.SignHashWithPassphrase(account, passphrase, crypto.Keccak256(data))
}
func (w *keystoreWallet) SignText(account accounts.Account, text []byte) ([]byte, error) {
return w.signHash(account, accounts.TextHash(text))
}
// SignTextWithPassphrase implements accounts.Wallet, attempting to sign the
// given hash with the given account using passphrase as extra authentication.
func (w *keystoreWallet) SignTextWithPassphrase(account accounts.Account, passphrase string, text []byte) ([]byte, error) {
// Make sure the requested account is contained within
if !w.Contains(account) {
return nil, accounts.ErrUnknownAccount
}
// Account seems valid, request the keystore to sign
return w.keystore.SignHashWithPassphrase(account, passphrase, accounts.TextHash(text))
}
// SignTx implements accounts.Wallet, attempting to sign the given transaction
// with the given account. If the wallet does not wrap this particular account,
// an error is returned to avoid account leakage (even though in theory we may
@ -104,17 +135,6 @@ func (w *keystoreWallet) SignTx(account accounts.Account, tx *types.Transaction,
return w.keystore.SignTx(account, tx, chainID)
}
// SignHashWithPassphrase implements accounts.Wallet, attempting to sign the
// given hash with the given account using passphrase as extra authentication.
func (w *keystoreWallet) SignHashWithPassphrase(account accounts.Account, passphrase string, hash []byte) ([]byte, error) {
// Make sure the requested account is contained within
if !w.Contains(account) {
return nil, accounts.ErrUnknownAccount
}
// Account seems valid, request the keystore to sign
return w.keystore.SignHashWithPassphrase(account, passphrase, hash)
}
// SignTxWithPassphrase implements accounts.Wallet, attempting to sign the given
// transaction with the given account using passphrase as extra authentication.
func (w *keystoreWallet) SignTxWithPassphrase(account accounts.Account, passphrase string, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {

25
accounts/usbwallet/wallet.go
View file

@ -29,6 +29,7 @@ import (
"github.com/ethereum/go-ethereum/accounts"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/karalabe/hid"
)
@ -495,12 +496,28 @@ func (w *wallet) SelfDerive(base accounts.DerivationPath, chain ethereum.ChainSt
w.deriveChain = chain
}
// SignHash implements accounts.Wallet, however signing arbitrary data is not
// signHash implements accounts.Wallet, however signing arbitrary data is not
// supported for hardware wallets, so this method will always return an error.
func (w *wallet) SignHash(account accounts.Account, hash []byte) ([]byte, error) {
func (w *wallet) signHash(account accounts.Account, hash []byte) ([]byte, error) {
return nil, accounts.ErrNotSupported
}
// SignData signs keccak256(data). The mimetype parameter describes the type of data being signed
func (w *wallet) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) {
return w.signHash(account, crypto.Keccak256(data))
}
// SignDataWithPassphrase implements accounts.Wallet, attempting to sign the given
// data with the given account using passphrase as extra authentication.
// Since USB wallets don't rely on passphrases, these are silently ignored.
func (w *wallet) SignDataWithPassphrase(account accounts.Account, passphrase, mimeType string, data []byte) ([]byte, error) {
return w.SignData(account, mimeType, data)
}
func (w *wallet) SignText(account accounts.Account, text []byte) ([]byte, error) {
return w.signHash(account, accounts.TextHash(text))
}
// SignTx implements accounts.Wallet. It sends the transaction over to the Ledger
// wallet to request a confirmation from the user. It returns either the signed
// transaction or a failure if the user denied the transaction.
@ -550,8 +567,8 @@ func (w *wallet) SignTx(account accounts.Account, tx *types.Transaction, chainID
// SignHashWithPassphrase implements accounts.Wallet, however signing arbitrary
// data is not supported for Ledger wallets, so this method will always return
// an error.
func (w *wallet) SignHashWithPassphrase(account accounts.Account, passphrase string, hash []byte) ([]byte, error) {
return w.SignHash(account, hash)
func (w *wallet) SignTextWithPassphrase(account accounts.Account, passphrase string, text []byte) ([]byte, error) {
return w.SignText(account, accounts.TextHash(text))
}
// SignTxWithPassphrase implements accounts.Wallet, attempting to sign the given

13
build/ci-notes.md
View file

@ -7,11 +7,18 @@ Canonical.
Packages of develop branch commits have suffix -unstable and cannot be installed alongside
the stable version. Switching between release streams requires user intervention.
## Launchpad
The packages are built and served by launchpad.net. We generate a Debian source package
for each distribution and upload it. Their builder picks up the source package, builds it
and installs the new version into the PPA repository. Launchpad requires a valid signature
by a team member for source package uploads. The signing key is stored in an environment
variable which Travis CI makes available to certain builds.
by a team member for source package uploads.
The signing key is stored in an environment variable which Travis CI makes available to
certain builds. Since Travis CI doesn't support FTP, SFTP is used to transfer the
packages. To set this up yourself, you need to create a Launchpad user and add a GPG key
and SSH key to it. Then encode both keys as base64 and configure 'secret' environment
variables `PPA_SIGNING_KEY` and `PPA_SSH_KEY` on Travis.
We want to build go-ethereum with the most recent version of Go, irrespective of the Go
version that is available in the main Ubuntu repository. In order to make this possible,
@ -27,7 +34,7 @@ Add the gophers PPA and install Go 1.10 and Debian packaging tools:
$ sudo apt-add-repository ppa:gophers/ubuntu/archive
$ sudo apt-get update
$ sudo apt-get install build-essential golang-1.10 devscripts debhelper
$ sudo apt-get install build-essential golang-1.10 devscripts debhelper python-bzrlib python-paramiko
Create the source packages:

67
build/ci.go
View file

@ -441,11 +441,8 @@ func archiveBasename(arch string, archiveVersion string) string {
func archiveUpload(archive string, blobstore string, signer string) error {
// If signing was requested, generate the signature files
if signer != "" {
pgpkey, err := base64.StdEncoding.DecodeString(os.Getenv(signer))
if err != nil {
return fmt.Errorf("invalid base64 %s", signer)
}
if err := build.PGPSignFile(archive, archive+".asc", string(pgpkey)); err != nil {
key := getenvBase64(signer)
if err := build.PGPSignFile(archive, archive+".asc", string(key)); err != nil {
return err
}
}
@ -488,7 +485,8 @@ func maybeSkipArchive(env build.Environment) {
func doDebianSource(cmdline []string) {
var (
signer = flag.String("signer", "", `Signing key name, also used as package author`)
upload = flag.String("upload", "", `Where to upload the source package (usually "ppa:ethereum/ethereum")`)
upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`)
sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`)
workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
now = time.Now()
)
@ -498,11 +496,7 @@ func doDebianSource(cmdline []string) {
maybeSkipArchive(env)
// Import the signing key.
if b64key := os.Getenv("PPA_SIGNING_KEY"); b64key != "" {
key, err := base64.StdEncoding.DecodeString(b64key)
if err != nil {
log.Fatal("invalid base64 PPA_SIGNING_KEY")
}
if key := getenvBase64("PPA_SIGNING_KEY"); len(key) > 0 {
gpg := exec.Command("gpg", "--import")
gpg.Stdin = bytes.NewReader(key)
build.MustRun(gpg)
@ -513,22 +507,58 @@ func doDebianSource(cmdline []string) {
for _, distro := range debDistros {
meta := newDebMetadata(distro, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables)
pkgdir := stageDebianSource(*workdir, meta)
debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d")
debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz")
debuild.Dir = pkgdir
build.MustRun(debuild)
changes := fmt.Sprintf("%s_%s_source.changes", meta.Name(), meta.VersionString())
changes = filepath.Join(*workdir, changes)
var (
basename = fmt.Sprintf("%s_%s", meta.Name(), meta.VersionString())
source = filepath.Join(*workdir, basename+".tar.xz")
dsc = filepath.Join(*workdir, basename+".dsc")
changes = filepath.Join(*workdir, basename+"_source.changes")
)
if *signer != "" {
build.MustRunCommand("debsign", changes)
}
if *upload != "" {
build.MustRunCommand("dput", "--passive", "--no-upload-log", *upload, changes)
ppaUpload(*workdir, *upload, *sshUser, []string{source, dsc, changes})
}
}
}
}
func ppaUpload(workdir, ppa, sshUser string, files []string) {
p := strings.Split(ppa, "/")
if len(p) != 2 {
log.Fatal("-upload PPA name must contain single /")
}
if sshUser == "" {
sshUser = p[0]
}
incomingDir := fmt.Sprintf("~%s/ubuntu/%s", p[0], p[1])
// Create the SSH identity file if it doesn't exist.
var idfile string
if sshkey := getenvBase64("PPA_SSH_KEY"); len(sshkey) > 0 {
idfile = filepath.Join(workdir, "sshkey")
if _, err := os.Stat(idfile); os.IsNotExist(err) {
ioutil.WriteFile(idfile, sshkey, 0600)
}
}
// Upload
dest := sshUser + "@ppa.launchpad.net"
if err := build.UploadSFTP(idfile, dest, incomingDir, files); err != nil {
log.Fatal(err)
}
}
func getenvBase64(variable string) []byte {
dec, err := base64.StdEncoding.DecodeString(os.Getenv(variable))
if err != nil {
log.Fatal("invalid base64 " + variable)
}
return []byte(dec)
}
func makeWorkdir(wdflag string) string {
var err error
if wdflag != "" {
@ -800,15 +830,10 @@ func doAndroidArchive(cmdline []string) {
os.Rename(archive, meta.Package+".aar")
if *signer != "" && *deploy != "" {
// Import the signing key into the local GPG instance
b64key := os.Getenv(*signer)
key, err := base64.StdEncoding.DecodeString(b64key)
if err != nil {
log.Fatalf("invalid base64 %s", *signer)
}
key := getenvBase64(*signer)
gpg := exec.Command("gpg", "--import")
gpg.Stdin = bytes.NewReader(key)
build.MustRun(gpg)
keyID, err := build.PGPKeyID(string(key))
if err != nil {
log.Fatal(err)

46
build/travis_keepalive.sh Executable file
View file

@ -0,0 +1,46 @@
#!/usr/bin/env bash
# travis_keepalive runs the given command and preserves its return value,
# while it forks a child process what periodically produces a log line,
# so that Travis won't abort the build after 10 minutes.
# Why?
# `t.Log()` in Go holds the buffer until the test does not pass or fail,
# and `-race` can increase the execution time by 2-20x.
set -euo pipefail
readonly KEEPALIVE_INTERVAL=300 # seconds => 5m
main() {
keepalive
$@
}
# Keepalive produces a log line in each KEEPALIVE_INTERVAL.
keepalive() {
local child_pid
# Note: We fork here!
repeat "keepalive" &
child_pid=$!
ensureChildOnEXIT "${child_pid}"
}
repeat() {
local this="$1"
while true; do
echo "${this}"
sleep "${KEEPALIVE_INTERVAL}"
done
}
# Ensures that the child gets killed on normal program exit.
ensureChildOnEXIT() {
# Note: SIGINT and SIGTERM are forwarded to the child process by Bash
# automatically, so we don't have to deal with signals.
local child_pid="$1"
trap "kill ${child_pid}" EXIT
}
main "$@"

2
cmd/abigen/main.go
View file

@ -124,7 +124,7 @@ func main() {
}
abis = append(abis, string(abi))
bin := []byte{}
var bin []byte
if *binFlag != "" {
if bin, err = ioutil.ReadFile(*binFlag); err != nil {
fmt.Printf("Failed to read input bytecode: %v\n", err)

153
cmd/clef/README.md
View file

@ -16,7 +16,8 @@ Check out
* the [tutorial](tutorial.md) for some concrete examples on how the signer works.
* the [setup docs](docs/setup.md) for some information on how to configure it to work on QubesOS or USBArmory.
* the [data types](datatypes.md) for detailed information on the json types used in the communication between
clef and an external UI
## Command line flags
Clef accepts the following command line options:
@ -24,25 +25,31 @@ Clef accepts the following command line options:
COMMANDS:
init Initialize the signer, generate secret storage
attest Attest that a js-file is to be used
addpw Store a credential for a keystore file
setpw Store a credential for a keystore file
gendoc Generate documentation about json-rpc format
help Shows a list of commands or help for one command
GLOBAL OPTIONS:
--loglevel value log level to emit to the screen (default: 4)
--keystore value Directory for the keystore (default: "$HOME/.ethereum/keystore")
--configdir value Directory for clef configuration (default: "$HOME/.clef")
--networkid value Network identifier (integer, 1=Frontier, 2=Morden (disused), 3=Ropsten, 4=Rinkeby) (default: 1)
--configdir value Directory for Clef configuration (default: "$HOME/.clef")
--chainid value Chain id to use for signing (1=mainnet, 3=ropsten, 4=rinkeby, 5=Goerli) (default: 1)
--lightkdf Reduce key-derivation RAM & CPU usage at some expense of KDF strength
--nousb Disables monitoring for and managing USB hardware wallets
--rpcaddr value HTTP-RPC server listening interface (default: "localhost")
--rpcvhosts value Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard. (default: "localhost")
--ipcdisable Disable the IPC-RPC server
--ipcpath Filename for IPC socket/pipe within the datadir (explicit paths escape it)
--rpc Enable the HTTP-RPC server
--rpcport value HTTP-RPC server listening port (default: 8550)
--signersecret value A file containing the password used to encrypt signer credentials, e.g. keystore credentials and ruleset hash
--signersecret value A file containing the (encrypted) master seed to encrypt Clef data, e.g. keystore credentials and ruleset hash
--4bytedb value File containing 4byte-identifiers (default: "./4byte.json")
--4bytedb-custom value File used for writing new 4byte-identifiers submitted via API (default: "./4byte-custom.json")
--auditlog value File used to emit audit logs. Set to "" to disable (default: "audit.log")
--rules value Enable rule-engine (default: "rules.json")
--stdio-ui Use STDIN/STDOUT as a channel for an external UI. This means that an STDIN/STDOUT is used for RPC-communication with a e.g. a graphical user interface, and can be used when the signer is started by an external process.
--stdio-ui-test Mechanism to test interface between signer and UI. Requires 'stdio-ui'.
--stdio-ui Use STDIN/STDOUT as a channel for an external UI. This means that an STDIN/STDOUT is used for RPC-communication with a e.g. a graphical user interface, and can be used when Clef is started by an external process.
--stdio-ui-test Mechanism to test interface between Clef and UI. Requires 'stdio-ui'.
--advanced If enabled, issues warnings instead of rejections for suspicious requests. Default off
--help, -h show help
--version, -v print the version
@ -189,7 +196,9 @@ None
"method": "account_new",
"params": []
}
```
Response
```
{
"id": 0,
"jsonrpc": "2.0",
@ -222,7 +231,9 @@ None
"jsonrpc": "2.0",
"method": "account_list"
}
```
Response
```
{
"id": 1,
"jsonrpc": "2.0",
@ -285,8 +296,8 @@ Response
```json
{
"id": 2,
"jsonrpc": "2.0",
"id": 67,
"error": {
"code": -32000,
"message": "Request denied"
@ -298,6 +309,7 @@ Response
```json
{
"id": 67,
"jsonrpc": "2.0",
"method": "account_signTransaction",
"params": [
@ -311,8 +323,7 @@ Response
"data": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"
},
"safeSend(address)"
],
"id": 67
]
}
```
Response
@ -346,15 +357,18 @@ Bash example:
{"jsonrpc":"2.0","id":67,"result":{"raw":"0xf88380018203339407a565b7ed7d7a678680a4c162885bedbb695fe080a44401a6e4000000000000000000000000000000000000000000000000000000000000001226a0223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20ea02aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663","tx":{"nonce":"0x0","gasPrice":"0x1","gas":"0x333","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0","value":"0x0","input":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012","v":"0x26","r":"0x223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20e","s":"0x2aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663","hash":"0xeba2df809e7a612a0a0d444ccfa5c839624bdc00dd29e3340d46df3870f8a30e"}}}
```
### account_sign
### account_signData
#### Sign data
Signs a chunk of data and returns the calculated signature.
#### Arguments
- content type [string]: type of signed data
- `text/validator`: hex data with custom validator defined in a contract
- `application/clique`: [clique](https://github.com/ethereum/EIPs/issues/225) headers
- `text/plain`: simple hex data validated by `account_ecRecover`
- account [address]: account to sign with
- data [data]: data to sign
- data [object]: data to sign
#### Result
- calculated signature [data]
@ -364,8 +378,9 @@ Bash example:
{
"id": 3,
"jsonrpc": "2.0",
"method": "account_sign",
"method": "account_signData",
"params": [
"data/plain",
"0x1923f626bb8dc025849e00f99c25fe2b2f7fb0db",
"0xaabbccdd"
]
@ -381,10 +396,108 @@ Response
}
```
### account_signTypedData
#### Sign data
Signs a chunk of structured data conformant to [EIP712]([EIP-712](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-712.md)) and returns the calculated signature.
#### Arguments
- account [address]: account to sign with
- data [object]: data to sign
#### Result
- calculated signature [data]
#### Sample call
```json
{
"id": 68,
"jsonrpc": "2.0",
"method": "account_signTypedData",
"params": [
"0xcd2a3d9f938e13cd947ec05abc7fe734df8dd826",
{
"types": {
"EIP712Domain": [
{
"name": "name",
"type": "string"
},
{
"name": "version",
"type": "string"
},
{
"name": "chainId",
"type": "uint256"
},
{
"name": "verifyingContract",
"type": "address"
}
],
"Person": [
{
"name": "name",
"type": "string"
},
{
"name": "wallet",
"type": "address"
}
],
"Mail": [
{
"name": "from",
"type": "Person"
},
{
"name": "to",
"type": "Person"
},
{
"name": "contents",
"type": "string"
}
]
},
"primaryType": "Mail",
"domain": {
"name": "Ether Mail",
"version": "1",
"chainId": 1,
"verifyingContract": "0xCcCCccccCCCCcCCCCCCcCcCccCcCCCcCcccccccC"
},
"message": {
"from": {
"name": "Cow",
"wallet": "0xCD2a3d9F938E13CD947Ec05AbC7FE734Df8DD826"
},
"to": {
"name": "Bob",
"wallet": "0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB"
},
"contents": "Hello, Bob!"
}
}
]
}
```
Response
```json
{
"id": 1,
"jsonrpc": "2.0",
"result": "0x4355c47d63924e8a72e509b65029052eb6c299d53a04e167c5775fd466751c9d07299936d304c153f6443dfa05f40ff007d72911b6f72307f996231605b915621c"
}
```
### account_ecRecover
#### Recover address
Derive the address from the account that was used to sign data from the data and signature.
#### Sign data
Derive the address from the account that was used to sign data with content type `text/plain` and the signature.
#### Arguments
- data [data]: data that was signed
@ -400,6 +513,7 @@ Response
"jsonrpc": "2.0",
"method": "account_ecRecover",
"params": [
"data/plain",
"0xaabbccdd",
"0x5b6693f153b48ec1c706ba4169960386dbaa6903e249cc79a8e6ddc434451d417e1e57327872c7f538beeb323c300afa9999a3d4a5de6caf3be0d5ef832b67ef1c"
]
@ -413,7 +527,6 @@ Response
"jsonrpc": "2.0",
"result": "0x1923f626bb8dc025849e00f99c25fe2b2f7fb0db"
}
```
### account_import
@ -458,7 +571,7 @@ Response
},
"id": "09bccb61-b8d3-4e93-bf4f-205a8194f0b9",
"version": 3
},
}
]
}
```

219
cmd/clef/datatypes.md Normal file
View file

@ -0,0 +1,219 @@
## UI Client interface
These data types are defined in the channel between clef and the UI
### SignDataRequest
SignDataRequest contains information about a pending request to sign some data. The data to be signed can be of various types, defined by content-type. Clef has done most of the work in canonicalizing and making sense of the data, and it's up to the UI to presentthe user with the contents of the `message`
Example:
```json
{
"content_type": "text/plain",
"address": "0xDEADbEeF000000000000000000000000DeaDbeEf",
"raw_data": "GUV0aGVyZXVtIFNpZ25lZCBNZXNzYWdlOgoxMWhlbGxvIHdvcmxk",
"message": [
{
"name": "message",
"value": "\u0019Ethereum Signed Message:\n11hello world",
"type": "text/plain"
}
],
"hash": "0xd9eba16ed0ecae432b71fe008c98cc872bb4cc214d3220a36f365326cf807d68",
"meta": {
"remote": "localhost:9999",
"local": "localhost:8545",
"scheme": "http",
"User-Agent": "Firefox 3.2",
"Origin": "www.malicious.ru"
}
}
```
### SignDataResponse - approve
Response to SignDataRequest
Example:
```json
{
"approved": true,
"Password": "apassword"
}
```
### SignDataResponse - deny
Response to SignDataRequest
Example:
```json
{
"approved": false,
"Password": ""
}
```
### SignTxRequest
SignTxRequest contains information about a pending request to sign a transaction. Aside from the transaction itself, there is also a `call_info`-struct. That struct contains messages of various types, that the user should be informed of.
As in any request, it's important to consider that the `meta` info also contains untrusted data.
The `transaction` (on input into clef) can have either `data` or `input` -- if both are set, they must be identical, otherwise an error is generated. However, Clef will always use `data` when passing this struct on (if Clef does otherwise, please file a ticket)
Example:
```json
{
"transaction": {
"from": "0xDEADbEeF000000000000000000000000DeaDbeEf",
"to": null,
"gas": "0x3e8",
"gasPrice": "0x5",
"value": "0x6",
"nonce": "0x1",
"data": "0x01020304"
},
"call_info": [
{
"type": "Warning",
"message": "Something looks odd, show this message as a warning"
},
{
"type": "Info",
"message": "User should see this aswell"
}
],
"meta": {
"remote": "localhost:9999",
"local": "localhost:8545",
"scheme": "http",
"User-Agent": "Firefox 3.2",
"Origin": "www.malicious.ru"
}
}
```
### SignDataResponse - approve
Response to SignDataRequest. This response needs to contain the `transaction`, because the UI is free to make modifications to the transaction.
Example:
```json
{
"transaction": {
"from": "0xDEADbEeF000000000000000000000000DeaDbeEf",
"to": null,
"gas": "0x3e8",
"gasPrice": "0x5",
"value": "0x6",
"nonce": "0x4",
"data": "0x04030201"
},
"approved": true,
"password": "apassword"
}
```
### SignDataResponse - deny
Response to SignDataRequest. When denying a request, there's no need to provide the transaction in return
Example:
```json
{
"approved": false,
"Password": ""
}
```
### OnApproved - SignTransactionResult
SignTransactionResult is used in the call `clef` -> `OnApprovedTx(result)`
This occurs _after_ successful completion of the entire signing procedure, but right before the signed transaction is passed to the external caller. This method (and data) can be used by the UI to signal to the user that the transaction was signed, but it is primarily useful for ruleset implementations.
A ruleset that implements a rate limitation needs to know what transactions are sent out to the external interface. By hooking into this methods, the ruleset can maintain track of that count.
**OBS:** Note that if an attacker can restore your `clef` data to a previous point in time (e.g through a backup), the attacker can reset such windows, even if he/she is unable to decrypt the content.
The `OnApproved` method cannot be responded to, it's purely informative
Example:
```json
{
"raw": "0xf85d640101948a8eafb1cf62bfbeb1741769dae1a9dd47996192018026a0716bd90515acb1e68e5ac5867aa11a1e65399c3349d479f5fb698554ebc6f293a04e8a4ebfff434e971e0ef12c5bf3a881b06fd04fc3f8b8a7291fb67a26a1d4ed",
"tx": {
"nonce": "0x64",
"gasPrice": "0x1",
"gas": "0x1",
"to": "0x8a8eafb1cf62bfbeb1741769dae1a9dd47996192",
"value": "0x1",
"input": "0x",
"v": "0x26",
"r": "0x716bd90515acb1e68e5ac5867aa11a1e65399c3349d479f5fb698554ebc6f293",
"s": "0x4e8a4ebfff434e971e0ef12c5bf3a881b06fd04fc3f8b8a7291fb67a26a1d4ed",
"hash": "0x662f6d772692dd692f1b5e8baa77a9ff95bbd909362df3fc3d301aafebde5441"
}
}
```
### UserInputRequest
Sent when clef needs the user to provide data. If 'password' is true, the input field should be treated accordingly (echo-free)
Example:
```json
{
"prompt": "The question to ask the user",
"title": "The title here",
"isPassword": true
}
```
### UserInputResponse
Response to SignDataRequest
Example:
```json
{
"text": "The textual response from user"
}
```
### ListRequest
Sent when a request has been made to list addresses. The UI is provided with the full `account`s, including local directory names. Note: this information is not passed back to the external caller, who only sees the `address`es.
Example:
```json
{
"accounts": [
{
"address": "0xdeadbeef000000000000000000000000deadbeef",
"url": "keystore:///path/to/keyfile/a"
},
{
"address": "0x1111111122222222222233333333334444444444",
"url": "keystore:///path/to/keyfile/b"
}
],
"meta": {
"remote": "localhost:9999",
"local": "localhost:8545",
"scheme": "http",
"User-Agent": "Firefox 3.2",
"Origin": "www.malicious.ru"
}
}
```
### UserInputResponse
Response to list request. The response contains a list of all addresses to show to the caller. Note: the UI is free to respond with any address the caller, regardless of whether it exists or not
Example:
```json
{
"accounts": [
{
"address": "0x0000000000000000000000000000000000000000",
"url": ".. ignored .."
},
{
"address": "0xffffffffffffffffffffffffffffffffffffffff",
"url": ""
}
]
}
```

BIN
cmd/clef/docs/clef_architecture_pt1.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 68 KiB

BIN
cmd/clef/docs/clef_architecture_pt2.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 80 KiB

BIN
cmd/clef/docs/clef_architecture_pt3.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 99 KiB

BIN
cmd/clef/docs/clef_architecture_pt4.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 115 KiB

15
cmd/clef/extapi_changelog.md
View file

@ -1,5 +1,20 @@
### Changelog for external API
### 6.0.0
* `New` was changed to deliver only an address, not the full `Account` data
* `Export` was moved from External API to the UI Server API
#### 5.0.0
* The external `account_EcRecover`-method was reimplemented.
* The external method `account_sign(address, data)` was replaced with `account_signData(contentType, address, data)`.
The addition of `contentType` makes it possible to use the method for different types of objects, such as:
* signing data with an intended validator (not yet implemented)
* signing clique headers,
* signing plain personal messages,
* The external method `account_signTypedData` implements [EIP-712](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-712.md) and makes it possible to sign typed data.
#### 4.0.0
* The external `account_Ecrecover`-method was removed.

35
cmd/clef/intapi_changelog.md
View file

@ -1,5 +1,40 @@
### Changelog for internal API (ui-api)
### 4.0.0
* Bidirectional communication implemented, so the UI can query `clef` via the stdin/stdout RPC channel. Methods implemented are:
- `clef_listWallets`
- `clef_listAccounts`
- `clef_listWallets`
- `clef_deriveAccount`
- `clef_importRawKey`
- `clef_openWallet`
- `clef_chainId`
- `clef_setChainId`
- `clef_export`
- `clef_import`
* The type `Account` was modified (the json-field `type` was removed), to consist of
```golang
type Account struct {
Address common.Address `json:"address"` // Ethereum account address derived from the key
URL URL `json:"url"` // Optional resource locator within a backend
}
```
### 3.2.0
* Make `ShowError`, `OnApprovedTx`, `OnSignerStartup` be json-rpc [notifications](https://www.jsonrpc.org/specification#notification):
> A Notification is a Request object without an "id" member. A Request object that is a Notification signifies the Client's lack of interest in the corresponding Response object, and as such no Response object needs to be returned to the client. The Server MUST NOT reply to a Notification, including those that are within a batch request.
>
> Notifications are not confirmable by definition, since they do not have a Response object to be returned. As such, the Client would not be aware of any errors (like e.g. "Invalid params","Internal error"
### 3.1.0
* Add `ContentType` `string` to `SignDataRequest` to accommodate the latest EIP-191 and EIP-712 implementations.
### 3.0.0
* Make use of `OnInputRequired(info UserInputRequest)` for obtaining master password during startup

253
cmd/clef/main.go
View file

@ -28,6 +28,7 @@ import (
"fmt"
"io"
"io/ioutil"
"math/big"
"os"
"os/signal"
"os/user"
@ -35,13 +36,19 @@ import (
"runtime"
"strings"
"github.com/ethereum/go-ethereum/accounts"
"github.com/ethereum/go-ethereum/accounts/keystore"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/console"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/internal/ethapi"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/signer/core"
"github.com/ethereum/go-ethereum/signer/rules"
@ -49,12 +56,6 @@ import (
"gopkg.in/urfave/cli.v1"
)
// ExternalAPIVersion -- see extapi_changelog.md
const ExternalAPIVersion = "4.0.0"
// InternalAPIVersion -- see intapi_changelog.md
const InternalAPIVersion = "3.0.0"
const legalWarning = `
WARNING!
@ -86,6 +87,11 @@ var (
Value: DefaultConfigDir(),
Usage: "Directory for Clef configuration",
}
chainIdFlag = cli.Int64Flag{
Name: "chainid",
Value: params.MainnetChainConfig.ChainID.Int64(),
Usage: "Chain id to use for signing (1=mainnet, 3=ropsten, 4=rinkeby, 5=Goerli)",
}
rpcPortFlag = cli.IntFlag{
Name: "rpcport",
Usage: "HTTP-RPC server listening port",
@ -170,8 +176,14 @@ Clef that the file is 'safe' to execute.`,
Description: `
The setpw command stores a password for a given address (keyfile). If you enter a blank passphrase, it will
remove any stored credential for that address (keyfile)
`,
}
`}
gendocCommand = cli.Command{
Action: GenDoc,
Name: "gendoc",
Usage: "Generate documentation about json-rpc format",
Description: `
The gendoc generates example structures of the json-rpc communication types.
`}
)
func init() {
@ -181,7 +193,7 @@ func init() {
logLevelFlag,
keystoreFlag,
configdirFlag,
utils.NetworkIdFlag,
chainIdFlag,
utils.LightKDFFlag,
utils.NoUSBFlag,
utils.RPCListenAddrFlag,
@ -200,7 +212,7 @@ func init() {
advancedMode,
}
app.Action = signer
app.Commands = []cli.Command{initCommand, attestCommand, setCredentialCommand}
app.Commands = []cli.Command{initCommand, attestCommand, setCredentialCommand, gendocCommand}
}
func main() {
@ -341,7 +353,7 @@ func signer(c *cli.Context) error {
return err
}
var (
ui core.SignerUI
ui core.UIClientAPI
)
if c.GlobalBool(stdiouiFlag.Name) {
log.Info("Using stdin/stdout as UI-channel")
@ -405,14 +417,21 @@ func signer(c *cli.Context) error {
}
}
}
var (
chainId = c.GlobalInt64(chainIdFlag.Name)
ksLoc = c.GlobalString(keystoreFlag.Name)
lightKdf = c.GlobalBool(utils.LightKDFFlag.Name)
advanced = c.GlobalBool(advancedMode.Name)
nousb = c.GlobalBool(utils.NoUSBFlag.Name)
)
log.Info("Starting signer", "chainid", chainId, "keystore", ksLoc,
"light-kdf", lightKdf, "advanced", advanced)
am := core.StartClefAccountManager(ksLoc, nousb, lightKdf)
apiImpl := core.NewSignerAPI(am, chainId, nousb, ui, db, advanced)
apiImpl := core.NewSignerAPI(
c.GlobalInt64(utils.NetworkIdFlag.Name),
c.GlobalString(keystoreFlag.Name),
c.GlobalBool(utils.NoUSBFlag.Name),
ui, db,
c.GlobalBool(utils.LightKDFFlag.Name),
c.GlobalBool(advancedMode.Name))
// Establish the bidirectional communication, by creating a new UI backend and registering
// it with the UI.
ui.RegisterUIServer(core.NewUIServerAPI(apiImpl))
api = apiImpl
// Audit logging
if logfile := c.GlobalString(auditLogFlag.Name); logfile != "" {
@ -479,8 +498,8 @@ func signer(c *cli.Context) error {
}
ui.OnSignerStartup(core.StartupInfo{
Info: map[string]interface{}{
"extapi_version": ExternalAPIVersion,
"intapi_version": InternalAPIVersion,
"extapi_version": core.ExternalAPIVersion,
"intapi_version": core.InternalAPIVersion,
"extapi_http": extapiURL,
"extapi_ipc": ipcapiURL,
},
@ -514,7 +533,12 @@ func DefaultConfigDir() string {
if runtime.GOOS == "darwin" {
return filepath.Join(home, "Library", "Signer")
} else if runtime.GOOS == "windows" {
appdata := os.Getenv("APPDATA")
if appdata != "" {
return filepath.Join(appdata, "Signer")
} else {
return filepath.Join(home, "AppData", "Roaming", "Signer")
}
} else {
return filepath.Join(home, ".clef")
}
@ -532,7 +556,7 @@ func homeDir() string {
}
return ""
}
func readMasterKey(ctx *cli.Context, ui core.SignerUI) ([]byte, error) {
func readMasterKey(ctx *cli.Context, ui core.UIClientAPI) ([]byte, error) {
var (
file string
configDir = ctx.GlobalString(configdirFlag.Name)
@ -629,18 +653,44 @@ func testExternalUI(api *core.SignerAPI) {
}
var err error
cliqueHeader := types.Header{
common.HexToHash("0000H45H"),
common.HexToHash("0000H45H"),
common.HexToAddress("0000H45H"),
common.HexToHash("0000H00H"),
common.HexToHash("0000H45H"),
common.HexToHash("0000H45H"),
types.Bloom{},
big.NewInt(1337),
big.NewInt(1337),
1338,
1338,
big.NewInt(1338),
[]byte("Extra data Extra data Extra data Extra data Extra data Extra data Extra data Extra data"),
common.HexToHash("0x0000H45H"),
types.BlockNonce{},
}
cliqueRlp, err := rlp.EncodeToBytes(cliqueHeader)
if err != nil {
utils.Fatalf("Should not error: %v", err)
}
addr, err := common.NewMixedcaseAddressFromString("0x0011223344556677889900112233445566778899")
if err != nil {
utils.Fatalf("Should not error: %v", err)
}
_, err = api.SignData(ctx, "application/clique", *addr, cliqueRlp)
checkErr("SignData", err)
_, err = api.SignTransaction(ctx, core.SendTxArgs{From: common.MixedcaseAddress{}}, nil)
checkErr("SignTransaction", err)
_, err = api.Sign(ctx, common.MixedcaseAddress{}, common.Hex2Bytes("01020304"))
checkErr("Sign", err)
_, err = api.SignData(ctx, "text/plain", common.MixedcaseAddress{}, common.Hex2Bytes("01020304"))
checkErr("SignData", err)
//_, err = api.SignTypedData(ctx, common.MixedcaseAddress{}, core.TypedData{})
//checkErr("SignTypedData", err)
_, err = api.List(ctx)
checkErr("List", err)
_, err = api.New(ctx)
checkErr("New", err)
_, err = api.Export(ctx, common.Address{})
checkErr("Export", err)
_, err = api.Import(ctx, json.RawMessage{})
checkErr("Import", err)
api.UI.ShowInfo("Tests completed")
@ -652,7 +702,6 @@ func testExternalUI(api *core.SignerAPI) {
} else {
log.Info("No errors")
}
}
// getPassPhrase retrieves the password associated with clef, either fetched
@ -708,6 +757,154 @@ func decryptSeed(keyjson []byte, auth string) ([]byte, error) {
return seed, err
}
// GenDoc outputs examples of all structures used in json-rpc communication
func GenDoc(ctx *cli.Context) {
var (
a = common.HexToAddress("0xdeadbeef000000000000000000000000deadbeef")
b = common.HexToAddress("0x1111111122222222222233333333334444444444")
meta = core.Metadata{
Scheme: "http",
Local: "localhost:8545",
Origin: "www.malicious.ru",
Remote: "localhost:9999",
UserAgent: "Firefox 3.2",
}
output []string
add = func(name, desc string, v interface{}) {
if data, err := json.MarshalIndent(v, "", " "); err == nil {
output = append(output, fmt.Sprintf("### %s\n\n%s\n\nExample:\n```json\n%s\n```", name, desc, data))
} else {
log.Error("Error generating output", err)
}
}
)
{ // Sign plain text request
desc := "SignDataRequest contains information about a pending request to sign some data. " +
"The data to be signed can be of various types, defined by content-type. Clef has done most " +
"of the work in canonicalizing and making sense of the data, and it's up to the UI to present" +
"the user with the contents of the `message`"
sighash, msg := accounts.TextAndHash([]byte("hello world"))
message := []*core.NameValueType{{"message", msg, accounts.MimetypeTextPlain}}
add("SignDataRequest", desc, &core.SignDataRequest{
Address: common.NewMixedcaseAddress(a),
Meta: meta,
ContentType: accounts.MimetypeTextPlain,
Rawdata: []byte(msg),
Message: message,
Hash: sighash})
}
{ // Sign plain text response
add("SignDataResponse - approve", "Response to SignDataRequest",
&core.SignDataResponse{Password: "apassword", Approved: true})
add("SignDataResponse - deny", "Response to SignDataRequest",
&core.SignDataResponse{})
}
{ // Sign transaction request
desc := "SignTxRequest contains information about a pending request to sign a transaction. " +
"Aside from the transaction itself, there is also a `call_info`-struct. That struct contains " +
"messages of various types, that the user should be informed of." +
"\n\n" +
"As in any request, it's important to consider that the `meta` info also contains untrusted data." +
"\n\n" +
"The `transaction` (on input into clef) can have either `data` or `input` -- if both are set, " +
"they must be identical, otherwise an error is generated. " +
"However, Clef will always use `data` when passing this struct on (if Clef does otherwise, please file a ticket)"
data := hexutil.Bytes([]byte{0x01, 0x02, 0x03, 0x04})
add("SignTxRequest", desc, &core.SignTxRequest{
Meta: meta,
Callinfo: []core.ValidationInfo{
{"Warning", "Something looks odd, show this message as a warning"},
{"Info", "User should see this aswell"},
},
Transaction: core.SendTxArgs{
Data: &data,
Nonce: 0x1,
Value: hexutil.Big(*big.NewInt(6)),
From: common.NewMixedcaseAddress(a),
To: nil,
GasPrice: hexutil.Big(*big.NewInt(5)),
Gas: 1000,
Input: nil,
}})
}
{ // Sign tx response
data := hexutil.Bytes([]byte{0x04, 0x03, 0x02, 0x01})
add("SignDataResponse - approve", "Response to SignDataRequest. This response needs to contain the `transaction`"+
", because the UI is free to make modifications to the transaction.",
&core.SignTxResponse{Password: "apassword", Approved: true,
Transaction: core.SendTxArgs{
Data: &data,
Nonce: 0x4,
Value: hexutil.Big(*big.NewInt(6)),
From: common.NewMixedcaseAddress(a),
To: nil,
GasPrice: hexutil.Big(*big.NewInt(5)),
Gas: 1000,
Input: nil,
}})
add("SignDataResponse - deny", "Response to SignDataRequest. When denying a request, there's no need to "+
"provide the transaction in return",
&core.SignDataResponse{})
}
{ // WHen a signed tx is ready to go out
desc := "SignTransactionResult is used in the call `clef` -> `OnApprovedTx(result)`" +
"\n\n" +
"This occurs _after_ successful completion of the entire signing procedure, but right before the signed " +
"transaction is passed to the external caller. This method (and data) can be used by the UI to signal " +
"to the user that the transaction was signed, but it is primarily useful for ruleset implementations." +
"\n\n" +
"A ruleset that implements a rate limitation needs to know what transactions are sent out to the external " +
"interface. By hooking into this methods, the ruleset can maintain track of that count." +
"\n\n" +
"**OBS:** Note that if an attacker can restore your `clef` data to a previous point in time" +
" (e.g through a backup), the attacker can reset such windows, even if he/she is unable to decrypt the content. " +
"\n\n" +
"The `OnApproved` method cannot be responded to, it's purely informative"
rlpdata := common.FromHex("0xf85d640101948a8eafb1cf62bfbeb1741769dae1a9dd47996192018026a0716bd90515acb1e68e5ac5867aa11a1e65399c3349d479f5fb698554ebc6f293a04e8a4ebfff434e971e0ef12c5bf3a881b06fd04fc3f8b8a7291fb67a26a1d4ed")
var tx types.Transaction
rlp.DecodeBytes(rlpdata, &tx)
add("OnApproved - SignTransactionResult", desc, &ethapi.SignTransactionResult{Raw: rlpdata, Tx: &tx})
}
{ // User input
add("UserInputRequest", "Sent when clef needs the user to provide data. If 'password' is true, the input field should be treated accordingly (echo-free)",
&core.UserInputRequest{IsPassword: true, Title: "The title here", Prompt: "The question to ask the user"})
add("UserInputResponse", "Response to SignDataRequest",
&core.UserInputResponse{Text: "The textual response from user"})
}
{ // List request
add("ListRequest", "Sent when a request has been made to list addresses. The UI is provided with the "+
"full `account`s, including local directory names. Note: this information is not passed back to the external caller, "+
"who only sees the `address`es. ",
&core.ListRequest{
Meta: meta,
Accounts: []accounts.Account{
{a, accounts.URL{Scheme: "keystore", Path: "/path/to/keyfile/a"}},
{b, accounts.URL{Scheme: "keystore", Path: "/path/to/keyfile/b"}}},
})
add("UserInputResponse", "Response to list request. The response contains a list of all addresses to show to the caller. "+
"Note: the UI is free to respond with any address the caller, regardless of whether it exists or not",
&core.ListResponse{
Accounts: []accounts.Account{
{common.HexToAddress("0xcowbeef000000cowbeef00000000000000000c0w"), accounts.URL{Path: ".. ignored .."}},
{common.HexToAddress("0xffffffffffffffffffffffffffffffffffffffff"), accounts.URL{}},
}})
}
fmt.Println(`## UI Client interface
These data types are defined in the channel between clef and the UI`)
for _, elem := range output {
fmt.Println(elem)
}
}
/**
//Create Account

73
cmd/clef/tests/testsigner.js Normal file
View file

@ -0,0 +1,73 @@
// This file is a test-utility for testing clef-functionality
//
// Start clef with
//
// build/bin/clef --4bytedb=./cmd/clef/4byte.json --rpc
//
// Start geth with
//
// build/bin/geth --nodiscover --maxpeers 0 --signer http://localhost:8550 console --preload=cmd/clef/tests/testsigner.js
//
// and in the console simply invoke
//
// > test()
//
// You can reload the file via `reload()`
function reload(){
loadScript("./cmd/clef/tests/testsigner.js");
}
function init(){
if (typeof accts == 'undefined' || accts.length == 0){
accts = eth.accounts
console.log("Got accounts ", accts);
}
}
init()
function testTx(){
if( accts && accts.length > 0) {
var a = accts[0]
var txdata = eth.signTransaction({from: a, to: a, value: 1, nonce: 1, gas: 1, gasPrice: 1})
var v = parseInt(txdata.tx.v)
console.log("V value: ", v)
if (v == 37 || v == 38){
console.log("Mainnet 155-protected chainid was used")
}
if (v == 27 || v == 28){
throw new Error("Mainnet chainid was used, but without replay protection!")
}
}
}
function testSignText(){
if( accts && accts.length > 0){
var a = accts[0]
var r = eth.sign(a, "0x68656c6c6f20776f726c64"); //hello world
console.log("signing response", r)
}
}
function testClique(){
if( accts && accts.length > 0){
var a = accts[0]
var r = debug.testSignCliqueBlock(a, 0); // Sign genesis
console.log("signing response", r)
if( a != r){
throw new Error("Requested signing by "+a+ " but got sealer "+r)
}
}
}
function test(){
var tests = [
testTx,
testSignText,
testClique,
]
for( i in tests){
try{
tests[i]()
}catch(err){
console.log(err)
}
}
}

2
cmd/faucet/faucet.go
View file

@ -282,7 +282,7 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network u
// close terminates the Ethereum connection and tears down the faucet.
func (f *faucet) close() error {
return f.stack.Stop()
return f.stack.Close()
}
// listenAndServe registers the HTTP handlers for the faucet and boots it up

19
cmd/geth/chaincmd.go
View file

@ -190,6 +190,8 @@ func initGenesis(ctx *cli.Context) error {
}
// Open an initialise both full and light databases
stack := makeFullNode(ctx)
defer stack.Close()
for _, name := range []string{"chaindata", "lightchaindata"} {
chaindb, err := stack.OpenDatabase(name, 0, 0)
if err != nil {
@ -209,6 +211,8 @@ func importChain(ctx *cli.Context) error {
utils.Fatalf("This command requires an argument.")
}
stack := makeFullNode(ctx)
defer stack.Close()
chain, chainDb := utils.MakeChain(ctx, stack)
defer chainDb.Close()
@ -303,6 +307,8 @@ func exportChain(ctx *cli.Context) error {
utils.Fatalf("This command requires an argument.")
}
stack := makeFullNode(ctx)
defer stack.Close()
chain, _ := utils.MakeChain(ctx, stack)
start := time.Now()
@ -336,9 +342,11 @@ func importPreimages(ctx *cli.Context) error {
utils.Fatalf("This command requires an argument.")
}
stack := makeFullNode(ctx)
diskdb := utils.MakeChainDatabase(ctx, stack).(*ethdb.LDBDatabase)
defer stack.Close()
diskdb := utils.MakeChainDatabase(ctx, stack).(*ethdb.LDBDatabase)
start := time.Now()
if err := utils.ImportPreimages(diskdb, ctx.Args().First()); err != nil {
utils.Fatalf("Import error: %v\n", err)
}
@ -352,9 +360,11 @@ func exportPreimages(ctx *cli.Context) error {
utils.Fatalf("This command requires an argument.")
}
stack := makeFullNode(ctx)
diskdb := utils.MakeChainDatabase(ctx, stack).(*ethdb.LDBDatabase)
defer stack.Close()
diskdb := utils.MakeChainDatabase(ctx, stack).(*ethdb.LDBDatabase)
start := time.Now()
if err := utils.ExportPreimages(diskdb, ctx.Args().First()); err != nil {
utils.Fatalf("Export error: %v\n", err)
}
@ -369,8 +379,9 @@ func copyDb(ctx *cli.Context) error {
}
// Initialize a new chain for the running node to sync into
stack := makeFullNode(ctx)
chain, chainDb := utils.MakeChain(ctx, stack)
defer stack.Close()
chain, chainDb := utils.MakeChain(ctx, stack)
syncmode := *utils.GlobalTextMarshaler(ctx, utils.SyncModeFlag.Name).(*downloader.SyncMode)
dl := downloader.New(syncmode, chainDb, new(event.TypeMux), chain, nil, nil)
@ -441,6 +452,8 @@ func removeDB(ctx *cli.Context) error {
func dump(ctx *cli.Context) error {
stack := makeFullNode(ctx)
defer stack.Close()
chain, chainDb := utils.MakeChain(ctx, stack)
for _, arg := range ctx.Args() {
var block *types.Block

4
cmd/geth/consolecmd.go
View file

@ -79,7 +79,7 @@ func localConsole(ctx *cli.Context) error {
// Create and start the node based on the CLI flags
node := makeFullNode(ctx)
startNode(ctx, node)
defer node.Stop()
defer node.Close()
// Attach to the newly started node and start the JavaScript console
client, err := node.Attach()
@ -180,7 +180,7 @@ func ephemeralConsole(ctx *cli.Context) error {
// Create and start the node based on the CLI flags
node := makeFullNode(ctx)
startNode(ctx, node)
defer node.Stop()
defer node.Close()
// Attach to the newly started node and start the JavaScript console
client, err := node.Attach()

37
cmd/geth/main.go
View file

@ -31,8 +31,10 @@ import (
"github.com/ethereum/go-ethereum/accounts"
"github.com/ethereum/go-ethereum/accounts/keystore"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/console"
"github.com/ethereum/go-ethereum/eth"
"github.com/ethereum/go-ethereum/eth/downloader"
"github.com/ethereum/go-ethereum/ethclient"
"github.com/ethereum/go-ethereum/internal/debug"
"github.com/ethereum/go-ethereum/log"
@ -60,6 +62,7 @@ var (
utils.BootnodesV5Flag,
utils.DataDirFlag,
utils.KeyStoreDirFlag,
utils.ExternalSignerFlag,
utils.NoUSBFlag,
utils.DashboardEnabledFlag,
utils.DashboardAddrFlag,
@ -87,6 +90,7 @@ var (
utils.ULCTrustedNodesFlag,
utils.ULCMinTrustedFractionFlag,
utils.SyncModeFlag,
utils.ExitWhenSyncedFlag,
utils.GCModeFlag,
utils.LightServFlag,
utils.LightPeersFlag,
@ -125,6 +129,7 @@ var (
utils.DeveloperPeriodFlag,
utils.TestnetFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.VMEnableDebugFlag,
utils.NetworkIdFlag,
utils.ConstantinopleOverrideFlag,
@ -274,6 +279,7 @@ func geth(ctx *cli.Context) error {
return fmt.Errorf("invalid command: %q", args[0])
}
node := makeFullNode(ctx)
defer node.Close()
startNode(ctx, node)
node.Wait()
return nil
@ -289,8 +295,8 @@ func startNode(ctx *cli.Context, stack *node.Node) {
utils.StartNode(stack)
// Unlock any account specifically requested
ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
if keystores := stack.AccountManager().Backends(keystore.KeyStoreType); len(keystores) > 0 {
ks := keystores[0].(*keystore.KeyStore)
passwords := utils.MakePasswordList(ctx)
unlocks := strings.Split(ctx.GlobalString(utils.UnlockedAccountFlag.Name), ",")
for i, account := range unlocks {
@ -298,6 +304,7 @@ func startNode(ctx *cli.Context, stack *node.Node) {
unlockAccount(ctx, ks, trimmed, i, passwords)
}
}
}
// Register wallet event handlers to open and auto-derive wallets
events := make(chan accounts.WalletEvent, 16)
stack.AccountManager().Subscribe(events)
@ -339,6 +346,32 @@ func startNode(ctx *cli.Context, stack *node.Node) {
}
}
}()
// Spawn a standalone goroutine for status synchronization monitoring,
// close the node when synchronization is complete if user required.
if ctx.GlobalBool(utils.ExitWhenSyncedFlag.Name) {
go func() {
sub := stack.EventMux().Subscribe(downloader.DoneEvent{})
defer sub.Unsubscribe()
for {
event := <-sub.Chan()
if event == nil {
continue
}
done, ok := event.Data.(downloader.DoneEvent)
if !ok {
continue
}
if timestamp := time.Unix(done.Latest.Time.Int64(), 0); time.Since(timestamp) < 10*time.Minute {
log.Info("Synchronisation completed", "latestnum", done.Latest.Number, "latesthash", done.Latest.Hash(),
"age", common.PrettyAge(timestamp))
stack.Stop()
}
}
}()
}
// Start auxiliary services if enabled
if ctx.GlobalBool(utils.MiningEnabledFlag.Name) || ctx.GlobalBool(utils.DeveloperFlag.Name) {
// Mining only makes sense if a full Ethereum node is running

6
cmd/geth/monitorcmd.go
View file

@ -169,7 +169,7 @@ func retrieveMetrics(client *rpc.Client) (map[string]interface{}, error) {
// resolveMetrics takes a list of input metric patterns, and resolves each to one
// or more canonical metric names.
func resolveMetrics(metrics map[string]interface{}, patterns []string) []string {
res := []string{}
var res []string
for _, pattern := range patterns {
res = append(res, resolveMetric(metrics, pattern, "")...)
}
@ -179,7 +179,7 @@ func resolveMetrics(metrics map[string]interface{}, patterns []string) []string
// resolveMetrics takes a single of input metric pattern, and resolves it to one
// or more canonical metric names.
func resolveMetric(metrics map[string]interface{}, pattern string, path string) []string {
results := []string{}
var results []string
// If a nested metric was requested, recurse optionally branching (via comma)
parts := strings.SplitN(pattern, "/", 2)
@ -215,7 +215,7 @@ func resolveMetric(metrics map[string]interface{}, pattern string, path string)
// expandMetrics expands the entire tree of metrics into a flat list of paths.
func expandMetrics(metrics map[string]interface{}, path string) []string {
// Iterate over all fields and expand individually
list := []string{}
var list []string
for name, metric := range metrics {
switch metric := metric.(type) {
case float64:

5
cmd/geth/usage.go
View file

@ -74,7 +74,9 @@ var AppHelpFlagGroups = []flagGroup{
utils.NetworkIdFlag,
utils.TestnetFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.SyncModeFlag,
utils.ExitWhenSyncedFlag,
utils.GCModeFlag,
utils.EthStatsURLFlag,
utils.IdentityFlag,
@ -143,6 +145,7 @@ var AppHelpFlagGroups = []flagGroup{
Flags: []cli.Flag{
utils.UnlockedAccountFlag,
utils.PasswordFileFlag,
utils.ExternalSignerFlag,
},
},
{
@ -305,7 +308,7 @@ func init() {
categorized[flag.String()] = struct{}{}
}
}
uncategorized := []cli.Flag{}
var uncategorized []cli.Flag
for _, flag := range data.(*cli.App).Flags {
if _, ok := categorized[flag.String()]; !ok {
if strings.HasPrefix(flag.GetName(), "dashboard") {

10
cmd/puppeth/wizard_genesis.go
View file

@ -222,14 +222,18 @@ func (w *wizard) manageGenesis() {
fmt.Println()
fmt.Printf("Which block should Constantinople come into effect? (default = %v)\n", w.conf.Genesis.Config.ConstantinopleBlock)
w.conf.Genesis.Config.ConstantinopleBlock = w.readDefaultBigInt(w.conf.Genesis.Config.ConstantinopleBlock)
if w.conf.Genesis.Config.PetersburgBlock == nil {
w.conf.Genesis.Config.PetersburgBlock = w.conf.Genesis.Config.ConstantinopleBlock
}
fmt.Println()
fmt.Printf("Which block should Constantinople-Fix (remove EIP-1283) come into effect? (default = %v)\n", w.conf.Genesis.Config.ConstantinopleBlock)
w.conf.Genesis.Config.PetersburgBlock = w.readDefaultBigInt(w.conf.Genesis.Config.ConstantinopleBlock)
fmt.Printf("Which block should Constantinople-Fix (remove EIP-1283) come into effect? (default = %v)\n", w.conf.Genesis.Config.PetersburgBlock)
w.conf.Genesis.Config.PetersburgBlock = w.readDefaultBigInt(w.conf.Genesis.Config.PetersburgBlock)
out, _ := json.MarshalIndent(w.conf.Genesis.Config, "", " ")
fmt.Printf("Chain configuration updated:\n\n%s\n", out)
w.conf.flush()
case "2":
// Save whatever genesis configuration we currently have
fmt.Println()

4
cmd/swarm/access.go
View file

@ -178,8 +178,8 @@ func accessNewACT(ctx *cli.Context) {
accessKey []byte
err error
ref = args[0]
pkGrantees = []string{}
passGrantees = []string{}
pkGrantees []string
passGrantees []string
pkGranteesFilename = ctx.String(SwarmAccessGrantKeysFlag.Name)
passGranteesFilename = ctx.String(utils.PasswordFileFlag.Name)
privateKey = getPrivKey(ctx)

4
cmd/swarm/access_test.go
View file

@ -397,7 +397,7 @@ func testACT(t *testing.T, bogusEntries int) {
}
ref := matches[0]
grantees := []string{}
var grantees []string
for i, v := range cluster.Nodes {
if i == nodeToSkip {
continue
@ -408,7 +408,7 @@ func testACT(t *testing.T, bogusEntries int) {
}
if bogusEntries > 0 {
bogusGrantees := []string{}
var bogusGrantees []string
for i := 0; i < bogusEntries; i++ {
prv, err := ecies.GenerateKey(rand.Reader, DefaultCurve, nil)

109
cmd/swarm/config.go
View file

@ -59,30 +59,31 @@ var (
//constants for environment variables
const (
SWARM_ENV_CHEQUEBOOK_ADDR = "SWARM_CHEQUEBOOK_ADDR"
SWARM_ENV_ACCOUNT = "SWARM_ACCOUNT"
SWARM_ENV_LISTEN_ADDR = "SWARM_LISTEN_ADDR"
SWARM_ENV_PORT = "SWARM_PORT"
SWARM_ENV_NETWORK_ID = "SWARM_NETWORK_ID"
SWARM_ENV_SWAP_ENABLE = "SWARM_SWAP_ENABLE"
SWARM_ENV_SWAP_API = "SWARM_SWAP_API"
SWARM_ENV_SYNC_DISABLE = "SWARM_SYNC_DISABLE"
SWARM_ENV_SYNC_UPDATE_DELAY = "SWARM_ENV_SYNC_UPDATE_DELAY"
SWARM_ENV_MAX_STREAM_PEER_SERVERS = "SWARM_ENV_MAX_STREAM_PEER_SERVERS"
SWARM_ENV_LIGHT_NODE_ENABLE = "SWARM_LIGHT_NODE_ENABLE"
SWARM_ENV_DELIVERY_SKIP_CHECK = "SWARM_DELIVERY_SKIP_CHECK"
SWARM_ENV_ENS_API = "SWARM_ENS_API"
SWARM_ENV_ENS_ADDR = "SWARM_ENS_ADDR"
SWARM_ENV_CORS = "SWARM_CORS"
SWARM_ENV_BOOTNODES = "SWARM_BOOTNODES"
SWARM_ENV_PSS_ENABLE = "SWARM_PSS_ENABLE"
SWARM_ENV_STORE_PATH = "SWARM_STORE_PATH"
SWARM_ENV_STORE_CAPACITY = "SWARM_STORE_CAPACITY"
SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY"
SWARM_ENV_BOOTNODE_MODE = "SWARM_BOOTNODE_MODE"
SWARM_ACCESS_PASSWORD = "SWARM_ACCESS_PASSWORD"
SWARM_AUTO_DEFAULTPATH = "SWARM_AUTO_DEFAULTPATH"
GETH_ENV_DATADIR = "GETH_DATADIR"
SwarmEnvChequebookAddr = "SWARM_CHEQUEBOOK_ADDR"
SwarmEnvAccount = "SWARM_ACCOUNT"
SwarmEnvListenAddr = "SWARM_LISTEN_ADDR"
SwarmEnvPort = "SWARM_PORT"
SwarmEnvNetworkID = "SWARM_NETWORK_ID"
SwarmEnvSwapEnable = "SWARM_SWAP_ENABLE"
SwarmEnvSwapAPI = "SWARM_SWAP_API"
SwarmEnvSyncDisable = "SWARM_SYNC_DISABLE"
SwarmEnvSyncUpdateDelay = "SWARM_ENV_SYNC_UPDATE_DELAY"
SwarmEnvMaxStreamPeerServers = "SWARM_ENV_MAX_STREAM_PEER_SERVERS"
SwarmEnvLightNodeEnable = "SWARM_LIGHT_NODE_ENABLE"
SwarmEnvDeliverySkipCheck = "SWARM_DELIVERY_SKIP_CHECK"
SwarmEnvENSAPI = "SWARM_ENS_API"
SwarmEnvENSAddr = "SWARM_ENS_ADDR"
SwarmEnvCORS = "SWARM_CORS"
SwarmEnvBootnodes = "SWARM_BOOTNODES"
SwarmEnvPSSEnable = "SWARM_PSS_ENABLE"
SwarmEnvStorePath = "SWARM_STORE_PATH"
SwarmEnvStoreCapacity = "SWARM_STORE_CAPACITY"
SwarmEnvStoreCacheCapacity = "SWARM_STORE_CACHE_CAPACITY"
SwarmEnvBootnodeMode = "SWARM_BOOTNODE_MODE"
SwarmAccessPassword = "SWARM_ACCESS_PASSWORD"
SwarmAutoDefaultPath = "SWARM_AUTO_DEFAULTPATH"
SwarmGlobalstoreAPI = "SWARM_GLOBALSTORE_API"
GethEnvDataDir = "GETH_DATADIR"
)
// These settings ensure that TOML keys use the same names as Go struct fields.
@ -226,7 +227,7 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
currentConfig.SwapAPI = ctx.GlobalString(SwarmSwapAPIFlag.Name)
if currentConfig.SwapEnabled && currentConfig.SwapAPI == "" {
utils.Fatalf(SWARM_ERR_SWAP_SET_NO_API)
utils.Fatalf(SwarmErrSwapSetNoAPI)
}
if ctx.GlobalIsSet(EnsAPIFlag.Name) {
@ -262,6 +263,10 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
currentConfig.BootnodeMode = ctx.GlobalBool(SwarmBootnodeModeFlag.Name)
}
if ctx.GlobalIsSet(SwarmGlobalStoreAPIFlag.Name) {
currentConfig.GlobalStoreAPI = ctx.GlobalString(SwarmGlobalStoreAPIFlag.Name)
}
return currentConfig
}
@ -269,112 +274,116 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
// envVarsOverride overrides the current config with whatver is provided in environment variables
// most values are not allowed a zero value (empty string), if not otherwise noted
func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
if keyid := os.Getenv(SWARM_ENV_ACCOUNT); keyid != "" {
if keyid := os.Getenv(SwarmEnvAccount); keyid != "" {
currentConfig.BzzAccount = keyid
}
if chbookaddr := os.Getenv(SWARM_ENV_CHEQUEBOOK_ADDR); chbookaddr != "" {
if chbookaddr := os.Getenv(SwarmEnvChequebookAddr); chbookaddr != "" {
currentConfig.Contract = common.HexToAddress(chbookaddr)
}
if networkid := os.Getenv(SWARM_ENV_NETWORK_ID); networkid != "" {
if networkid := os.Getenv(SwarmEnvNetworkID); networkid != "" {
id, err := strconv.ParseUint(networkid, 10, 64)
if err != nil {
utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_NETWORK_ID, err)
utils.Fatalf("invalid environment variable %s: %v", SwarmEnvNetworkID, err)
}
if id != 0 {
currentConfig.NetworkID = id
}
}
if datadir := os.Getenv(GETH_ENV_DATADIR); datadir != "" {
if datadir := os.Getenv(GethEnvDataDir); datadir != "" {
currentConfig.Path = expandPath(datadir)
}
bzzport := os.Getenv(SWARM_ENV_PORT)
bzzport := os.Getenv(SwarmEnvPort)
if len(bzzport) > 0 {
currentConfig.Port = bzzport
}
if bzzaddr := os.Getenv(SWARM_ENV_LISTEN_ADDR); bzzaddr != "" {
if bzzaddr := os.Getenv(SwarmEnvListenAddr); bzzaddr != "" {
currentConfig.ListenAddr = bzzaddr
}
if swapenable := os.Getenv(SWARM_ENV_SWAP_ENABLE); swapenable != "" {
if swapenable := os.Getenv(SwarmEnvSwapEnable); swapenable != "" {
swap, err := strconv.ParseBool(swapenable)
if err != nil {
utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SWAP_ENABLE, err)
utils.Fatalf("invalid environment variable %s: %v", SwarmEnvSwapEnable, err)
}
currentConfig.SwapEnabled = swap
}
if syncdisable := os.Getenv(SWARM_ENV_SYNC_DISABLE); syncdisable != "" {
if syncdisable := os.Getenv(SwarmEnvSyncDisable); syncdisable != "" {
sync, err := strconv.ParseBool(syncdisable)
if err != nil {
utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SYNC_DISABLE, err)
utils.Fatalf("invalid environment variable %s: %v", SwarmEnvSyncDisable, err)
}
currentConfig.SyncEnabled = !sync
}
if v := os.Getenv(SWARM_ENV_DELIVERY_SKIP_CHECK); v != "" {
if v := os.Getenv(SwarmEnvDeliverySkipCheck); v != "" {
skipCheck, err := strconv.ParseBool(v)
if err != nil {
currentConfig.DeliverySkipCheck = skipCheck
}
}
if v := os.Getenv(SWARM_ENV_SYNC_UPDATE_DELAY); v != "" {
if v := os.Getenv(SwarmEnvSyncUpdateDelay); v != "" {
d, err := time.ParseDuration(v)
if err != nil {
utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SYNC_UPDATE_DELAY, err)
utils.Fatalf("invalid environment variable %s: %v", SwarmEnvSyncUpdateDelay, err)
}
currentConfig.SyncUpdateDelay = d
}
if max := os.Getenv(SWARM_ENV_MAX_STREAM_PEER_SERVERS); max != "" {
if max := os.Getenv(SwarmEnvMaxStreamPeerServers); max != "" {
m, err := strconv.Atoi(max)
if err != nil {
utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_MAX_STREAM_PEER_SERVERS, err)
utils.Fatalf("invalid environment variable %s: %v", SwarmEnvMaxStreamPeerServers, err)
}
currentConfig.MaxStreamPeerServers = m
}
if lne := os.Getenv(SWARM_ENV_LIGHT_NODE_ENABLE); lne != "" {
if lne := os.Getenv(SwarmEnvLightNodeEnable); lne != "" {
lightnode, err := strconv.ParseBool(lne)
if err != nil {
utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_LIGHT_NODE_ENABLE, err)
utils.Fatalf("invalid environment variable %s: %v", SwarmEnvLightNodeEnable, err)
}
currentConfig.LightNodeEnabled = lightnode
}
if swapapi := os.Getenv(SWARM_ENV_SWAP_API); swapapi != "" {
if swapapi := os.Getenv(SwarmEnvSwapAPI); swapapi != "" {
currentConfig.SwapAPI = swapapi
}
if currentConfig.SwapEnabled && currentConfig.SwapAPI == "" {
utils.Fatalf(SWARM_ERR_SWAP_SET_NO_API)
utils.Fatalf(SwarmErrSwapSetNoAPI)
}
if ensapi := os.Getenv(SWARM_ENV_ENS_API); ensapi != "" {
if ensapi := os.Getenv(SwarmEnvENSAPI); ensapi != "" {
currentConfig.EnsAPIs = strings.Split(ensapi, ",")
}
if ensaddr := os.Getenv(SWARM_ENV_ENS_ADDR); ensaddr != "" {
if ensaddr := os.Getenv(SwarmEnvENSAddr); ensaddr != "" {
currentConfig.EnsRoot = common.HexToAddress(ensaddr)
}
if cors := os.Getenv(SWARM_ENV_CORS); cors != "" {
if cors := os.Getenv(SwarmEnvCORS); cors != "" {
currentConfig.Cors = cors
}
if bm := os.Getenv(SWARM_ENV_BOOTNODE_MODE); bm != "" {
if bm := os.Getenv(SwarmEnvBootnodeMode); bm != "" {
bootnodeMode, err := strconv.ParseBool(bm)
if err != nil {
utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_BOOTNODE_MODE, err)
utils.Fatalf("invalid environment variable %s: %v", SwarmEnvBootnodeMode, err)
}
currentConfig.BootnodeMode = bootnodeMode
}
if api := os.Getenv(SwarmGlobalstoreAPI); api != "" {
currentConfig.GlobalStoreAPI = api
}
return currentConfig
}

4
cmd/swarm/config_test.go
View file

@ -52,7 +52,7 @@ func TestConfigFailsSwapEnabledNoSwapApi(t *testing.T) {
}
swarm := runSwarm(t, flags...)
swarm.Expect("Fatal: " + SWARM_ERR_SWAP_SET_NO_API + "\n")
swarm.Expect("Fatal: " + SwarmErrSwapSetNoAPI + "\n")
swarm.ExpectExit()
}
@ -63,7 +63,7 @@ func TestConfigFailsNoBzzAccount(t *testing.T) {
}
swarm := runSwarm(t, flags...)
swarm.Expect("Fatal: " + SWARM_ERR_NO_BZZACCOUNT + "\n")
swarm.Expect("Fatal: " + SwarmErrNoBZZAccount + "\n")
swarm.ExpectExit()
}

59
cmd/swarm/explore.go Normal file
View file

@ -0,0 +1,59 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
// Command bzzhash computes a swarm tree hash.
package main
import (
"context"
"fmt"
"os"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/swarm/storage"
"gopkg.in/urfave/cli.v1"
)
var hashesCommand = cli.Command{
Action: hashes,
CustomHelpTemplate: helpTemplate,
Name: "hashes",
Usage: "print all hashes of a file to STDOUT",
ArgsUsage: "<file>",
Description: "Prints all hashes of a file to STDOUT",
}
func hashes(ctx *cli.Context) {
args := ctx.Args()
if len(args) < 1 {
utils.Fatalf("Usage: swarm hashes <file name>")
}
f, err := os.Open(args[0])
if err != nil {
utils.Fatalf("Error opening file " + args[1])
}
defer f.Close()
fileStore := storage.NewFileStore(&storage.FakeChunkStore{}, storage.NewFileStoreParams())
refs, err := fileStore.GetAllReferences(context.TODO(), f, false)
if err != nil {
utils.Fatalf("%v\n", err)
} else {
for _, r := range refs {
fmt.Println(r.String())
}
}
}

41
cmd/swarm/flags.go
View file

@ -23,68 +23,68 @@ var (
ChequebookAddrFlag = cli.StringFlag{
Name: "chequebook",
Usage: "chequebook contract address",
EnvVar: SWARM_ENV_CHEQUEBOOK_ADDR,
EnvVar: SwarmEnvChequebookAddr,
}
SwarmAccountFlag = cli.StringFlag{
Name: "bzzaccount",
Usage: "Swarm account key file",
EnvVar: SWARM_ENV_ACCOUNT,
EnvVar: SwarmEnvAccount,
}
SwarmListenAddrFlag = cli.StringFlag{
Name: "httpaddr",
Usage: "Swarm HTTP API listening interface",
EnvVar: SWARM_ENV_LISTEN_ADDR,
EnvVar: SwarmEnvListenAddr,
}
SwarmPortFlag = cli.StringFlag{
Name: "bzzport",
Usage: "Swarm local http api port",
EnvVar: SWARM_ENV_PORT,
EnvVar: SwarmEnvPort,
}
SwarmNetworkIdFlag = cli.IntFlag{
Name: "bzznetworkid",
Usage: "Network identifier (integer, default 3=swarm testnet)",
EnvVar: SWARM_ENV_NETWORK_ID,
EnvVar: SwarmEnvNetworkID,
}
SwarmSwapEnabledFlag = cli.BoolFlag{
Name: "swap",
Usage: "Swarm SWAP enabled (default false)",
EnvVar: SWARM_ENV_SWAP_ENABLE,
EnvVar: SwarmEnvSwapEnable,
}
SwarmSwapAPIFlag = cli.StringFlag{
Name: "swap-api",
Usage: "URL of the Ethereum API provider to use to settle SWAP payments",
EnvVar: SWARM_ENV_SWAP_API,
EnvVar: SwarmEnvSwapAPI,
}
SwarmSyncDisabledFlag = cli.BoolTFlag{
Name: "nosync",
Usage: "Disable swarm syncing",
EnvVar: SWARM_ENV_SYNC_DISABLE,
EnvVar: SwarmEnvSyncDisable,
}
SwarmSyncUpdateDelay = cli.DurationFlag{
Name: "sync-update-delay",
Usage: "Duration for sync subscriptions update after no new peers are added (default 15s)",
EnvVar: SWARM_ENV_SYNC_UPDATE_DELAY,
EnvVar: SwarmEnvSyncUpdateDelay,
}
SwarmMaxStreamPeerServersFlag = cli.IntFlag{
Name: "max-stream-peer-servers",
Usage: "Limit of Stream peer servers, 0 denotes unlimited",
EnvVar: SWARM_ENV_MAX_STREAM_PEER_SERVERS,
EnvVar: SwarmEnvMaxStreamPeerServers,
Value: 10000, // A very large default value is possible as stream servers have very small memory footprint
}
SwarmLightNodeEnabled = cli.BoolFlag{
Name: "lightnode",
Usage: "Enable Swarm LightNode (default false)",
EnvVar: SWARM_ENV_LIGHT_NODE_ENABLE,
EnvVar: SwarmEnvLightNodeEnable,
}
SwarmDeliverySkipCheckFlag = cli.BoolFlag{
Name: "delivery-skip-check",
Usage: "Skip chunk delivery check (default false)",
EnvVar: SWARM_ENV_DELIVERY_SKIP_CHECK,
EnvVar: SwarmEnvDeliverySkipCheck,
}
EnsAPIFlag = cli.StringSliceFlag{
Name: "ens-api",
Usage: "ENS API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url",
EnvVar: SWARM_ENV_ENS_API,
EnvVar: SwarmEnvENSAPI,
}
SwarmApiFlag = cli.StringFlag{
Name: "bzzapi",
@ -126,7 +126,7 @@ var (
SwarmAccessPasswordFlag = cli.StringFlag{
Name: "password",
Usage: "Password",
EnvVar: SWARM_ACCESS_PASSWORD,
EnvVar: SwarmAccessPassword,
}
SwarmDryRunFlag = cli.BoolFlag{
Name: "dry-run",
@ -135,22 +135,22 @@ var (
CorsStringFlag = cli.StringFlag{
Name: "corsdomain",
Usage: "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')",
EnvVar: SWARM_ENV_CORS,
EnvVar: SwarmEnvCORS,
}
SwarmStorePath = cli.StringFlag{
Name: "store.path",
Usage: "Path to leveldb chunk DB (default <$GETH_ENV_DIR>/swarm/bzz-<$BZZ_KEY>/chunks)",
EnvVar: SWARM_ENV_STORE_PATH,
EnvVar: SwarmEnvStorePath,
}
SwarmStoreCapacity = cli.Uint64Flag{
Name: "store.size",
Usage: "Number of chunks (5M is roughly 20-25GB) (default 5000000)",
EnvVar: SWARM_ENV_STORE_CAPACITY,
EnvVar: SwarmEnvStoreCapacity,
}
SwarmStoreCacheCapacity = cli.UintFlag{
Name: "store.cache.size",
Usage: "Number of recent chunks cached in memory (default 5000)",
EnvVar: SWARM_ENV_STORE_CACHE_CAPACITY,
EnvVar: SwarmEnvStoreCacheCapacity,
}
SwarmCompressedFlag = cli.BoolFlag{
Name: "compressed",
@ -176,4 +176,9 @@ var (
Name: "user",
Usage: "Indicates the user who updates the feed",
}
SwarmGlobalStoreAPIFlag = cli.StringFlag{
Name: "globalstore-api",
Usage: "URL of the Global Store API provider (only for testing)",
EnvVar: SwarmGlobalstoreAPI,
}
)

2
cmd/swarm/fs.go
View file

@ -123,7 +123,7 @@ func listMounts(cliContext *cli.Context) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
mf := []fuse.MountInfo{}
var mf []fuse.MountInfo
err = client.CallContext(ctx, &mf, "swarmfs_listmounts")
if err != nil {
utils.Fatalf("encountered an error calling the RPC endpoint while listing mounts: %v", err)

66
cmd/swarm/global-store/explorer.go Normal file
View file

@ -0,0 +1,66 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"context"
"fmt"
"net"
"net/http"
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/storage/mock"
"github.com/ethereum/go-ethereum/swarm/storage/mock/explorer"
cli "gopkg.in/urfave/cli.v1"
)
// serveChunkExplorer starts an http server in background with chunk explorer handler
// using the provided global store. Server is started if the returned shutdown function
// is not nil.
func serveChunkExplorer(ctx *cli.Context, globalStore mock.GlobalStorer) (shutdown func(), err error) {
if !ctx.IsSet("explorer-address") {
return nil, nil
}
corsOrigins := ctx.StringSlice("explorer-cors-origin")
server := &http.Server{
Handler: explorer.NewHandler(globalStore, corsOrigins),
IdleTimeout: 30 * time.Minute,
ReadTimeout: 2 * time.Minute,
WriteTimeout: 2 * time.Minute,
}
listener, err := net.Listen("tcp", ctx.String("explorer-address"))
if err != nil {
return nil, fmt.Errorf("explorer: %v", err)
}
log.Info("chunk explorer http", "address", listener.Addr().String(), "origins", corsOrigins)
go func() {
if err := server.Serve(listener); err != nil {
log.Error("chunk explorer", "err", err)
}
}()
return func() {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := server.Shutdown(ctx); err != nil {
log.Error("chunk explorer: shutdown", "err", err)
}
}, nil
}

254
cmd/swarm/global-store/explorer_test.go Normal file
View file

@ -0,0 +1,254 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"encoding/json"
"fmt"
"net/http"
"sort"
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/swarm/storage/mock/explorer"
mockRPC "github.com/ethereum/go-ethereum/swarm/storage/mock/rpc"
)
// TestExplorer validates basic chunk explorer functionality by storing
// a small set of chunk and making http requests on exposed endpoint.
// Full chunk explorer validation is done in mock/explorer package.
func TestExplorer(t *testing.T) {
addr := findFreeTCPAddress(t)
explorerAddr := findFreeTCPAddress(t)
testCmd := runGlobalStore(t, "ws", "--addr", addr, "--explorer-address", explorerAddr)
defer testCmd.Kill()
client := websocketClient(t, addr)
store := mockRPC.NewGlobalStore(client)
defer store.Close()
nodeKeys := map[string][]string{
"a1": {"b1", "b2", "b3"},
"a2": {"b3", "b4", "b5"},
}
keyNodes := make(map[string][]string)
for addr, keys := range nodeKeys {
for _, key := range keys {
keyNodes[key] = append(keyNodes[key], addr)
}
}
invalidAddr := "c1"
invalidKey := "d1"
for addr, keys := range nodeKeys {
for _, key := range keys {
err := store.Put(common.HexToAddress(addr), common.Hex2Bytes(key), []byte("data"))
if err != nil {
t.Fatal(err)
}
}
}
endpoint := "http://" + explorerAddr
t.Run("has key", func(t *testing.T) {
for addr, keys := range nodeKeys {
for _, key := range keys {
testStatusResponse(t, endpoint+"/api/has-key/"+addr+"/"+key, http.StatusOK)
testStatusResponse(t, endpoint+"/api/has-key/"+invalidAddr+"/"+key, http.StatusNotFound)
}
testStatusResponse(t, endpoint+"/api/has-key/"+addr+"/"+invalidKey, http.StatusNotFound)
}
testStatusResponse(t, endpoint+"/api/has-key/"+invalidAddr+"/"+invalidKey, http.StatusNotFound)
})
t.Run("keys", func(t *testing.T) {
var keys []string
for key := range keyNodes {
keys = append(keys, key)
}
sort.Strings(keys)
testKeysResponse(t, endpoint+"/api/keys", explorer.KeysResponse{
Keys: keys,
})
})
t.Run("nodes", func(t *testing.T) {
var nodes []string
for addr := range nodeKeys {
nodes = append(nodes, common.HexToAddress(addr).Hex())
}
sort.Strings(nodes)
testNodesResponse(t, endpoint+"/api/nodes", explorer.NodesResponse{
Nodes: nodes,
})
})
t.Run("node keys", func(t *testing.T) {
for addr, keys := range nodeKeys {
testKeysResponse(t, endpoint+"/api/keys?node="+addr, explorer.KeysResponse{
Keys: keys,
})
}
testKeysResponse(t, endpoint+"/api/keys?node="+invalidAddr, explorer.KeysResponse{})
})
t.Run("key nodes", func(t *testing.T) {
for key, addrs := range keyNodes {
var nodes []string
for _, addr := range addrs {
nodes = append(nodes, common.HexToAddress(addr).Hex())
}
sort.Strings(nodes)
testNodesResponse(t, endpoint+"/api/nodes?key="+key, explorer.NodesResponse{
Nodes: nodes,
})
}
testNodesResponse(t, endpoint+"/api/nodes?key="+invalidKey, explorer.NodesResponse{})
})
}
// TestExplorer_CORSOrigin validates if chunk explorer returns
// correct CORS origin header in GET and OPTIONS requests.
func TestExplorer_CORSOrigin(t *testing.T) {
origin := "http://localhost/"
addr := findFreeTCPAddress(t)
explorerAddr := findFreeTCPAddress(t)
testCmd := runGlobalStore(t, "ws",
"--addr", addr,
"--explorer-address", explorerAddr,
"--explorer-cors-origin", origin,
)
defer testCmd.Kill()
// wait until the server is started
waitHTTPEndpoint(t, explorerAddr)
url := "http://" + explorerAddr + "/api/keys"
t.Run("get", func(t *testing.T) {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Origin", origin)
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
header := resp.Header.Get("Access-Control-Allow-Origin")
if header != origin {
t.Errorf("got Access-Control-Allow-Origin header %q, want %q", header, origin)
}
})
t.Run("preflight", func(t *testing.T) {
req, err := http.NewRequest(http.MethodOptions, url, nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Origin", origin)
req.Header.Set("Access-Control-Request-Method", "GET")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
header := resp.Header.Get("Access-Control-Allow-Origin")
if header != origin {
t.Errorf("got Access-Control-Allow-Origin header %q, want %q", header, origin)
}
})
}
// testStatusResponse makes an http request to provided url
// and validates if response is explorer.StatusResponse for
// the expected status code.
func testStatusResponse(t *testing.T, url string, code int) {
t.Helper()
resp, err := http.Get(url)
if err != nil {
t.Fatal(err)
}
if resp.StatusCode != code {
t.Errorf("got status code %v, want %v", resp.StatusCode, code)
}
var r explorer.StatusResponse
if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
t.Fatal(err)
}
if r.Code != code {
t.Errorf("got response code %v, want %v", r.Code, code)
}
if r.Message != http.StatusText(code) {
t.Errorf("got response message %q, want %q", r.Message, http.StatusText(code))
}
}
// testKeysResponse makes an http request to provided url
// and validates if response machhes expected explorer.KeysResponse.
func testKeysResponse(t *testing.T, url string, want explorer.KeysResponse) {
t.Helper()
resp, err := http.Get(url)
if err != nil {
t.Fatal(err)
}
if resp.StatusCode != http.StatusOK {
t.Errorf("got status code %v, want %v", resp.StatusCode, http.StatusOK)
}
var r explorer.KeysResponse
if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
t.Fatal(err)
}
if fmt.Sprint(r.Keys) != fmt.Sprint(want.Keys) {
t.Errorf("got keys %v, want %v", r.Keys, want.Keys)
}
if r.Next != want.Next {
t.Errorf("got next %s, want %s", r.Next, want.Next)
}
}
// testNodeResponse makes an http request to provided url
// and validates if response machhes expected explorer.NodeResponse.
func testNodesResponse(t *testing.T, url string, want explorer.NodesResponse) {
t.Helper()
resp, err := http.Get(url)
if err != nil {
t.Fatal(err)
}
if resp.StatusCode != http.StatusOK {
t.Errorf("got status code %v, want %v", resp.StatusCode, http.StatusOK)
}
var r explorer.NodesResponse
if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
t.Fatal(err)
}
if fmt.Sprint(r.Nodes) != fmt.Sprint(want.Nodes) {
t.Errorf("got nodes %v, want %v", r.Nodes, want.Nodes)
}
if r.Next != want.Next {
t.Errorf("got next %s, want %s", r.Next, want.Next)
}
}

120
cmd/swarm/global-store/global_store.go Normal file
View file

@ -0,0 +1,120 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"io"
"net"
"net/http"
"os"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/swarm/storage/mock"
"github.com/ethereum/go-ethereum/swarm/storage/mock/db"
"github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
cli "gopkg.in/urfave/cli.v1"
)
// startHTTP starts a global store with HTTP RPC server.
// It is used for "http" cli command.
func startHTTP(ctx *cli.Context) (err error) {
server, cleanup, err := newServer(ctx)
if err != nil {
return err
}
defer cleanup()
listener, err := net.Listen("tcp", ctx.String("addr"))
if err != nil {
return err
}
log.Info("http", "address", listener.Addr().String())
return http.Serve(listener, server)
}
// startWS starts a global store with WebSocket RPC server.
// It is used for "websocket" cli command.
func startWS(ctx *cli.Context) (err error) {
server, cleanup, err := newServer(ctx)
if err != nil {
return err
}
defer cleanup()
listener, err := net.Listen("tcp", ctx.String("addr"))
if err != nil {
return err
}
origins := ctx.StringSlice("origins")
log.Info("websocket", "address", listener.Addr().String(), "origins", origins)
return http.Serve(listener, server.WebsocketHandler(origins))
}
// newServer creates a global store and starts a chunk explorer server if configured.
// Returned cleanup function should be called only if err is nil.
func newServer(ctx *cli.Context) (server *rpc.Server, cleanup func(), err error) {
log.PrintOrigins(true)
log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(ctx.Int("verbosity")), log.StreamHandler(os.Stdout, log.TerminalFormat(false))))
cleanup = func() {}
var globalStore mock.GlobalStorer
dir := ctx.String("dir")
if dir != "" {
dbStore, err := db.NewGlobalStore(dir)
if err != nil {
return nil, nil, err
}
cleanup = func() {
if err := dbStore.Close(); err != nil {
log.Error("global store: close", "err", err)
}
}
globalStore = dbStore
log.Info("database global store", "dir", dir)
} else {
globalStore = mem.NewGlobalStore()
log.Info("in-memory global store")
}
server = rpc.NewServer()
if err := server.RegisterName("mockStore", globalStore); err != nil {
cleanup()
return nil, nil, err
}
shutdown, err := serveChunkExplorer(ctx, globalStore)
if err != nil {
cleanup()
return nil, nil, err
}
if shutdown != nil {
cleanup = func() {
shutdown()
if c, ok := globalStore.(io.Closer); ok {
if err := c.Close(); err != nil {
log.Error("global store: close", "err", err)
}
}
}
}
return server, cleanup, nil
}

207
cmd/swarm/global-store/global_store_test.go Normal file
View file

@ -0,0 +1,207 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"context"
"io/ioutil"
"net"
"net/http"
"os"
"testing"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/rpc"
mockRPC "github.com/ethereum/go-ethereum/swarm/storage/mock/rpc"
)
// TestHTTP_InMemory tests in-memory global store that exposes
// HTTP server.
func TestHTTP_InMemory(t *testing.T) {
testHTTP(t, true)
}
// TestHTTP_Database tests global store with persisted database
// that exposes HTTP server.
func TestHTTP_Database(t *testing.T) {
dir, err := ioutil.TempDir("", "swarm-global-store-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
// create a fresh global store
testHTTP(t, true, "--dir", dir)
// check if data saved by the previous global store instance
testHTTP(t, false, "--dir", dir)
}
// testWebsocket starts global store binary with HTTP server
// and validates that it can store and retrieve data.
// If put is false, no data will be stored, only retrieved,
// giving the possibility to check if data is present in the
// storage directory.
func testHTTP(t *testing.T, put bool, args ...string) {
addr := findFreeTCPAddress(t)
testCmd := runGlobalStore(t, append([]string{"http", "--addr", addr}, args...)...)
defer testCmd.Kill()
client, err := rpc.DialHTTP("http://" + addr)
if err != nil {
t.Fatal(err)
}
// wait until global store process is started as
// rpc.DialHTTP is actually not connecting
waitHTTPEndpoint(t, addr)
store := mockRPC.NewGlobalStore(client)
defer store.Close()
node := store.NewNodeStore(common.HexToAddress("123abc"))
wantKey := "key"
wantValue := "value"
if put {
err = node.Put([]byte(wantKey), []byte(wantValue))
if err != nil {
t.Fatal(err)
}
}
gotValue, err := node.Get([]byte(wantKey))
if err != nil {
t.Fatal(err)
}
if string(gotValue) != wantValue {
t.Errorf("got value %s for key %s, want %s", string(gotValue), wantKey, wantValue)
}
}
// TestWebsocket_InMemory tests in-memory global store that exposes
// WebSocket server.
func TestWebsocket_InMemory(t *testing.T) {
testWebsocket(t, true)
}
// TestWebsocket_Database tests global store with persisted database
// that exposes HTTP server.
func TestWebsocket_Database(t *testing.T) {
dir, err := ioutil.TempDir("", "swarm-global-store-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
// create a fresh global store
testWebsocket(t, true, "--dir", dir)
// check if data saved by the previous global store instance
testWebsocket(t, false, "--dir", dir)
}
// testWebsocket starts global store binary with WebSocket server
// and validates that it can store and retrieve data.
// If put is false, no data will be stored, only retrieved,
// giving the possibility to check if data is present in the
// storage directory.
func testWebsocket(t *testing.T, put bool, args ...string) {
addr := findFreeTCPAddress(t)
testCmd := runGlobalStore(t, append([]string{"ws", "--addr", addr}, args...)...)
defer testCmd.Kill()
client := websocketClient(t, addr)
store := mockRPC.NewGlobalStore(client)
defer store.Close()
node := store.NewNodeStore(common.HexToAddress("123abc"))
wantKey := "key"
wantValue := "value"
if put {
err := node.Put([]byte(wantKey), []byte(wantValue))
if err != nil {
t.Fatal(err)
}
}
gotValue, err := node.Get([]byte(wantKey))
if err != nil {
t.Fatal(err)
}
if string(gotValue) != wantValue {
t.Errorf("got value %s for key %s, want %s", string(gotValue), wantKey, wantValue)
}
}
// findFreeTCPAddress returns a local address (IP:Port) to which
// global store can listen on.
func findFreeTCPAddress(t *testing.T) (addr string) {
t.Helper()
listener, err := net.Listen("tcp", "")
if err != nil {
t.Fatal(err)
}
defer listener.Close()
return listener.Addr().String()
}
// websocketClient waits until global store process is started
// and returns rpc client.
func websocketClient(t *testing.T, addr string) (client *rpc.Client) {
t.Helper()
var err error
for i := 0; i < 1000; i++ {
client, err = rpc.DialWebsocket(context.Background(), "ws://"+addr, "")
if err == nil {
break
}
time.Sleep(10 * time.Millisecond)
}
if err != nil {
t.Fatal(err)
}
return client
}
// waitHTTPEndpoint retries http requests to a provided
// address until the connection is established.
func waitHTTPEndpoint(t *testing.T, addr string) {
t.Helper()
var err error
for i := 0; i < 1000; i++ {
_, err = http.Get("http://" + addr)
if err == nil {
break
}
time.Sleep(10 * time.Millisecond)
}
if err != nil {
t.Fatal(err)
}
}

120
cmd/swarm/global-store/main.go Normal file
View file

@ -0,0 +1,120 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"os"
"github.com/ethereum/go-ethereum/log"
cli "gopkg.in/urfave/cli.v1"
)
var (
version = "0.1"
gitCommit string // Git SHA1 commit hash of the release (set via linker flags)
)
func main() {
err := newApp().Run(os.Args)
if err != nil {
log.Error(err.Error())
os.Exit(1)
}
}
// newApp construct a new instance of Swarm Global Store.
// Method Run is called on it in the main function and in tests.
func newApp() (app *cli.App) {
app = cli.NewApp()
app.Name = "global-store"
app.Version = version
if len(gitCommit) >= 8 {
app.Version += "-" + gitCommit[:8]
}
app.Usage = "Swarm Global Store"
// app flags (for all commands)
app.Flags = []cli.Flag{
cli.IntFlag{
Name: "verbosity",
Value: 3,
Usage: "Verbosity level.",
},
cli.StringFlag{
Name: "explorer-address",
Value: "",
Usage: "Chunk explorer HTTP listener address.",
},
cli.StringSliceFlag{
Name: "explorer-cors-origin",
Value: nil,
Usage: "Chunk explorer CORS origin (can be specified multiple times).",
},
}
app.Commands = []cli.Command{
{
Name: "http",
Aliases: []string{"h"},
Usage: "Start swarm global store with HTTP server.",
Action: startHTTP,
// Flags only for "start" command.
// Allow app flags to be specified after the
// command argument.
Flags: append(app.Flags,
cli.StringFlag{
Name: "dir",
Value: "",
Usage: "Data directory.",
},
cli.StringFlag{
Name: "addr",
Value: "0.0.0.0:3033",
Usage: "Address to listen for HTTP connections.",
},
),
},
{
Name: "websocket",
Aliases: []string{"ws"},
Usage: "Start swarm global store with WebSocket server.",
Action: startWS,
// Flags only for "start" command.
// Allow app flags to be specified after the
// command argument.
Flags: append(app.Flags,
cli.StringFlag{
Name: "dir",
Value: "",
Usage: "Data directory.",
},
cli.StringFlag{
Name: "addr",
Value: "0.0.0.0:3033",
Usage: "Address to listen for WebSocket connections.",
},
cli.StringSliceFlag{
Name: "origin",
Value: nil,
Usage: "WebSocket CORS origin (can be specified multiple times).",
},
),
},
}
return app
}

49
cmd/swarm/global-store/run_test.go Normal file
View file

@ -0,0 +1,49 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"fmt"
"os"
"testing"
"github.com/docker/docker/pkg/reexec"
"github.com/ethereum/go-ethereum/internal/cmdtest"
)
func init() {
reexec.Register("swarm-global-store", func() {
if err := newApp().Run(os.Args); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
os.Exit(0)
})
}
func runGlobalStore(t *testing.T, args ...string) *cmdtest.TestCmd {
tt := cmdtest.NewTestCmd(t, nil)
tt.Run("swarm-global-store", args...)
return tt
}
func TestMain(m *testing.M) {
if reexec.Init() {
return
}
os.Exit(m.Run())
}

33
cmd/swarm/main.go
View file

@ -39,13 +39,16 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/swarm"
bzzapi "github.com/ethereum/go-ethereum/swarm/api"
swarmmetrics "github.com/ethereum/go-ethereum/swarm/metrics"
"github.com/ethereum/go-ethereum/swarm/storage/mock"
mockrpc "github.com/ethereum/go-ethereum/swarm/storage/mock/rpc"
"github.com/ethereum/go-ethereum/swarm/tracing"
sv "github.com/ethereum/go-ethereum/swarm/version"
"gopkg.in/urfave/cli.v1"
cli "gopkg.in/urfave/cli.v1"
)
const clientIdentifier = "swarm"
@ -73,8 +76,8 @@ var gitCommit string
//declare a few constant error messages, useful for later error check comparisons in test
var (
SWARM_ERR_NO_BZZACCOUNT = "bzzaccount option is required but not set; check your config file, command line or environment variables"
SWARM_ERR_SWAP_SET_NO_API = "SWAP is enabled but --swap-api is not set"
SwarmErrNoBZZAccount = "bzzaccount option is required but not set; check your config file, command line or environment variables"
SwarmErrSwapSetNoAPI = "SWAP is enabled but --swap-api is not set"
)
// this help command gets added to any subcommand that does not define it explicitly
@ -142,6 +145,8 @@ func init() {
dbCommand,
// See config.go
DumpConfigCommand,
// hashesCommand
hashesCommand,
}
// append a hidden help subcommand to all commands that have subcommands
@ -194,6 +199,7 @@ func init() {
SwarmStorePath,
SwarmStoreCapacity,
SwarmStoreCacheCapacity,
SwarmGlobalStoreAPIFlag,
}
rpcFlags := []cli.Flag{
utils.WSEnabledFlag,
@ -288,6 +294,7 @@ func bzzd(ctx *cli.Context) error {
if err != nil {
utils.Fatalf("can't create node: %v", err)
}
defer stack.Close()
//a few steps need to be done after the config phase is completed,
//due to overriding behavior
@ -322,8 +329,18 @@ func bzzd(ctx *cli.Context) error {
func registerBzzService(bzzconfig *bzzapi.Config, stack *node.Node) {
//define the swarm service boot function
boot := func(_ *node.ServiceContext) (node.Service, error) {
// In production, mockStore must be always nil.
return swarm.NewSwarm(bzzconfig, nil)
var nodeStore *mock.NodeStore
if bzzconfig.GlobalStoreAPI != "" {
// connect to global store
client, err := rpc.Dial(bzzconfig.GlobalStoreAPI)
if err != nil {
return nil, fmt.Errorf("global store: %v", err)
}
globalStore := mockrpc.NewGlobalStore(client)
// create a node store for this swarm key on global store
nodeStore = globalStore.NewNodeStore(common.HexToAddress(bzzconfig.BzzKey))
}
return swarm.NewSwarm(bzzconfig, nodeStore)
}
//register within the ethereum node
if err := stack.Register(boot); err != nil {
@ -334,7 +351,7 @@ func registerBzzService(bzzconfig *bzzapi.Config, stack *node.Node) {
func getAccount(bzzaccount string, ctx *cli.Context, stack *node.Node) *ecdsa.PrivateKey {
//an account is mandatory
if bzzaccount == "" {
utils.Fatalf(SWARM_ERR_NO_BZZACCOUNT)
utils.Fatalf(SwarmErrNoBZZAccount)
}
// Try to load the arg as a hex key file.
if key, err := crypto.LoadECDSA(bzzaccount); err == nil {
@ -365,6 +382,8 @@ func getPrivKey(ctx *cli.Context) *ecdsa.PrivateKey {
if err != nil {
utils.Fatalf("can't create node: %v", err)
}
defer stack.Close()
return getAccount(bzzconfig.BzzAccount, ctx, stack)
}
@ -449,5 +468,5 @@ func setSwarmBootstrapNodes(ctx *cli.Context, cfg *node.Config) {
}
cfg.P2P.BootstrapNodes = append(cfg.P2P.BootstrapNodes, node)
}
log.Debug("added default swarm bootnodes", "length", len(cfg.P2P.BootstrapNodes))
}

127
cmd/swarm/swarm-smoke/feed_upload_and_sync.go
View file

@ -2,13 +2,10 @@ package main
import (
"bytes"
"context"
"crypto/md5"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/http/httptrace"
"os"
"os/exec"
"strings"
@ -19,12 +16,8 @@ import (
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/swarm/api/client"
"github.com/ethereum/go-ethereum/swarm/spancontext"
"github.com/ethereum/go-ethereum/swarm/storage/feed"
"github.com/ethereum/go-ethereum/swarm/testutil"
colorable "github.com/mattn/go-colorable"
opentracing "github.com/opentracing/opentracing-go"
"github.com/pborman/uuid"
cli "gopkg.in/urfave/cli.v1"
)
@ -33,34 +26,28 @@ const (
feedRandomDataLength = 8
)
func cliFeedUploadAndSync(c *cli.Context) error {
metrics.GetOrRegisterCounter("feed-and-sync", nil).Inc(1)
log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))))
func feedUploadAndSyncCmd(ctx *cli.Context, tuid string) error {
errc := make(chan error)
go func() {
errc <- feedUploadAndSync(c)
errc <- feedUploadAndSync(ctx, tuid)
}()
select {
case err := <-errc:
if err != nil {
metrics.GetOrRegisterCounter("feed-and-sync.fail", nil).Inc(1)
metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", commandName), nil).Inc(1)
}
return err
case <-time.After(time.Duration(timeout) * time.Second):
metrics.GetOrRegisterCounter("feed-and-sync.timeout", nil).Inc(1)
metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1)
return fmt.Errorf("timeout after %v sec", timeout)
}
}
// TODO: retrieve with manifest + extract repeating code
func feedUploadAndSync(c *cli.Context) error {
defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now())
generateEndpoints(scheme, cluster, appName, from, to)
log.Info("generating and uploading feeds to " + endpoints[0] + " and syncing")
func feedUploadAndSync(c *cli.Context, tuid string) error {
log.Info("generating and uploading feeds to " + httpEndpoint(hosts[0]) + " and syncing")
// create a random private key to sign updates with and derive the address
pkFile, err := ioutil.TempFile("", "swarm-feed-smoke-test")
@ -114,7 +101,7 @@ func feedUploadAndSync(c *cli.Context) error {
// create feed manifest, topic only
var out bytes.Buffer
cmd := exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--topic", topicHex, "--user", userHex)
cmd := exec.Command("swarm", "--bzzapi", httpEndpoint(hosts[0]), "feed", "create", "--topic", topicHex, "--user", userHex)
cmd.Stdout = &out
log.Debug("create feed manifest topic cmd", "cmd", cmd)
err = cmd.Run()
@ -129,7 +116,7 @@ func feedUploadAndSync(c *cli.Context) error {
out.Reset()
// create feed manifest, subtopic only
cmd = exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--name", subTopicHex, "--user", userHex)
cmd = exec.Command("swarm", "--bzzapi", httpEndpoint(hosts[0]), "feed", "create", "--name", subTopicHex, "--user", userHex)
cmd.Stdout = &out
log.Debug("create feed manifest subtopic cmd", "cmd", cmd)
err = cmd.Run()
@ -144,7 +131,7 @@ func feedUploadAndSync(c *cli.Context) error {
out.Reset()
// create feed manifest, merged topic
cmd = exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--topic", topicHex, "--name", subTopicHex, "--user", userHex)
cmd = exec.Command("swarm", "--bzzapi", httpEndpoint(hosts[0]), "feed", "create", "--topic", topicHex, "--name", subTopicHex, "--user", userHex)
cmd.Stdout = &out
log.Debug("create feed manifest mergetopic cmd", "cmd", cmd)
err = cmd.Run()
@ -170,7 +157,7 @@ func feedUploadAndSync(c *cli.Context) error {
dataHex := hexutil.Encode(data)
// update with topic
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, dataHex)
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, dataHex)
cmd.Stdout = &out
log.Debug("update feed manifest topic cmd", "cmd", cmd)
err = cmd.Run()
@ -181,7 +168,7 @@ func feedUploadAndSync(c *cli.Context) error {
out.Reset()
// update with subtopic
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--name", subTopicHex, dataHex)
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--name", subTopicHex, dataHex)
cmd.Stdout = &out
log.Debug("update feed manifest subtopic cmd", "cmd", cmd)
err = cmd.Run()
@ -192,7 +179,7 @@ func feedUploadAndSync(c *cli.Context) error {
out.Reset()
// update with merged topic
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, "--name", subTopicHex, dataHex)
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, "--name", subTopicHex, dataHex)
cmd.Stdout = &out
log.Debug("update feed manifest merged topic cmd", "cmd", cmd)
err = cmd.Run()
@ -206,14 +193,14 @@ func feedUploadAndSync(c *cli.Context) error {
// retrieve the data
wg := sync.WaitGroup{}
for _, endpoint := range endpoints {
for _, host := range hosts {
// raw retrieve, topic only
for _, hex := range []string{topicHex, subTopicOnlyHex, mergedSubTopicHex} {
wg.Add(1)
ruid := uuid.New()[:8]
go func(hex string, endpoint string, ruid string) {
for {
err := fetchFeed(hex, userHex, endpoint, dataHash, ruid)
err := fetchFeed(hex, userHex, httpEndpoint(host), dataHash, ruid)
if err != nil {
continue
}
@ -221,20 +208,18 @@ func feedUploadAndSync(c *cli.Context) error {
wg.Done()
return
}
}(hex, endpoint, ruid)
}(hex, httpEndpoint(host), ruid)
}
}
wg.Wait()
log.Info("all endpoints synced random data successfully")
// upload test file
seed := int(time.Now().UnixNano() / 1e6)
log.Info("feed uploading to "+endpoints[0]+" and syncing", "seed", seed)
log.Info("feed uploading to "+httpEndpoint(hosts[0])+" and syncing", "seed", seed)
randomBytes := testutil.RandomBytes(seed, filesize*1000)
hash, err := upload(&randomBytes, endpoints[0])
hash, err := upload(randomBytes, httpEndpoint(hosts[0]))
if err != nil {
return err
}
@ -243,15 +228,12 @@ func feedUploadAndSync(c *cli.Context) error {
return err
}
multihashHex := hexutil.Encode(hashBytes)
fileHash, err := digest(bytes.NewReader(randomBytes))
if err != nil {
return err
}
fileHash := h.Sum(nil)
log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fileHash))
// update file with topic
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, multihashHex)
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, multihashHex)
cmd.Stdout = &out
err = cmd.Run()
if err != nil {
@ -261,7 +243,7 @@ func feedUploadAndSync(c *cli.Context) error {
out.Reset()
// update file with subtopic
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--name", subTopicHex, multihashHex)
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--name", subTopicHex, multihashHex)
cmd.Stdout = &out
err = cmd.Run()
if err != nil {
@ -271,7 +253,7 @@ func feedUploadAndSync(c *cli.Context) error {
out.Reset()
// update file with merged topic
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, "--name", subTopicHex, multihashHex)
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, "--name", subTopicHex, multihashHex)
cmd.Stdout = &out
err = cmd.Run()
if err != nil {
@ -282,7 +264,7 @@ func feedUploadAndSync(c *cli.Context) error {
time.Sleep(3 * time.Second)
for _, endpoint := range endpoints {
for _, host := range hosts {
// manifest retrieve, topic only
for _, url := range []string{manifestWithTopic, manifestWithSubTopic, manifestWithMergedTopic} {
@ -290,7 +272,7 @@ func feedUploadAndSync(c *cli.Context) error {
ruid := uuid.New()[:8]
go func(url string, endpoint string, ruid string) {
for {
err := fetch(url, endpoint, fileHash, ruid)
err := fetch(url, endpoint, fileHash, ruid, "")
if err != nil {
continue
}
@ -298,7 +280,7 @@ func feedUploadAndSync(c *cli.Context) error {
wg.Done()
return
}
}(url, endpoint, ruid)
}(url, httpEndpoint(host), ruid)
}
}
@ -307,60 +289,3 @@ func feedUploadAndSync(c *cli.Context) error {
return nil
}
func fetchFeed(topic string, user string, endpoint string, original []byte, ruid string) error {
ctx, sp := spancontext.StartSpan(context.Background(), "feed-and-sync.fetch")
defer sp.Finish()
log.Trace("sleeping", "ruid", ruid)
time.Sleep(3 * time.Second)
log.Trace("http get request (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user)
var tn time.Time
reqUri := endpoint + "/bzz-feed:/?topic=" + topic + "&user=" + user
req, _ := http.NewRequest("GET", reqUri, nil)
opentracing.GlobalTracer().Inject(
sp.Context(),
opentracing.HTTPHeaders,
opentracing.HTTPHeadersCarrier(req.Header))
trace := client.GetClientTrace("feed-and-sync - http get", "feed-and-sync", ruid, &tn)
req = req.WithContext(httptrace.WithClientTrace(ctx, trace))
transport := http.DefaultTransport
//transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
tn = time.Now()
res, err := transport.RoundTrip(req)
if err != nil {
log.Error(err.Error(), "ruid", ruid)
return err
}
log.Trace("http get response (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user, "code", res.StatusCode, "len", res.ContentLength)
if res.StatusCode != 200 {
return fmt.Errorf("expected status code %d, got %v (ruid %v)", 200, res.StatusCode, ruid)
}
defer res.Body.Close()
rdigest, err := digest(res.Body)
if err != nil {
log.Warn(err.Error(), "ruid", ruid)
return err
}
if !bytes.Equal(rdigest, original) {
err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original)
log.Warn(err.Error(), "ruid", ruid)
return err
}
log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength)
return nil
}

71
cmd/swarm/swarm-smoke/main.go
View file

@ -37,18 +37,16 @@ var (
)
var (
endpoints []string
includeLocalhost bool
cluster string
appName string
scheme string
allhosts string
hosts []string
filesize int
syncDelay int
from int
to int
httpPort int
wsPort int
verbosity int
timeout int
single bool
trackTimeout int
)
func main() {
@ -59,39 +57,22 @@ func main() {
app.Flags = []cli.Flag{
cli.StringFlag{
Name: "cluster-endpoint",
Value: "prod",
Usage: "cluster to point to (prod or a given namespace)",
Destination: &cluster,
},
cli.StringFlag{
Name: "app",
Value: "swarm",
Usage: "application to point to (swarm or swarm-private)",
Destination: &appName,
Name: "hosts",
Value: "",
Usage: "comma-separated list of swarm hosts",
Destination: &allhosts,
},
cli.IntFlag{
Name: "cluster-from",
Value: 8501,
Usage: "swarm node (from)",
Destination: &from,
Name: "http-port",
Value: 80,
Usage: "http port",
Destination: &httpPort,
},
cli.IntFlag{
Name: "cluster-to",
Value: 8512,
Usage: "swarm node (to)",
Destination: &to,
},
cli.StringFlag{
Name: "cluster-scheme",
Value: "http",
Usage: "http or https",
Destination: &scheme,
},
cli.BoolFlag{
Name: "include-localhost",
Usage: "whether to include localhost:8500 as an endpoint",
Destination: &includeLocalhost,
Name: "ws-port",
Value: 8546,
Usage: "ws port",
Destination: &wsPort,
},
cli.IntFlag{
Name: "filesize",
@ -122,6 +103,12 @@ func main() {
Usage: "whether to fetch content from a single node or from all nodes",
Destination: &single,
},
cli.IntFlag{
Name: "track-timeout",
Value: 5,
Usage: "timeout in seconds to wait for GetAllReferences to return",
Destination: &trackTimeout,
},
}
app.Flags = append(app.Flags, []cli.Flag{
@ -140,19 +127,25 @@ func main() {
Name: "upload_and_sync",
Aliases: []string{"c"},
Usage: "upload and sync",
Action: cliUploadAndSync,
Action: wrapCliCommand("upload-and-sync", uploadAndSyncCmd),
},
{
Name: "feed_sync",
Aliases: []string{"f"},
Usage: "feed update generate, upload and sync",
Action: cliFeedUploadAndSync,
Action: wrapCliCommand("feed-and-sync", feedUploadAndSyncCmd),
},
{
Name: "upload_speed",
Aliases: []string{"u"},
Usage: "measure upload speed",
Action: cliUploadSpeed,
Action: wrapCliCommand("upload-speed", uploadSpeedCmd),
},
{
Name: "sliding_window",
Aliases: []string{"s"},
Usage: "measure network aggregate capacity",
Action: wrapCliCommand("sliding-window", slidingWindowCmd),
},
}

131
cmd/swarm/swarm-smoke/sliding_window.go Normal file
View file

@ -0,0 +1,131 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"bytes"
"fmt"
"math/rand"
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/swarm/testutil"
"github.com/pborman/uuid"
cli "gopkg.in/urfave/cli.v1"
)
type uploadResult struct {
hash string
digest []byte
}
func slidingWindowCmd(ctx *cli.Context, tuid string) error {
errc := make(chan error)
go func() {
errc <- slidingWindow(ctx, tuid)
}()
select {
case err := <-errc:
if err != nil {
metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", commandName), nil).Inc(1)
}
return err
case <-time.After(time.Duration(timeout) * time.Second):
metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1)
return fmt.Errorf("timeout after %v sec", timeout)
}
}
func slidingWindow(ctx *cli.Context, tuid string) error {
var hashes []uploadResult //swarm hashes of the uploads
nodes := len(hosts)
const iterationTimeout = 30 * time.Second
log.Info("sliding window test started", "tuid", tuid, "nodes", nodes, "filesize(kb)", filesize, "timeout", timeout)
uploadedBytes := 0
networkDepth := 0
errored := false
outer:
for {
log.Info("uploading to "+httpEndpoint(hosts[0])+" and syncing", "seed", seed)
t1 := time.Now()
randomBytes := testutil.RandomBytes(seed, filesize*1000)
hash, err := upload(randomBytes, httpEndpoint(hosts[0]))
if err != nil {
log.Error(err.Error())
return err
}
metrics.GetOrRegisterResettingTimer("sliding-window.upload-time", nil).UpdateSince(t1)
fhash, err := digest(bytes.NewReader(randomBytes))
if err != nil {
log.Error(err.Error())
return err
}
log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash), "sleeping", syncDelay)
hashes = append(hashes, uploadResult{hash: hash, digest: fhash})
time.Sleep(time.Duration(syncDelay) * time.Second)
uploadedBytes += filesize * 1000
for i, v := range hashes {
timeout := time.After(time.Duration(timeout) * time.Second)
errored = false
inner:
for {
select {
case <-timeout:
errored = true
log.Error("error retrieving hash. timeout", "hash idx", i, "err", err)
metrics.GetOrRegisterCounter("sliding-window.single.error", nil).Inc(1)
break inner
default:
idx := 1 + rand.Intn(len(hosts)-1)
ruid := uuid.New()[:8]
start := time.Now()
err := fetch(v.hash, httpEndpoint(hosts[idx]), v.digest, ruid, "")
if err != nil {
continue inner
}
metrics.GetOrRegisterResettingTimer("sliding-window.single.fetch-time", nil).UpdateSince(start)
break inner
}
}
if errored {
break outer
}
networkDepth = i
metrics.GetOrRegisterGauge("sliding-window.network-depth", nil).Update(int64(networkDepth))
}
}
log.Info("sliding window test finished", "errored?", errored, "networkDepth", networkDepth, "networkDepth(kb)", networkDepth*filesize)
log.Info("stats", "uploadedFiles", len(hashes), "uploadedKb", uploadedBytes/1000, "filesizeKb", filesize)
return nil
}

246
cmd/swarm/swarm-smoke/upload_and_sync.go
View file

@ -19,94 +19,122 @@ package main
import (
"bytes"
"context"
"crypto/md5"
crand "crypto/rand"
"errors"
"fmt"
"io"
"io/ioutil"
"math/rand"
"net/http"
"net/http/httptrace"
"os"
"sync"
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/swarm/api"
"github.com/ethereum/go-ethereum/swarm/api/client"
"github.com/ethereum/go-ethereum/swarm/spancontext"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/testutil"
opentracing "github.com/opentracing/opentracing-go"
"github.com/pborman/uuid"
cli "gopkg.in/urfave/cli.v1"
)
func generateEndpoints(scheme string, cluster string, app string, from int, to int) {
if cluster == "prod" {
for port := from; port < to; port++ {
endpoints = append(endpoints, fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, port))
}
} else if cluster == "private-internal" {
for port := from; port < to; port++ {
endpoints = append(endpoints, fmt.Sprintf("%s://swarm-private-internal-%v:8500", scheme, port))
}
} else {
for port := from; port < to; port++ {
endpoints = append(endpoints, fmt.Sprintf("%s://%s-%v-%s.stg.swarm-gateways.net", scheme, app, port, cluster))
}
}
if includeLocalhost {
endpoints = append(endpoints, "http://localhost:8500")
}
}
func cliUploadAndSync(c *cli.Context) error {
log.PrintOrigins(true)
log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(true))))
metrics.GetOrRegisterCounter("upload-and-sync", nil).Inc(1)
func uploadAndSyncCmd(ctx *cli.Context, tuid string) error {
randomBytes := testutil.RandomBytes(seed, filesize*1000)
errc := make(chan error)
go func() {
errc <- uploadAndSync(c)
errc <- uplaodAndSync(ctx, randomBytes, tuid)
}()
select {
case err := <-errc:
if err != nil {
metrics.GetOrRegisterCounter("upload-and-sync.fail", nil).Inc(1)
metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", commandName), nil).Inc(1)
}
return err
case <-time.After(time.Duration(timeout) * time.Second):
metrics.GetOrRegisterCounter("upload-and-sync.timeout", nil).Inc(1)
return fmt.Errorf("timeout after %v sec", timeout)
metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1)
e := fmt.Errorf("timeout after %v sec", timeout)
// trigger debug functionality on randomBytes
err := trackChunks(randomBytes[:])
if err != nil {
e = fmt.Errorf("%v; triggerChunkDebug failed: %v", e, err)
}
return e
}
}
func uploadAndSync(c *cli.Context) error {
defer func(now time.Time) {
totalTime := time.Since(now)
log.Info("total time", "time", totalTime, "kb", filesize)
metrics.GetOrRegisterResettingTimer("upload-and-sync.total-time", nil).Update(totalTime)
}(time.Now())
func trackChunks(testData []byte) error {
log.Warn("Test timed out; running chunk debug sequence")
generateEndpoints(scheme, cluster, appName, from, to)
seed := int(time.Now().UnixNano() / 1e6)
log.Info("uploading to "+endpoints[0]+" and syncing", "seed", seed)
addrs, err := getAllRefs(testData)
if err != nil {
return err
}
log.Trace("All references retrieved")
randomBytes := testutil.RandomBytes(seed, filesize*1000)
// has-chunks
for _, host := range hosts {
httpHost := fmt.Sprintf("ws://%s:%d", host, 8546)
log.Trace("Calling `Has` on host", "httpHost", httpHost)
rpcClient, err := rpc.Dial(httpHost)
if err != nil {
log.Trace("Error dialing host", "err", err)
return err
}
log.Trace("rpc dial ok")
var hasInfo []api.HasInfo
err = rpcClient.Call(&hasInfo, "bzz_has", addrs)
if err != nil {
log.Trace("Error calling host", "err", err)
return err
}
log.Trace("rpc call ok")
count := 0
for _, info := range hasInfo {
if !info.Has {
count++
log.Error("Host does not have chunk", "host", httpHost, "chunk", info.Addr)
}
}
if count == 0 {
log.Info("Host reported to have all chunks", "host", httpHost)
}
}
return nil
}
func getAllRefs(testData []byte) (storage.AddressCollection, error) {
log.Trace("Getting all references for given root hash")
datadir, err := ioutil.TempDir("", "chunk-debug")
if err != nil {
return nil, fmt.Errorf("unable to create temp dir: %v", err)
}
defer os.RemoveAll(datadir)
fileStore, err := storage.NewLocalFileStore(datadir, make([]byte, 32))
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(trackTimeout)*time.Second)
defer cancel()
reader := bytes.NewReader(testData)
return fileStore.GetAllReferences(ctx, reader, false)
}
func uplaodAndSync(c *cli.Context, randomBytes []byte, tuid string) error {
log.Info("uploading to "+httpEndpoint(hosts[0])+" and syncing", "tuid", tuid, "seed", seed)
t1 := time.Now()
hash, err := upload(&randomBytes, endpoints[0])
hash, err := upload(randomBytes, httpEndpoint(hosts[0]))
if err != nil {
log.Error(err.Error())
return err
}
metrics.GetOrRegisterResettingTimer("upload-and-sync.upload-time", nil).UpdateSince(t1)
t2 := time.Since(t1)
metrics.GetOrRegisterResettingTimer("upload-and-sync.upload-time", nil).Update(t2)
fhash, err := digest(bytes.NewReader(randomBytes))
if err != nil {
@ -114,143 +142,53 @@ func uploadAndSync(c *cli.Context) error {
return err
}
log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash))
log.Info("uploaded successfully", "tuid", tuid, "hash", hash, "took", t2, "digest", fmt.Sprintf("%x", fhash))
time.Sleep(time.Duration(syncDelay) * time.Second)
wg := sync.WaitGroup{}
if single {
rand.Seed(time.Now().UTC().UnixNano())
randIndex := 1 + rand.Intn(len(endpoints)-1)
randIndex := 1 + rand.Intn(len(hosts)-1)
ruid := uuid.New()[:8]
wg.Add(1)
go func(endpoint string, ruid string) {
for {
start := time.Now()
err := fetch(hash, endpoint, fhash, ruid)
err := fetch(hash, endpoint, fhash, ruid, tuid)
if err != nil {
continue
}
ended := time.Since(start)
metrics.GetOrRegisterResettingTimer("upload-and-sync.single.fetch-time", nil).UpdateSince(start)
metrics.GetOrRegisterResettingTimer("upload-and-sync.single.fetch-time", nil).Update(ended)
log.Info("fetch successful", "tuid", tuid, "ruid", ruid, "took", ended, "endpoint", endpoint)
wg.Done()
return
}
}(endpoints[randIndex], ruid)
}(httpEndpoint(hosts[randIndex]), ruid)
} else {
for _, endpoint := range endpoints[1:] {
for _, endpoint := range hosts[1:] {
ruid := uuid.New()[:8]
wg.Add(1)
go func(endpoint string, ruid string) {
for {
start := time.Now()
err := fetch(hash, endpoint, fhash, ruid)
err := fetch(hash, endpoint, fhash, ruid, tuid)
if err != nil {
continue
}
ended := time.Since(start)
metrics.GetOrRegisterResettingTimer("upload-and-sync.each.fetch-time", nil).UpdateSince(start)
metrics.GetOrRegisterResettingTimer("upload-and-sync.each.fetch-time", nil).Update(ended)
log.Info("fetch successful", "tuid", tuid, "ruid", ruid, "took", ended, "endpoint", endpoint)
wg.Done()
return
}
}(endpoint, ruid)
}(httpEndpoint(endpoint), ruid)
}
}
wg.Wait()
log.Info("all endpoints synced random file successfully")
log.Info("all hosts synced random file successfully")
return nil
}
// fetch is getting the requested `hash` from the `endpoint` and compares it with the `original` file
func fetch(hash string, endpoint string, original []byte, ruid string) error {
ctx, sp := spancontext.StartSpan(context.Background(), "upload-and-sync.fetch")
defer sp.Finish()
log.Trace("http get request", "ruid", ruid, "api", endpoint, "hash", hash)
var tn time.Time
reqUri := endpoint + "/bzz:/" + hash + "/"
req, _ := http.NewRequest("GET", reqUri, nil)
opentracing.GlobalTracer().Inject(
sp.Context(),
opentracing.HTTPHeaders,
opentracing.HTTPHeadersCarrier(req.Header))
trace := client.GetClientTrace("upload-and-sync - http get", "upload-and-sync", ruid, &tn)
req = req.WithContext(httptrace.WithClientTrace(ctx, trace))
transport := http.DefaultTransport
//transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
tn = time.Now()
res, err := transport.RoundTrip(req)
if err != nil {
log.Error(err.Error(), "ruid", ruid)
return err
}
log.Trace("http get response", "ruid", ruid, "api", endpoint, "hash", hash, "code", res.StatusCode, "len", res.ContentLength)
if res.StatusCode != 200 {
err := fmt.Errorf("expected status code %d, got %v", 200, res.StatusCode)
log.Warn(err.Error(), "ruid", ruid)
return err
}
defer res.Body.Close()
rdigest, err := digest(res.Body)
if err != nil {
log.Warn(err.Error(), "ruid", ruid)
return err
}
if !bytes.Equal(rdigest, original) {
err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original)
log.Warn(err.Error(), "ruid", ruid)
return err
}
log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength)
return nil
}
// upload is uploading a file `f` to `endpoint` via the `swarm up` cmd
func upload(dataBytes *[]byte, endpoint string) (string, error) {
swarm := client.NewClient(endpoint)
f := &client.File{
ReadCloser: ioutil.NopCloser(bytes.NewReader(*dataBytes)),
ManifestEntry: api.ManifestEntry{
ContentType: "text/plain",
Mode: 0660,
Size: int64(len(*dataBytes)),
},
}
// upload data to bzz:// and retrieve the content-addressed manifest hash, hex-encoded.
return swarm.Upload(f, "", false)
}
func digest(r io.Reader) ([]byte, error) {
h := md5.New()
_, err := io.Copy(h, r)
if err != nil {
return nil, err
}
return h.Sum(nil), nil
}
// generates random data in heap buffer
func generateRandomData(datasize int) ([]byte, error) {
b := make([]byte, datasize)
c, err := crand.Read(b)
if err != nil {
return nil, err
} else if c != datasize {
return nil, errors.New("short read")
}
return b, nil
}

51
cmd/swarm/swarm-smoke/upload_speed.go
View file

@ -19,7 +19,6 @@ package main
import (
"bytes"
"fmt"
"os"
"time"
"github.com/ethereum/go-ethereum/log"
@ -29,65 +28,41 @@ import (
cli "gopkg.in/urfave/cli.v1"
)
var endpoint string
//just use the first endpoint
func generateEndpoint(scheme string, cluster string, app string, from int) {
if cluster == "prod" {
endpoint = fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, from)
} else if cluster == "private-internal" {
endpoint = fmt.Sprintf("%s://swarm-private-internal-%v:8500", scheme, from)
} else {
endpoint = fmt.Sprintf("%s://%s-%v-%s.stg.swarm-gateways.net", scheme, app, from, cluster)
}
}
func cliUploadSpeed(c *cli.Context) error {
log.PrintOrigins(true)
log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(true))))
metrics.GetOrRegisterCounter("upload-speed", nil).Inc(1)
func uploadSpeedCmd(ctx *cli.Context, tuid string) error {
log.Info("uploading to "+hosts[0], "tuid", tuid, "seed", seed)
randomBytes := testutil.RandomBytes(seed, filesize*1000)
errc := make(chan error)
go func() {
errc <- uploadSpeed(c)
errc <- uploadSpeed(ctx, tuid, randomBytes)
}()
select {
case err := <-errc:
if err != nil {
metrics.GetOrRegisterCounter("upload-speed.fail", nil).Inc(1)
metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", commandName), nil).Inc(1)
}
return err
case <-time.After(time.Duration(timeout) * time.Second):
metrics.GetOrRegisterCounter("upload-speed.timeout", nil).Inc(1)
metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1)
// trigger debug functionality on randomBytes
return fmt.Errorf("timeout after %v sec", timeout)
}
}
func uploadSpeed(c *cli.Context) error {
defer func(now time.Time) {
totalTime := time.Since(now)
log.Info("total time", "time", totalTime, "kb", filesize)
metrics.GetOrRegisterCounter("upload-speed.total-time", nil).Inc(int64(totalTime))
}(time.Now())
generateEndpoint(scheme, cluster, appName, from)
seed := int(time.Now().UnixNano() / 1e6)
log.Info("uploading to "+endpoint, "seed", seed)
randomBytes := testutil.RandomBytes(seed, filesize*1000)
func uploadSpeed(c *cli.Context, tuid string, data []byte) error {
t1 := time.Now()
hash, err := upload(&randomBytes, endpoint)
hash, err := upload(data, hosts[0])
if err != nil {
log.Error(err.Error())
return err
}
metrics.GetOrRegisterCounter("upload-speed.upload-time", nil).Inc(int64(time.Since(t1)))
fhash, err := digest(bytes.NewReader(randomBytes))
fhash, err := digest(bytes.NewReader(data))
if err != nil {
log.Error(err.Error())
return err

235
cmd/swarm/swarm-smoke/util.go Normal file
View file

@ -0,0 +1,235 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"bytes"
"context"
"crypto/md5"
crand "crypto/rand"
"errors"
"fmt"
"io"
"io/ioutil"
"math/rand"
"net/http"
"net/http/httptrace"
"os"
"strings"
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/swarm/api"
"github.com/ethereum/go-ethereum/swarm/api/client"
"github.com/ethereum/go-ethereum/swarm/spancontext"
opentracing "github.com/opentracing/opentracing-go"
"github.com/pborman/uuid"
cli "gopkg.in/urfave/cli.v1"
)
var (
commandName = ""
seed = int(time.Now().UTC().UnixNano())
)
func init() {
rand.Seed(int64(seed))
}
func httpEndpoint(host string) string {
return fmt.Sprintf("http://%s:%d", host, httpPort)
}
func wsEndpoint(host string) string {
return fmt.Sprintf("ws://%s:%d", host, wsPort)
}
func wrapCliCommand(name string, command func(*cli.Context, string) error) func(*cli.Context) error {
return func(ctx *cli.Context) error {
log.PrintOrigins(true)
log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(false))))
// test uuid
tuid := uuid.New()[:8]
commandName = name
hosts = strings.Split(allhosts, ",")
defer func(now time.Time) {
totalTime := time.Since(now)
log.Info("total time", "tuid", tuid, "time", totalTime, "kb", filesize)
metrics.GetOrRegisterResettingTimer(name+".total-time", nil).Update(totalTime)
}(time.Now())
log.Info("smoke test starting", "tuid", tuid, "task", name, "timeout", timeout)
metrics.GetOrRegisterCounter(name, nil).Inc(1)
return command(ctx, tuid)
}
}
func fetchFeed(topic string, user string, endpoint string, original []byte, ruid string) error {
ctx, sp := spancontext.StartSpan(context.Background(), "feed-and-sync.fetch")
defer sp.Finish()
log.Trace("sleeping", "ruid", ruid)
time.Sleep(3 * time.Second)
log.Trace("http get request (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user)
var tn time.Time
reqUri := endpoint + "/bzz-feed:/?topic=" + topic + "&user=" + user
req, _ := http.NewRequest("GET", reqUri, nil)
opentracing.GlobalTracer().Inject(
sp.Context(),
opentracing.HTTPHeaders,
opentracing.HTTPHeadersCarrier(req.Header))
trace := client.GetClientTrace("feed-and-sync - http get", "feed-and-sync", ruid, &tn)
req = req.WithContext(httptrace.WithClientTrace(ctx, trace))
transport := http.DefaultTransport
//transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
tn = time.Now()
res, err := transport.RoundTrip(req)
if err != nil {
log.Error(err.Error(), "ruid", ruid)
return err
}
log.Trace("http get response (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user, "code", res.StatusCode, "len", res.ContentLength)
if res.StatusCode != 200 {
return fmt.Errorf("expected status code %d, got %v (ruid %v)", 200, res.StatusCode, ruid)
}
defer res.Body.Close()
rdigest, err := digest(res.Body)
if err != nil {
log.Warn(err.Error(), "ruid", ruid)
return err
}
if !bytes.Equal(rdigest, original) {
err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original)
log.Warn(err.Error(), "ruid", ruid)
return err
}
log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength)
return nil
}
// fetch is getting the requested `hash` from the `endpoint` and compares it with the `original` file
func fetch(hash string, endpoint string, original []byte, ruid string, tuid string) error {
ctx, sp := spancontext.StartSpan(context.Background(), "upload-and-sync.fetch")
defer sp.Finish()
log.Info("http get request", "tuid", tuid, "ruid", ruid, "endpoint", endpoint, "hash", hash)
var tn time.Time
reqUri := endpoint + "/bzz:/" + hash + "/"
req, _ := http.NewRequest("GET", reqUri, nil)
opentracing.GlobalTracer().Inject(
sp.Context(),
opentracing.HTTPHeaders,
opentracing.HTTPHeadersCarrier(req.Header))
trace := client.GetClientTrace(commandName+" - http get", commandName, ruid, &tn)
req = req.WithContext(httptrace.WithClientTrace(ctx, trace))
transport := http.DefaultTransport
//transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
tn = time.Now()
res, err := transport.RoundTrip(req)
if err != nil {
log.Error(err.Error(), "ruid", ruid)
return err
}
log.Info("http get response", "tuid", tuid, "ruid", ruid, "endpoint", endpoint, "hash", hash, "code", res.StatusCode, "len", res.ContentLength)
if res.StatusCode != 200 {
err := fmt.Errorf("expected status code %d, got %v", 200, res.StatusCode)
log.Warn(err.Error(), "ruid", ruid)
return err
}
defer res.Body.Close()
rdigest, err := digest(res.Body)
if err != nil {
log.Warn(err.Error(), "ruid", ruid)
return err
}
if !bytes.Equal(rdigest, original) {
err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original)
log.Warn(err.Error(), "ruid", ruid)
return err
}
log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength)
return nil
}
// upload an arbitrary byte as a plaintext file to `endpoint` using the api client
func upload(data []byte, endpoint string) (string, error) {
swarm := client.NewClient(endpoint)
f := &client.File{
ReadCloser: ioutil.NopCloser(bytes.NewReader(data)),
ManifestEntry: api.ManifestEntry{
ContentType: "text/plain",
Mode: 0660,
Size: int64(len(data)),
},
}
// upload data to bzz:// and retrieve the content-addressed manifest hash, hex-encoded.
return swarm.Upload(f, "", false)
}
func digest(r io.Reader) ([]byte, error) {
h := md5.New()
_, err := io.Copy(h, r)
if err != nil {
return nil, err
}
return h.Sum(nil), nil
}
// generates random data in heap buffer
func generateRandomData(datasize int) ([]byte, error) {
b := make([]byte, datasize)
c, err := crand.Read(b)
if err != nil {
return nil, err
} else if c != datasize {
return nil, errors.New("short read")
}
return b, nil
}

4
cmd/swarm/upload.go
View file

@ -60,10 +60,10 @@ func upload(ctx *cli.Context) {
autoDefaultPath = false
file string
)
if autoDefaultPathString := os.Getenv(SWARM_AUTO_DEFAULTPATH); autoDefaultPathString != "" {
if autoDefaultPathString := os.Getenv(SwarmAutoDefaultPath); autoDefaultPathString != "" {
b, err := strconv.ParseBool(autoDefaultPathString)
if err != nil {
utils.Fatalf("invalid environment variable %s: %v", SWARM_AUTO_DEFAULTPATH, err)
utils.Fatalf("invalid environment variable %s: %v", SwarmAutoDefaultPath, err)
}
autoDefaultPath = b
}

61
cmd/utils/flags.go
View file

@ -141,6 +141,10 @@ var (
Name: "rinkeby",
Usage: "Rinkeby network: pre-configured proof-of-authority test network",
}
GoerliFlag = cli.BoolFlag{
Name: "goerli",
Usage: "Görli network: pre-configured proof-of-authority test network",
}
ConstantinopleOverrideFlag = cli.Uint64Flag{
Name: "override.constantinople",
Usage: "Manually specify constantinople fork-block, overriding the bundled setting",
@ -162,6 +166,10 @@ var (
Usage: "Document Root for HTTPClient file scheme",
Value: DirectoryString{homeDir()},
}
ExitWhenSyncedFlag = cli.BoolFlag{
Name: "exitwhensynced",
Usage: "Exists syncing after block synchronisation",
}
ULCModeConfigFlag = cli.StringFlag{
Name: "ulc.config",
Usage: "Config file to use for ultra light client mode",
@ -178,7 +186,6 @@ var (
Name: "ulc.trusted",
Usage: "List of trusted ULC servers",
}
defaultSyncMode = eth.DefaultConfig.SyncMode
SyncModeFlag = TextMarshalerFlag{
Name: "syncmode",
@ -325,12 +332,12 @@ var (
}
CacheTrieFlag = cli.IntFlag{
Name: "cache.trie",
Usage: "Percentage of cache memory allowance to use for trie caching",
Usage: "Percentage of cache memory allowance to use for trie caching (default = 25% full mode, 50% archive mode)",
Value: 25,
}
CacheGCFlag = cli.IntFlag{
Name: "cache.gc",
Usage: "Percentage of cache memory allowance to use for trie pruning",
Usage: "Percentage of cache memory allowance to use for trie pruning (default = 25% full mode, 0% archive mode)",
Value: 25,
}
TrieCacheGenFlag = cli.IntFlag{
@ -420,7 +427,11 @@ var (
Usage: "Password file to use for non-interactive password input",
Value: "",
}
ExternalSignerFlag = cli.StringFlag{
Name: "signer",
Usage: "External signer (url or path to ipc file)",
Value: "",
}
VMEnableDebugFlag = cli.BoolFlag{
Name: "vmdebug",
Usage: "Record information useful for VM and contract debugging",
@ -689,6 +700,9 @@ func MakeDataDir(ctx *cli.Context) string {
if ctx.GlobalBool(RinkebyFlag.Name) {
return filepath.Join(path, "rinkeby")
}
if ctx.GlobalBool(GoerliFlag.Name) {
return filepath.Join(path, "goerli")
}
return path
}
Fatalf("Cannot determine default data directory, please set manually (--datadir)")
@ -743,6 +757,8 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) {
urls = params.TestnetBootnodes
case ctx.GlobalBool(RinkebyFlag.Name):
urls = params.RinkebyBootnodes
case ctx.GlobalBool(GoerliFlag.Name):
urls = params.GoerliBootnodes
case cfg.BootstrapNodes != nil:
return // already set, don't apply defaults.
}
@ -772,6 +788,8 @@ func setBootstrapNodesV5(ctx *cli.Context, cfg *p2p.Config) {
}
case ctx.GlobalBool(RinkebyFlag.Name):
urls = params.RinkebyBootnodes
case ctx.GlobalBool(GoerliFlag.Name):
urls = params.GoerliBootnodes
case cfg.BootstrapNodesV5 != nil:
return // already set, don't apply defaults.
}
@ -932,10 +950,11 @@ func makeDatabaseHandles() int {
if err != nil {
Fatalf("Failed to retrieve file descriptor allowance: %v", err)
}
if err := fdlimit.Raise(uint64(limit)); err != nil {
raised, err := fdlimit.Raise(uint64(limit))
if err != nil {
Fatalf("Failed to raise file descriptor allowance: %v", err)
}
return limit / 2 // Leave half for networking and other stuff
return int(raised / 2) // Leave half for networking and other stuff
}
// MakeAddress converts an account specified directly as a hex encoded string or
@ -976,11 +995,15 @@ func setEtherbase(ctx *cli.Context, ks *keystore.KeyStore, cfg *eth.Config) {
}
// Convert the etherbase into an address and configure it
if etherbase != "" {
if ks != nil {
account, err := MakeAddress(ks, etherbase)
if err != nil {
Fatalf("Invalid miner etherbase: %v", err)
}
cfg.Etherbase = account.Address
} else {
Fatalf("No etherbase configured")
}
}
}
@ -1077,9 +1100,12 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
setGraphQL(ctx, cfg)
setWS(ctx, cfg)
setNodeUserIdent(ctx, cfg)
setDataDir(ctx, cfg)
if ctx.GlobalIsSet(ExternalSignerFlag.Name) {
cfg.ExternalSigner = ctx.GlobalString(ExternalSignerFlag.Name)
}
if ctx.GlobalIsSet(KeyStoreDirFlag.Name) {
cfg.KeyStoreDir = ctx.GlobalString(KeyStoreDirFlag.Name)
}
@ -1101,6 +1127,8 @@ func setDataDir(ctx *cli.Context, cfg *node.Config) {
cfg.DataDir = filepath.Join(node.DefaultDataDir(), "testnet")
case ctx.GlobalBool(RinkebyFlag.Name):
cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby")
case ctx.GlobalBool(GoerliFlag.Name):
cfg.DataDir = filepath.Join(node.DefaultDataDir(), "goerli")
}
}
@ -1257,10 +1285,14 @@ func SetShhConfig(ctx *cli.Context, stack *node.Node, cfg *whisper.Config) {
// SetEthConfig applies eth-related command line flags to the config.
func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
// Avoid conflicting network flags
checkExclusive(ctx, DeveloperFlag, TestnetFlag, RinkebyFlag)
checkExclusive(ctx, DeveloperFlag, TestnetFlag, RinkebyFlag, GoerliFlag)
checkExclusive(ctx, LightServFlag, SyncModeFlag, "light")
ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
// Can't use both ephemeral unlocked and external signer
checkExclusive(ctx, DeveloperFlag, ExternalSignerFlag)
var ks *keystore.KeyStore
if keystores := stack.AccountManager().Backends(keystore.KeyStoreType); len(keystores) > 0 {
ks = keystores[0].(*keystore.KeyStore)
}
setEtherbase(ctx, ks, cfg)
setGPO(ctx, &cfg.GPO)
setTxPool(ctx, &cfg.TxPool)
@ -1356,6 +1388,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
cfg.NetworkId = 4
}
cfg.Genesis = core.DefaultRinkebyGenesisBlock()
case ctx.GlobalBool(GoerliFlag.Name):
if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
cfg.NetworkId = 5
}
cfg.Genesis = core.DefaultGoerliGenesisBlock()
case ctx.GlobalBool(DeveloperFlag.Name):
if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
cfg.NetworkId = 1337
@ -1513,6 +1550,8 @@ func MakeGenesis(ctx *cli.Context) *core.Genesis {
genesis = core.DefaultTestnetGenesisBlock()
case ctx.GlobalBool(RinkebyFlag.Name):
genesis = core.DefaultRinkebyGenesisBlock()
case ctx.GlobalBool(GoerliFlag.Name):
genesis = core.DefaultGoerliGenesisBlock()
case ctx.GlobalBool(DeveloperFlag.Name):
Fatalf("Developer chains are ephemeral")
}
@ -1574,7 +1613,7 @@ func MakeConsolePreloads(ctx *cli.Context) []string {
return nil
}
// Otherwise resolve absolute paths and return them
preloads := []string{}
var preloads []string
assets := ctx.GlobalString(JSpathFlag.Name)
for _, file := range strings.Split(ctx.GlobalString(PreloadJSFlag.Name), ",") {

3
common/compiler/solidity_test.go
View file

@ -23,9 +23,10 @@ import (
const (
testSource = `
pragma solidity >0.0.0;
contract test {
/// @notice Will multiply ` + "`a`" + ` by 7.
function multiply(uint a) returns(uint d) {
function multiply(uint a) public returns(uint d) {
return a * 7;
}
}

11
common/fdlimit/fdlimit_freebsd.go
View file

@ -26,11 +26,11 @@ import "syscall"
// Raise tries to maximize the file descriptor allowance of this process
// to the maximum hard-limit allowed by the OS.
func Raise(max uint64) error {
func Raise(max uint64) (uint64, error) {
// Get the current limit
var limit syscall.Rlimit
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
return err
return 0, err
}
// Try to update the limit to the max allowance
limit.Cur = limit.Max
@ -38,9 +38,12 @@ func Raise(max uint64) error {
limit.Cur = int64(max)
}
if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
return err
return 0, err
}
return nil
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
return 0, err
}
return uint64(limit.Cur), nil
}
// Current retrieves the number of file descriptors allowed to be opened by this

2
common/fdlimit/fdlimit_test.go
View file

@ -36,7 +36,7 @@ func TestFileDescriptorLimits(t *testing.T) {
if limit, err := Current(); err != nil || limit <= 0 {
t.Fatalf("failed to retrieve file descriptor limit (%d): %v", limit, err)
}
if err := Raise(uint64(target)); err != nil {
if _, err := Raise(uint64(target)); err != nil {
t.Fatalf("failed to raise file allowance")
}
if limit, err := Current(); err != nil || limit < target {

13
common/fdlimit/fdlimit_unix.go
View file

@ -22,11 +22,12 @@ import "syscall"
// Raise tries to maximize the file descriptor allowance of this process
// to the maximum hard-limit allowed by the OS.
func Raise(max uint64) error {
// Returns the size it was set to (may differ from the desired 'max')
func Raise(max uint64) (uint64, error) {
// Get the current limit
var limit syscall.Rlimit
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
return err
return 0, err
}
// Try to update the limit to the max allowance
limit.Cur = limit.Max
@ -34,9 +35,13 @@ func Raise(max uint64) error {
limit.Cur = max
}
if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
return err
return 0, err
}
return nil
// MacOS can silently apply further caps, so retrieve the actually set limit
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
return 0, err
}
return limit.Cur, nil
}
// Current retrieves the number of file descriptors allowed to be opened by this

14
common/fdlimit/fdlimit_windows.go
View file

@ -16,28 +16,30 @@
package fdlimit
import "errors"
import "fmt"
const hardlimit = 16384
// Raise tries to maximize the file descriptor allowance of this process
// to the maximum hard-limit allowed by the OS.
func Raise(max uint64) error {
func Raise(max uint64) (uint64, error) {
// This method is NOP by design:
// * Linux/Darwin counterparts need to manually increase per process limits
// * On Windows Go uses the CreateFile API, which is limited to 16K files, non
// changeable from within a running process
// This way we can always "request" raising the limits, which will either have
// or not have effect based on the platform we're running on.
if max > 16384 {
return errors.New("file descriptor limit (16384) reached")
if max > hardlimit {
return hardlimit, fmt.Errorf("file descriptor limit (%d) reached", hardlimit)
}
return nil
return max, nil
}
// Current retrieves the number of file descriptors allowed to be opened by this
// process.
func Current() (int, error) {
// Please see Raise for the reason why we use hard coded 16K as the limit
return 16384, nil
return hardlimit, nil
}
// Maximum retrieves the maximum number of file descriptors this process is

16
common/size.go
View file

@ -26,10 +26,10 @@ type StorageSize float64
// String implements the stringer interface.
func (s StorageSize) String() string {
if s > 1000000 {
return fmt.Sprintf("%.2f mB", s/1000000)
} else if s > 1000 {
return fmt.Sprintf("%.2f kB", s/1000)
if s > 1048576 {
return fmt.Sprintf("%.2f MiB", s/1048576)
} else if s > 1024 {
return fmt.Sprintf("%.2f KiB", s/1024)
} else {
return fmt.Sprintf("%.2f B", s)
}
@ -38,10 +38,10 @@ func (s StorageSize) String() string {
// TerminalString implements log.TerminalStringer, formatting a string for console
// output during logging.
func (s StorageSize) TerminalString() string {
if s > 1000000 {
return fmt.Sprintf("%.2fmB", s/1000000)
} else if s > 1000 {
return fmt.Sprintf("%.2fkB", s/1000)
if s > 1048576 {
return fmt.Sprintf("%.2fMiB", s/1048576)
} else if s > 1024 {
return fmt.Sprintf("%.2fKiB", s/1024)
} else {
return fmt.Sprintf("%.2fB", s)
}

4
common/size_test.go
View file

@ -25,8 +25,8 @@ func TestStorageSizeString(t *testing.T) {
size StorageSize
str string
}{
{2381273, "2.38 mB"},
{2192, "2.19 kB"},
{2381273, "2.27 MiB"},
{2192, "2.14 KiB"},
{12, "12.00 B"},
}

88
consensus/clique/clique.go
View file

@ -20,6 +20,7 @@ package clique
import (
"bytes"
"errors"
"io"
"math/big"
"math/rand"
"sync"
@ -136,40 +137,9 @@ var (
errRecentlySigned = errors.New("recently signed")
)
// SignerFn is a signer callback function to request a hash to be signed by a
// SignerFn is a signer callback function to request a header to be signed by a
// backing account.
type SignerFn func(accounts.Account, []byte) ([]byte, error)
// sigHash returns the hash which is used as input for the proof-of-authority
// signing. It is the hash of the entire header apart from the 65 byte signature
// contained at the end of the extra data.
//
// Note, the method requires the extra data to be at least 65 bytes, otherwise it
// panics. This is done to avoid accidentally using both forms (signature present
// or not), which could be abused to produce different hashes for the same header.
func sigHash(header *types.Header) (hash common.Hash) {
hasher := sha3.NewLegacyKeccak256()
rlp.Encode(hasher, []interface{}{
header.ParentHash,
header.UncleHash,
header.Coinbase,
header.Root,
header.TxHash,
header.ReceiptHash,
header.Bloom,
header.Difficulty,
header.Number,
header.GasLimit,
header.GasUsed,
header.Time,
header.Extra[:len(header.Extra)-65], // Yes, this will panic if extra is too short
header.MixDigest,
header.Nonce,
})
hasher.Sum(hash[:0])
return hash
}
type SignerFn func(accounts.Account, string, []byte) ([]byte, error)
// ecrecover extracts the Ethereum account address from a signed header.
func ecrecover(header *types.Header, sigcache *lru.ARCCache) (common.Address, error) {
@ -185,7 +155,7 @@ func ecrecover(header *types.Header, sigcache *lru.ARCCache) (common.Address, er
signature := header.Extra[len(header.Extra)-extraSeal:]
// Recover the public key and the Ethereum address
pubkey, err := crypto.Ecrecover(sigHash(header).Bytes(), signature)
pubkey, err := crypto.Ecrecover(SealHash(header).Bytes(), signature)
if err != nil {
return common.Address{}, err
}
@ -646,7 +616,7 @@ func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, results c
log.Trace("Out-of-turn signing requested", "wiggle", common.PrettyDuration(wiggle))
}
// Sign all the things!
sighash, err := signFn(accounts.Account{Address: signer}, sigHash(header).Bytes())
sighash, err := signFn(accounts.Account{Address: signer}, accounts.MimetypeClique, CliqueRLP(header))
if err != nil {
return err
}
@ -663,7 +633,7 @@ func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, results c
select {
case results <- block.WithSeal(header):
default:
log.Warn("Sealing result is not read by miner", "sealhash", c.SealHash(header))
log.Warn("Sealing result is not read by miner", "sealhash", SealHash(header))
}
}()
@ -693,7 +663,7 @@ func CalcDifficulty(snap *Snapshot, signer common.Address) *big.Int {
// SealHash returns the hash of a block prior to it being sealed.
func (c *Clique) SealHash(header *types.Header) common.Hash {
return sigHash(header)
return SealHash(header)
}
// Close implements consensus.Engine. It's a noop for clique as there are no background threads.
@ -711,3 +681,47 @@ func (c *Clique) APIs(chain consensus.ChainReader) []rpc.API {
Public: false,
}}
}
// SealHash returns the hash of a block prior to it being sealed.
func SealHash(header *types.Header) (hash common.Hash) {
hasher := sha3.NewLegacyKeccak256()
encodeSigHeader(hasher, header)
hasher.Sum(hash[:0])
return hash
}
// CliqueRLP returns the rlp bytes which needs to be signed for the proof-of-authority
// sealing. The RLP to sign consists of the entire header apart from the 65 byte signature
// contained at the end of the extra data.
//
// Note, the method requires the extra data to be at least 65 bytes, otherwise it
// panics. This is done to avoid accidentally using both forms (signature present
// or not), which could be abused to produce different hashes for the same header.
func CliqueRLP(header *types.Header) []byte {
b := new(bytes.Buffer)
encodeSigHeader(b, header)
return b.Bytes()
}
func encodeSigHeader(w io.Writer, header *types.Header) {
err := rlp.Encode(w, []interface{}{
header.ParentHash,
header.UncleHash,
header.Coinbase,
header.Root,
header.TxHash,
header.ReceiptHash,
header.Bloom,
header.Difficulty,
header.Number,
header.GasLimit,
header.GasUsed,
header.Time,
header.Extra[:len(header.Extra)-65], // Yes, this will panic if extra is too short
header.MixDigest,
header.Nonce,
})
if err != nil {
panic("can't encode: " + err.Error())
}
}

2
consensus/clique/snapshot_test.go
View file

@ -80,7 +80,7 @@ func (ap *testerAccountPool) sign(header *types.Header, signer string) {
ap.accounts[signer], _ = crypto.GenerateKey()
}
// Sign the header and embed the signature in extra data
sig, _ := crypto.Sign(sigHash(header).Bytes(), ap.accounts[signer])
sig, _ := crypto.Sign(SealHash(header).Bytes(), ap.accounts[signer])
copy(header.Extra[len(header.Extra)-extraSeal:], sig)
}

6
console/console.go
View file

@ -120,10 +120,10 @@ func (c *Console) init(preload []string) error {
consoleObj.Object().Set("error", c.consoleOutput)
// Load all the internal utility JavaScript libraries
if err := c.jsre.Compile("bignumber.js", jsre.BigNumber_JS); err != nil {
if err := c.jsre.Compile("bignumber.js", jsre.BignumberJs); err != nil {
return fmt.Errorf("bignumber.js: %v", err)
}
if err := c.jsre.Compile("web3.js", jsre.Web3_JS); err != nil {
if err := c.jsre.Compile("web3.js", jsre.Web3Js); err != nil {
return fmt.Errorf("web3.js: %v", err)
}
if _, err := c.jsre.Run("var Web3 = require('web3');"); err != nil {
@ -236,7 +236,7 @@ func (c *Console) clearHistory() {
// consoleOutput is an override for the console.log and console.error methods to
// stream the output into the configured output stream instead of stdout.
func (c *Console) consoleOutput(call otto.FunctionCall) otto.Value {
output := []string{}
var output []string
for _, argument := range call.ArgumentList {
output = append(output, fmt.Sprintf("%v", argument))
}

4
console/console_test.go
View file

@ -149,8 +149,8 @@ func (env *tester) Close(t *testing.T) {
if err := env.console.Stop(false); err != nil {
t.Errorf("failed to stop embedded console: %v", err)
}
if err := env.stack.Stop(); err != nil {
t.Errorf("failed to stop embedded node: %v", err)
if err := env.stack.Close(); err != nil {
t.Errorf("failed to tear down embedded node: %v", err)
}
os.RemoveAll(env.workspace)
}

22
contracts/chequebook/api.go
View file

@ -27,40 +27,40 @@ const Version = "1.0"
var errNoChequebook = errors.New("no chequebook")
type Api struct {
type API struct {
chequebookf func() *Chequebook
}
func NewApi(ch func() *Chequebook) *Api {
return &Api{ch}
func NewAPI(ch func() *Chequebook) *API {
return &API{ch}
}
func (self *Api) Balance() (string, error) {
ch := self.chequebookf()
func (a *API) Balance() (string, error) {
ch := a.chequebookf()
if ch == nil {
return "", errNoChequebook
}
return ch.Balance().String(), nil
}
func (self *Api) Issue(beneficiary common.Address, amount *big.Int) (cheque *Cheque, err error) {
ch := self.chequebookf()
func (a *API) Issue(beneficiary common.Address, amount *big.Int) (cheque *Cheque, err error) {
ch := a.chequebookf()
if ch == nil {
return nil, errNoChequebook
}
return ch.Issue(beneficiary, amount)
}
func (self *Api) Cash(cheque *Cheque) (txhash string, err error) {
ch := self.chequebookf()
func (a *API) Cash(cheque *Cheque) (txhash string, err error) {
ch := a.chequebookf()
if ch == nil {
return "", errNoChequebook
}
return ch.Cash(cheque)
}
func (self *Api) Deposit(amount *big.Int) (txhash string, err error) {
ch := self.chequebookf()
func (a *API) Deposit(amount *big.Int) (txhash string, err error) {
ch := a.chequebookf()
if ch == nil {
return "", errNoChequebook
}

341
contracts/chequebook/cheque.go
View file

@ -75,8 +75,8 @@ type Cheque struct {
Sig []byte // signature Sign(Keccak256(contract, beneficiary, amount), prvKey)
}
func (self *Cheque) String() string {
return fmt.Sprintf("contract: %s, beneficiary: %s, amount: %v, signature: %x", self.Contract.Hex(), self.Beneficiary.Hex(), self.Amount, self.Sig)
func (ch *Cheque) String() string {
return fmt.Sprintf("contract: %s, beneficiary: %s, amount: %v, signature: %x", ch.Contract.Hex(), ch.Beneficiary.Hex(), ch.Amount, ch.Sig)
}
type Params struct {
@ -109,12 +109,12 @@ type Chequebook struct {
log log.Logger // contextual logger with the contract address embedded
}
func (self *Chequebook) String() string {
return fmt.Sprintf("contract: %s, owner: %s, balance: %v, signer: %x", self.contractAddr.Hex(), self.owner.Hex(), self.balance, self.prvKey.PublicKey)
func (cb *Chequebook) String() string {
return fmt.Sprintf("contract: %s, owner: %s, balance: %v, signer: %x", cb.contractAddr.Hex(), cb.owner.Hex(), cb.balance, cb.prvKey.PublicKey)
}
// NewChequebook creates a new Chequebook.
func NewChequebook(path string, contractAddr common.Address, prvKey *ecdsa.PrivateKey, backend Backend) (self *Chequebook, err error) {
func NewChequebook(path string, contractAddr common.Address, prvKey *ecdsa.PrivateKey, backend Backend) (*Chequebook, error) {
balance := new(big.Int)
sent := make(map[common.Address]*big.Int)
@ -128,7 +128,7 @@ func NewChequebook(path string, contractAddr common.Address, prvKey *ecdsa.Priva
TransactOpts: *transactOpts,
}
self = &Chequebook{
cb := &Chequebook{
prvKey: prvKey,
balance: balance,
contractAddr: contractAddr,
@ -140,42 +140,39 @@ func NewChequebook(path string, contractAddr common.Address, prvKey *ecdsa.Priva
session: session,
log: log.New("contract", contractAddr),
}
if (contractAddr != common.Address{}) {
self.setBalanceFromBlockChain()
self.log.Trace("New chequebook initialised", "owner", self.owner, "balance", self.balance)
cb.setBalanceFromBlockChain()
cb.log.Trace("New chequebook initialised", "owner", cb.owner, "balance", cb.balance)
}
return
return cb, nil
}
func (self *Chequebook) setBalanceFromBlockChain() {
balance, err := self.backend.BalanceAt(context.TODO(), self.contractAddr, nil)
func (cb *Chequebook) setBalanceFromBlockChain() {
balance, err := cb.backend.BalanceAt(context.TODO(), cb.contractAddr, nil)
if err != nil {
log.Error("Failed to retrieve chequebook balance", "err", err)
} else {
self.balance.Set(balance)
cb.balance.Set(balance)
}
}
// LoadChequebook loads a chequebook from disk (file path).
func LoadChequebook(path string, prvKey *ecdsa.PrivateKey, backend Backend, checkBalance bool) (self *Chequebook, err error) {
var data []byte
data, err = ioutil.ReadFile(path)
if err != nil {
return
}
self, _ = NewChequebook(path, common.Address{}, prvKey, backend)
err = json.Unmarshal(data, self)
func LoadChequebook(path string, prvKey *ecdsa.PrivateKey, backend Backend, checkBalance bool) (*Chequebook, error) {
data, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
cb, _ := NewChequebook(path, common.Address{}, prvKey, backend)
if err = json.Unmarshal(data, cb); err != nil {
return nil, err
}
if checkBalance {
self.setBalanceFromBlockChain()
cb.setBalanceFromBlockChain()
}
log.Trace("Loaded chequebook from disk", "path", path)
return
return cb, nil
}
// chequebookFile is the JSON representation of a chequebook.
@ -187,19 +184,19 @@ type chequebookFile struct {
}
// UnmarshalJSON deserialises a chequebook.
func (self *Chequebook) UnmarshalJSON(data []byte) error {
func (cb *Chequebook) UnmarshalJSON(data []byte) error {
var file chequebookFile
err := json.Unmarshal(data, &file)
if err != nil {
return err
}
_, ok := self.balance.SetString(file.Balance, 10)
_, ok := cb.balance.SetString(file.Balance, 10)
if !ok {
return fmt.Errorf("cumulative amount sent: unable to convert string to big integer: %v", file.Balance)
}
self.contractAddr = common.HexToAddress(file.Contract)
cb.contractAddr = common.HexToAddress(file.Contract)
for addr, sent := range file.Sent {
self.sent[common.HexToAddress(addr)], ok = new(big.Int).SetString(sent, 10)
cb.sent[common.HexToAddress(addr)], ok = new(big.Int).SetString(sent, 10)
if !ok {
return fmt.Errorf("beneficiary %v cumulative amount sent: unable to convert string to big integer: %v", addr, sent)
}
@ -208,14 +205,14 @@ func (self *Chequebook) UnmarshalJSON(data []byte) error {
}
// MarshalJSON serialises a chequebook.
func (self *Chequebook) MarshalJSON() ([]byte, error) {
func (cb *Chequebook) MarshalJSON() ([]byte, error) {
var file = &chequebookFile{
Balance: self.balance.String(),
Contract: self.contractAddr.Hex(),
Owner: self.owner.Hex(),
Balance: cb.balance.String(),
Contract: cb.contractAddr.Hex(),
Owner: cb.owner.Hex(),
Sent: make(map[string]string),
}
for addr, sent := range self.sent {
for addr, sent := range cb.sent {
file.Sent[addr.Hex()] = sent.String()
}
return json.Marshal(file)
@ -223,76 +220,78 @@ func (self *Chequebook) MarshalJSON() ([]byte, error) {
// Save persists the chequebook on disk, remembering balance, contract address and
// cumulative amount of funds sent for each beneficiary.
func (self *Chequebook) Save() (err error) {
data, err := json.MarshalIndent(self, "", " ")
func (cb *Chequebook) Save() error {
data, err := json.MarshalIndent(cb, "", " ")
if err != nil {
return err
}
self.log.Trace("Saving chequebook to disk", self.path)
cb.log.Trace("Saving chequebook to disk", cb.path)
return ioutil.WriteFile(self.path, data, os.ModePerm)
return ioutil.WriteFile(cb.path, data, os.ModePerm)
}
// Stop quits the autodeposit go routine to terminate
func (self *Chequebook) Stop() {
defer self.lock.Unlock()
self.lock.Lock()
if self.quit != nil {
close(self.quit)
self.quit = nil
func (cb *Chequebook) Stop() {
defer cb.lock.Unlock()
cb.lock.Lock()
if cb.quit != nil {
close(cb.quit)
cb.quit = nil
}
}
// Issue creates a cheque signed by the chequebook owner's private key. The
// signer commits to a contract (one that they own), a beneficiary and amount.
func (self *Chequebook) Issue(beneficiary common.Address, amount *big.Int) (ch *Cheque, err error) {
defer self.lock.Unlock()
self.lock.Lock()
func (cb *Chequebook) Issue(beneficiary common.Address, amount *big.Int) (*Cheque, error) {
defer cb.lock.Unlock()
cb.lock.Lock()
if amount.Sign() <= 0 {
return nil, fmt.Errorf("amount must be greater than zero (%v)", amount)
}
if self.balance.Cmp(amount) < 0 {
err = fmt.Errorf("insufficient funds to issue cheque for amount: %v. balance: %v", amount, self.balance)
var (
ch *Cheque
err error
)
if cb.balance.Cmp(amount) < 0 {
err = fmt.Errorf("insufficient funds to issue cheque for amount: %v. balance: %v", amount, cb.balance)
} else {
var sig []byte
sent, found := self.sent[beneficiary]
sent, found := cb.sent[beneficiary]
if !found {
sent = new(big.Int)
self.sent[beneficiary] = sent
cb.sent[beneficiary] = sent
}
sum := new(big.Int).Set(sent)
sum.Add(sum, amount)
sig, err = crypto.Sign(sigHash(self.contractAddr, beneficiary, sum), self.prvKey)
sig, err = crypto.Sign(sigHash(cb.contractAddr, beneficiary, sum), cb.prvKey)
if err == nil {
ch = &Cheque{
Contract: self.contractAddr,
Contract: cb.contractAddr,
Beneficiary: beneficiary,
Amount: sum,
Sig: sig,
}
sent.Set(sum)
self.balance.Sub(self.balance, amount) // subtract amount from balance
cb.balance.Sub(cb.balance, amount) // subtract amount from balance
}
}
// auto deposit if threshold is set and balance is less then threshold
// note this is called even if issuing cheque fails
// so we reattempt depositing
if self.threshold != nil {
if self.balance.Cmp(self.threshold) < 0 {
send := new(big.Int).Sub(self.buffer, self.balance)
self.deposit(send)
if cb.threshold != nil {
if cb.balance.Cmp(cb.threshold) < 0 {
send := new(big.Int).Sub(cb.buffer, cb.balance)
cb.deposit(send)
}
}
return
return ch, err
}
// Cash is a convenience method to cash any cheque.
func (self *Chequebook) Cash(ch *Cheque) (txhash string, err error) {
return ch.Cash(self.session)
func (cb *Chequebook) Cash(ch *Cheque) (string, error) {
return ch.Cash(cb.session)
}
// data to sign: contract address, beneficiary, cumulative amount of funds ever sent
@ -309,73 +308,73 @@ func sigHash(contract, beneficiary common.Address, sum *big.Int) []byte {
}
// Balance returns the current balance of the chequebook.
func (self *Chequebook) Balance() *big.Int {
defer self.lock.Unlock()
self.lock.Lock()
return new(big.Int).Set(self.balance)
func (cb *Chequebook) Balance() *big.Int {
defer cb.lock.Unlock()
cb.lock.Lock()
return new(big.Int).Set(cb.balance)
}
// Owner returns the owner account of the chequebook.
func (self *Chequebook) Owner() common.Address {
return self.owner
func (cb *Chequebook) Owner() common.Address {
return cb.owner
}
// Address returns the on-chain contract address of the chequebook.
func (self *Chequebook) Address() common.Address {
return self.contractAddr
func (cb *Chequebook) Address() common.Address {
return cb.contractAddr
}
// Deposit deposits money to the chequebook account.
func (self *Chequebook) Deposit(amount *big.Int) (string, error) {
defer self.lock.Unlock()
self.lock.Lock()
return self.deposit(amount)
func (cb *Chequebook) Deposit(amount *big.Int) (string, error) {
defer cb.lock.Unlock()
cb.lock.Lock()
return cb.deposit(amount)
}
// deposit deposits amount to the chequebook account.
// The caller must hold self.lock.
func (self *Chequebook) deposit(amount *big.Int) (string, error) {
// The caller must hold lock.
func (cb *Chequebook) deposit(amount *big.Int) (string, error) {
// since the amount is variable here, we do not use sessions
depositTransactor := bind.NewKeyedTransactor(self.prvKey)
depositTransactor := bind.NewKeyedTransactor(cb.prvKey)
depositTransactor.Value = amount
chbookRaw := &contract.ChequebookRaw{Contract: self.contract}
chbookRaw := &contract.ChequebookRaw{Contract: cb.contract}
tx, err := chbookRaw.Transfer(depositTransactor)
if err != nil {
self.log.Warn("Failed to fund chequebook", "amount", amount, "balance", self.balance, "target", self.buffer, "err", err)
cb.log.Warn("Failed to fund chequebook", "amount", amount, "balance", cb.balance, "target", cb.buffer, "err", err)
return "", err
}
// assume that transaction is actually successful, we add the amount to balance right away
self.balance.Add(self.balance, amount)
self.log.Trace("Deposited funds to chequebook", "amount", amount, "balance", self.balance, "target", self.buffer)
cb.balance.Add(cb.balance, amount)
cb.log.Trace("Deposited funds to chequebook", "amount", amount, "balance", cb.balance, "target", cb.buffer)
return tx.Hash().Hex(), nil
}
// AutoDeposit (re)sets interval time and amount which triggers sending funds to the
// chequebook. Contract backend needs to be set if threshold is not less than buffer, then
// deposit will be triggered on every new cheque issued.
func (self *Chequebook) AutoDeposit(interval time.Duration, threshold, buffer *big.Int) {
defer self.lock.Unlock()
self.lock.Lock()
self.threshold = threshold
self.buffer = buffer
self.autoDeposit(interval)
func (cb *Chequebook) AutoDeposit(interval time.Duration, threshold, buffer *big.Int) {
defer cb.lock.Unlock()
cb.lock.Lock()
cb.threshold = threshold
cb.buffer = buffer
cb.autoDeposit(interval)
}
// autoDeposit starts a goroutine that periodically sends funds to the chequebook
// contract caller holds the lock the go routine terminates if Chequebook.quit is closed.
func (self *Chequebook) autoDeposit(interval time.Duration) {
if self.quit != nil {
close(self.quit)
self.quit = nil
func (cb *Chequebook) autoDeposit(interval time.Duration) {
if cb.quit != nil {
close(cb.quit)
cb.quit = nil
}
// if threshold >= balance autodeposit after every cheque issued
if interval == time.Duration(0) || self.threshold != nil && self.buffer != nil && self.threshold.Cmp(self.buffer) >= 0 {
if interval == time.Duration(0) || cb.threshold != nil && cb.buffer != nil && cb.threshold.Cmp(cb.buffer) >= 0 {
return
}
ticker := time.NewTicker(interval)
self.quit = make(chan bool)
quit := self.quit
cb.quit = make(chan bool)
quit := cb.quit
go func() {
for {
@ -383,15 +382,15 @@ func (self *Chequebook) autoDeposit(interval time.Duration) {
case <-quit:
return
case <-ticker.C:
self.lock.Lock()
if self.balance.Cmp(self.buffer) < 0 {
amount := new(big.Int).Sub(self.buffer, self.balance)
txhash, err := self.deposit(amount)
cb.lock.Lock()
if cb.balance.Cmp(cb.buffer) < 0 {
amount := new(big.Int).Sub(cb.buffer, cb.balance)
txhash, err := cb.deposit(amount)
if err == nil {
self.txhash = txhash
cb.txhash = txhash
}
}
self.lock.Unlock()
cb.lock.Unlock()
}
}
}()
@ -404,26 +403,26 @@ type Outbox struct {
}
// NewOutbox creates an outbox.
func NewOutbox(chbook *Chequebook, beneficiary common.Address) *Outbox {
return &Outbox{chbook, beneficiary}
func NewOutbox(cb *Chequebook, beneficiary common.Address) *Outbox {
return &Outbox{cb, beneficiary}
}
// Issue creates cheque.
func (self *Outbox) Issue(amount *big.Int) (swap.Promise, error) {
return self.chequeBook.Issue(self.beneficiary, amount)
func (o *Outbox) Issue(amount *big.Int) (swap.Promise, error) {
return o.chequeBook.Issue(o.beneficiary, amount)
}
// AutoDeposit enables auto-deposits on the underlying chequebook.
func (self *Outbox) AutoDeposit(interval time.Duration, threshold, buffer *big.Int) {
self.chequeBook.AutoDeposit(interval, threshold, buffer)
func (o *Outbox) AutoDeposit(interval time.Duration, threshold, buffer *big.Int) {
o.chequeBook.AutoDeposit(interval, threshold, buffer)
}
// Stop helps satisfy the swap.OutPayment interface.
func (self *Outbox) Stop() {}
func (o *Outbox) Stop() {}
// String implements fmt.Stringer.
func (self *Outbox) String() string {
return fmt.Sprintf("chequebook: %v, beneficiary: %s, balance: %v", self.chequeBook.Address().Hex(), self.beneficiary.Hex(), self.chequeBook.Balance())
func (o *Outbox) String() string {
return fmt.Sprintf("chequebook: %v, beneficiary: %s, balance: %v", o.chequeBook.Address().Hex(), o.beneficiary.Hex(), o.chequeBook.Balance())
}
// Inbox can deposit, verify and cash cheques from a single contract to a single
@ -445,7 +444,7 @@ type Inbox struct {
// NewInbox creates an Inbox. An Inboxes is not persisted, the cumulative sum is updated
// from blockchain when first cheque is received.
func NewInbox(prvKey *ecdsa.PrivateKey, contractAddr, beneficiary common.Address, signer *ecdsa.PublicKey, abigen bind.ContractBackend) (self *Inbox, err error) {
func NewInbox(prvKey *ecdsa.PrivateKey, contractAddr, beneficiary common.Address, signer *ecdsa.PublicKey, abigen bind.ContractBackend) (*Inbox, error) {
if signer == nil {
return nil, fmt.Errorf("signer is null")
}
@ -461,7 +460,7 @@ func NewInbox(prvKey *ecdsa.PrivateKey, contractAddr, beneficiary common.Address
}
sender := transactOpts.From
self = &Inbox{
inbox := &Inbox{
contract: contractAddr,
beneficiary: beneficiary,
sender: sender,
@ -470,59 +469,61 @@ func NewInbox(prvKey *ecdsa.PrivateKey, contractAddr, beneficiary common.Address
cashed: new(big.Int).Set(common.Big0),
log: log.New("contract", contractAddr),
}
self.log.Trace("New chequebook inbox initialized", "beneficiary", self.beneficiary, "signer", hexutil.Bytes(crypto.FromECDSAPub(signer)))
return
inbox.log.Trace("New chequebook inbox initialized", "beneficiary", inbox.beneficiary, "signer", hexutil.Bytes(crypto.FromECDSAPub(signer)))
return inbox, nil
}
func (self *Inbox) String() string {
return fmt.Sprintf("chequebook: %v, beneficiary: %s, balance: %v", self.contract.Hex(), self.beneficiary.Hex(), self.cheque.Amount)
func (i *Inbox) String() string {
return fmt.Sprintf("chequebook: %v, beneficiary: %s, balance: %v", i.contract.Hex(), i.beneficiary.Hex(), i.cheque.Amount)
}
// Stop quits the autocash goroutine.
func (self *Inbox) Stop() {
defer self.lock.Unlock()
self.lock.Lock()
if self.quit != nil {
close(self.quit)
self.quit = nil
func (i *Inbox) Stop() {
defer i.lock.Unlock()
i.lock.Lock()
if i.quit != nil {
close(i.quit)
i.quit = nil
}
}
// Cash attempts to cash the current cheque.
func (self *Inbox) Cash() (txhash string, err error) {
if self.cheque != nil {
txhash, err = self.cheque.Cash(self.session)
self.log.Trace("Cashing in chequebook cheque", "amount", self.cheque.Amount, "beneficiary", self.beneficiary)
self.cashed = self.cheque.Amount
func (i *Inbox) Cash() (string, error) {
if i.cheque == nil {
return "", nil
}
return
txhash, err := i.cheque.Cash(i.session)
i.log.Trace("Cashing in chequebook cheque", "amount", i.cheque.Amount, "beneficiary", i.beneficiary)
i.cashed = i.cheque.Amount
return txhash, err
}
// AutoCash (re)sets maximum time and amount which triggers cashing of the last uncashed
// cheque if maxUncashed is set to 0, then autocash on receipt.
func (self *Inbox) AutoCash(cashInterval time.Duration, maxUncashed *big.Int) {
defer self.lock.Unlock()
self.lock.Lock()
self.maxUncashed = maxUncashed
self.autoCash(cashInterval)
func (i *Inbox) AutoCash(cashInterval time.Duration, maxUncashed *big.Int) {
defer i.lock.Unlock()
i.lock.Lock()
i.maxUncashed = maxUncashed
i.autoCash(cashInterval)
}
// autoCash starts a loop that periodically clears the last cheque
// if the peer is trusted. Clearing period could be 24h or a week.
// The caller must hold self.lock.
func (self *Inbox) autoCash(cashInterval time.Duration) {
if self.quit != nil {
close(self.quit)
self.quit = nil
// The caller must hold lock.
func (i *Inbox) autoCash(cashInterval time.Duration) {
if i.quit != nil {
close(i.quit)
i.quit = nil
}
// if maxUncashed is set to 0, then autocash on receipt
if cashInterval == time.Duration(0) || self.maxUncashed != nil && self.maxUncashed.Sign() == 0 {
if cashInterval == time.Duration(0) || i.maxUncashed != nil && i.maxUncashed.Sign() == 0 {
return
}
ticker := time.NewTicker(cashInterval)
self.quit = make(chan bool)
quit := self.quit
i.quit = make(chan bool)
quit := i.quit
go func() {
for {
@ -530,14 +531,14 @@ func (self *Inbox) autoCash(cashInterval time.Duration) {
case <-quit:
return
case <-ticker.C:
self.lock.Lock()
if self.cheque != nil && self.cheque.Amount.Cmp(self.cashed) != 0 {
txhash, err := self.Cash()
i.lock.Lock()
if i.cheque != nil && i.cheque.Amount.Cmp(i.cashed) != 0 {
txhash, err := i.Cash()
if err == nil {
self.txhash = txhash
i.txhash = txhash
}
}
self.lock.Unlock()
i.lock.Unlock()
}
}
}()
@ -545,56 +546,56 @@ func (self *Inbox) autoCash(cashInterval time.Duration) {
// Receive is called to deposit the latest cheque to the incoming Inbox.
// The given promise must be a *Cheque.
func (self *Inbox) Receive(promise swap.Promise) (*big.Int, error) {
func (i *Inbox) Receive(promise swap.Promise) (*big.Int, error) {
ch := promise.(*Cheque)
defer self.lock.Unlock()
self.lock.Lock()
defer i.lock.Unlock()
i.lock.Lock()
var sum *big.Int
if self.cheque == nil {
if i.cheque == nil {
// the sum is checked against the blockchain once a cheque is received
tally, err := self.session.Sent(self.beneficiary)
tally, err := i.session.Sent(i.beneficiary)
if err != nil {
return nil, fmt.Errorf("inbox: error calling backend to set amount: %v", err)
}
sum = tally
} else {
sum = self.cheque.Amount
sum = i.cheque.Amount
}
amount, err := ch.Verify(self.signer, self.contract, self.beneficiary, sum)
amount, err := ch.Verify(i.signer, i.contract, i.beneficiary, sum)
var uncashed *big.Int
if err == nil {
self.cheque = ch
i.cheque = ch
if self.maxUncashed != nil {
uncashed = new(big.Int).Sub(ch.Amount, self.cashed)
if self.maxUncashed.Cmp(uncashed) < 0 {
self.Cash()
if i.maxUncashed != nil {
uncashed = new(big.Int).Sub(ch.Amount, i.cashed)
if i.maxUncashed.Cmp(uncashed) < 0 {
i.Cash()
}
}
self.log.Trace("Received cheque in chequebook inbox", "amount", amount, "uncashed", uncashed)
i.log.Trace("Received cheque in chequebook inbox", "amount", amount, "uncashed", uncashed)
}
return amount, err
}
// Verify verifies cheque for signer, contract, beneficiary, amount, valid signature.
func (self *Cheque) Verify(signerKey *ecdsa.PublicKey, contract, beneficiary common.Address, sum *big.Int) (*big.Int, error) {
log.Trace("Verifying chequebook cheque", "cheque", self, "sum", sum)
func (ch *Cheque) Verify(signerKey *ecdsa.PublicKey, contract, beneficiary common.Address, sum *big.Int) (*big.Int, error) {
log.Trace("Verifying chequebook cheque", "cheque", ch, "sum", sum)
if sum == nil {
return nil, fmt.Errorf("invalid amount")
}
if self.Beneficiary != beneficiary {
return nil, fmt.Errorf("beneficiary mismatch: %v != %v", self.Beneficiary.Hex(), beneficiary.Hex())
if ch.Beneficiary != beneficiary {
return nil, fmt.Errorf("beneficiary mismatch: %v != %v", ch.Beneficiary.Hex(), beneficiary.Hex())
}
if self.Contract != contract {
return nil, fmt.Errorf("contract mismatch: %v != %v", self.Contract.Hex(), contract.Hex())
if ch.Contract != contract {
return nil, fmt.Errorf("contract mismatch: %v != %v", ch.Contract.Hex(), contract.Hex())
}
amount := new(big.Int).Set(self.Amount)
amount := new(big.Int).Set(ch.Amount)
if sum != nil {
amount.Sub(amount, sum)
if amount.Sign() <= 0 {
@ -602,7 +603,7 @@ func (self *Cheque) Verify(signerKey *ecdsa.PublicKey, contract, beneficiary com
}
}
pubKey, err := crypto.SigToPub(sigHash(self.Contract, beneficiary, self.Amount), self.Sig)
pubKey, err := crypto.SigToPub(sigHash(ch.Contract, beneficiary, ch.Amount), ch.Sig)
if err != nil {
return nil, fmt.Errorf("invalid signature: %v", err)
}
@ -621,9 +622,9 @@ func sig2vrs(sig []byte) (v byte, r, s [32]byte) {
}
// Cash cashes the cheque by sending an Ethereum transaction.
func (self *Cheque) Cash(session *contract.ChequebookSession) (string, error) {
v, r, s := sig2vrs(self.Sig)
tx, err := session.Cash(self.Beneficiary, self.Amount, v, r, s)
func (ch *Cheque) Cash(session *contract.ChequebookSession) (string, error) {
v, r, s := sig2vrs(ch.Sig)
tx, err := session.Cash(ch.Beneficiary, ch.Amount, v, r, s)
if err != nil {
return "", err
}
@ -632,7 +633,7 @@ func (self *Cheque) Cash(session *contract.ChequebookSession) (string, error) {
// ValidateCode checks that the on-chain code at address matches the expected chequebook
// contract code. This is used to detect suicided chequebooks.
func ValidateCode(ctx context.Context, b Backend, address common.Address) (ok bool, err error) {
func ValidateCode(ctx context.Context, b Backend, address common.Address) (bool, error) {
code, err := b.CodeAt(ctx, address, nil)
if err != nil {
return false, err

12
contracts/chequebook/contract/chequebook.go
View file

@ -205,22 +205,22 @@ func (_Chequebook *ChequebookCallerSession) Sent(arg0 common.Address) (*big.Int,
// Cash is a paid mutator transaction binding the contract method 0xfbf788d6.
//
// Solidity: function cash(beneficiary address, amount uint256, sig_v uint8, sig_r bytes32, sig_s bytes32) returns()
func (_Chequebook *ChequebookTransactor) Cash(opts *bind.TransactOpts, beneficiary common.Address, amount *big.Int, sig_v uint8, sig_r [32]byte, sig_s [32]byte) (*types.Transaction, error) {
return _Chequebook.contract.Transact(opts, "cash", beneficiary, amount, sig_v, sig_r, sig_s)
func (_Chequebook *ChequebookTransactor) Cash(opts *bind.TransactOpts, beneficiary common.Address, amount *big.Int, sigV uint8, sigR [32]byte, sigS [32]byte) (*types.Transaction, error) {
return _Chequebook.contract.Transact(opts, "cash", beneficiary, amount, sigV, sigR, sigS)
}
// Cash is a paid mutator transaction binding the contract method 0xfbf788d6.
//
// Solidity: function cash(beneficiary address, amount uint256, sig_v uint8, sig_r bytes32, sig_s bytes32) returns()
func (_Chequebook *ChequebookSession) Cash(beneficiary common.Address, amount *big.Int, sig_v uint8, sig_r [32]byte, sig_s [32]byte) (*types.Transaction, error) {
return _Chequebook.Contract.Cash(&_Chequebook.TransactOpts, beneficiary, amount, sig_v, sig_r, sig_s)
func (_Chequebook *ChequebookSession) Cash(beneficiary common.Address, amount *big.Int, sigV uint8, sigR [32]byte, sigS [32]byte) (*types.Transaction, error) {
return _Chequebook.Contract.Cash(&_Chequebook.TransactOpts, beneficiary, amount, sigV, sigR, sigS)
}
// Cash is a paid mutator transaction binding the contract method 0xfbf788d6.
//
// Solidity: function cash(beneficiary address, amount uint256, sig_v uint8, sig_r bytes32, sig_s bytes32) returns()
func (_Chequebook *ChequebookTransactorSession) Cash(beneficiary common.Address, amount *big.Int, sig_v uint8, sig_r [32]byte, sig_s [32]byte) (*types.Transaction, error) {
return _Chequebook.Contract.Cash(&_Chequebook.TransactOpts, beneficiary, amount, sig_v, sig_r, sig_s)
func (_Chequebook *ChequebookTransactorSession) Cash(beneficiary common.Address, amount *big.Int, sigV uint8, sigR [32]byte, sigS [32]byte) (*types.Transaction, error) {
return _Chequebook.Contract.Cash(&_Chequebook.TransactOpts, beneficiary, amount, sigV, sigR, sigS)
}
// Kill is a paid mutator transaction binding the contract method 0x41c0e1b5.

18
contracts/ens/contract/ens.go
View file

@ -227,10 +227,10 @@ func (_ENS *ENSCallerSession) Resolver(node [32]byte) (common.Address, error) {
return _ENS.Contract.Resolver(&_ENS.CallOpts, node)
}
// Ttl is a free data retrieval call binding the contract method 0x16a25cbd.
// TTL is a free data retrieval call binding the contract method 0x16a25cbd.
//
// Solidity: function ttl(node bytes32) constant returns(uint64)
func (_ENS *ENSCaller) Ttl(opts *bind.CallOpts, node [32]byte) (uint64, error) {
func (_ENS *ENSCaller) TTL(opts *bind.CallOpts, node [32]byte) (uint64, error) {
var (
ret0 = new(uint64)
)
@ -239,18 +239,18 @@ func (_ENS *ENSCaller) Ttl(opts *bind.CallOpts, node [32]byte) (uint64, error) {
return *ret0, err
}
// Ttl is a free data retrieval call binding the contract method 0x16a25cbd.
// TTL is a free data retrieval call binding the contract method 0x16a25cbd.
//
// Solidity: function ttl(node bytes32) constant returns(uint64)
func (_ENS *ENSSession) Ttl(node [32]byte) (uint64, error) {
return _ENS.Contract.Ttl(&_ENS.CallOpts, node)
func (_ENS *ENSSession) TTL(node [32]byte) (uint64, error) {
return _ENS.Contract.TTL(&_ENS.CallOpts, node)
}
// Ttl is a free data retrieval call binding the contract method 0x16a25cbd.
// TTL is a free data retrieval call binding the contract method 0x16a25cbd.
//
// Solidity: function ttl(node bytes32) constant returns(uint64)
func (_ENS *ENSCallerSession) Ttl(node [32]byte) (uint64, error) {
return _ENS.Contract.Ttl(&_ENS.CallOpts, node)
func (_ENS *ENSCallerSession) TTL(node [32]byte) (uint64, error) {
return _ENS.Contract.TTL(&_ENS.CallOpts, node)
}
// SetOwner is a paid mutator transaction binding the contract method 0x5b0fc9c3.
@ -682,7 +682,7 @@ func (it *ENSNewTTLIterator) Close() error {
// ENSNewTTL represents a NewTTL event raised by the ENS contract.
type ENSNewTTL struct {
Node [32]byte
Ttl uint64
TTL uint64
Raw types.Log // Blockchain specific contextual infos
}

67
contracts/ens/ens.go
View file

@ -35,7 +35,7 @@ var (
TestNetAddress = common.HexToAddress("0x112234455c3a32fd11230c42e7bccd4a84e02010")
)
// swarm domain name registry and resolver
// ENS is the swarm domain name registry and resolver
type ENS struct {
*contract.ENSSession
contractBackend bind.ContractBackend
@ -48,7 +48,6 @@ func NewENS(transactOpts *bind.TransactOpts, contractAddr common.Address, contra
if err != nil {
return nil, err
}
return &ENS{
&contract.ENSSession{
Contract: ens,
@ -60,27 +59,24 @@ func NewENS(transactOpts *bind.TransactOpts, contractAddr common.Address, contra
// DeployENS deploys an instance of the ENS nameservice, with a 'first-in, first-served' root registrar.
func DeployENS(transactOpts *bind.TransactOpts, contractBackend bind.ContractBackend) (common.Address, *ENS, error) {
// Deploy the ENS registry.
// Deploy the ENS registry
ensAddr, _, _, err := contract.DeployENS(transactOpts, contractBackend)
if err != nil {
return ensAddr, nil, err
}
ens, err := NewENS(transactOpts, ensAddr, contractBackend)
if err != nil {
return ensAddr, nil, err
}
// Deploy the registrar.
// Deploy the registrar
regAddr, _, _, err := contract.DeployFIFSRegistrar(transactOpts, contractBackend, ensAddr, [32]byte{})
if err != nil {
return ensAddr, nil, err
}
// Set the registrar as owner of the ENS root.
// Set the registrar as owner of the ENS root
if _, err = ens.SetOwner([32]byte{}, regAddr); err != nil {
return ensAddr, nil, err
}
return ensAddr, ens, nil
}
@ -89,122 +85,111 @@ func ensParentNode(name string) (common.Hash, common.Hash) {
label := crypto.Keccak256Hash([]byte(parts[0]))
if len(parts) == 1 {
return [32]byte{}, label
} else {
}
parentNode, parentLabel := ensParentNode(parts[1])
return crypto.Keccak256Hash(parentNode[:], parentLabel[:]), label
}
}
func EnsNode(name string) common.Hash {
parentNode, parentLabel := ensParentNode(name)
return crypto.Keccak256Hash(parentNode[:], parentLabel[:])
}
func (self *ENS) getResolver(node [32]byte) (*contract.PublicResolverSession, error) {
resolverAddr, err := self.Resolver(node)
func (ens *ENS) getResolver(node [32]byte) (*contract.PublicResolverSession, error) {
resolverAddr, err := ens.Resolver(node)
if err != nil {
return nil, err
}
resolver, err := contract.NewPublicResolver(resolverAddr, self.contractBackend)
resolver, err := contract.NewPublicResolver(resolverAddr, ens.contractBackend)
if err != nil {
return nil, err
}
return &contract.PublicResolverSession{
Contract: resolver,
TransactOpts: self.TransactOpts,
TransactOpts: ens.TransactOpts,
}, nil
}
func (self *ENS) getRegistrar(node [32]byte) (*contract.FIFSRegistrarSession, error) {
registrarAddr, err := self.Owner(node)
func (ens *ENS) getRegistrar(node [32]byte) (*contract.FIFSRegistrarSession, error) {
registrarAddr, err := ens.Owner(node)
if err != nil {
return nil, err
}
registrar, err := contract.NewFIFSRegistrar(registrarAddr, self.contractBackend)
registrar, err := contract.NewFIFSRegistrar(registrarAddr, ens.contractBackend)
if err != nil {
return nil, err
}
return &contract.FIFSRegistrarSession{
Contract: registrar,
TransactOpts: self.TransactOpts,
TransactOpts: ens.TransactOpts,
}, nil
}
// Resolve is a non-transactional call that returns the content hash associated with a name.
func (self *ENS) Resolve(name string) (common.Hash, error) {
func (ens *ENS) Resolve(name string) (common.Hash, error) {
node := EnsNode(name)
resolver, err := self.getResolver(node)
resolver, err := ens.getResolver(node)
if err != nil {
return common.Hash{}, err
}
ret, err := resolver.Content(node)
if err != nil {
return common.Hash{}, err
}
return common.BytesToHash(ret[:]), nil
}
// Addr is a non-transactional call that returns the address associated with a name.
func (self *ENS) Addr(name string) (common.Address, error) {
func (ens *ENS) Addr(name string) (common.Address, error) {
node := EnsNode(name)
resolver, err := self.getResolver(node)
resolver, err := ens.getResolver(node)
if err != nil {
return common.Address{}, err
}
ret, err := resolver.Addr(node)
if err != nil {
return common.Address{}, err
}
return common.BytesToAddress(ret[:]), nil
}
// SetAddress sets the address associated with a name. Only works if the caller
// owns the name, and the associated resolver implements a `setAddress` function.
func (self *ENS) SetAddr(name string, addr common.Address) (*types.Transaction, error) {
func (ens *ENS) SetAddr(name string, addr common.Address) (*types.Transaction, error) {
node := EnsNode(name)
resolver, err := self.getResolver(node)
resolver, err := ens.getResolver(node)
if err != nil {
return nil, err
}
opts := self.TransactOpts
opts := ens.TransactOpts
opts.GasLimit = 200000
return resolver.Contract.SetAddr(&opts, node, addr)
}
// Register registers a new domain name for the caller, making them the owner of the new name.
// Only works if the registrar for the parent domain implements the FIFS registrar protocol.
func (self *ENS) Register(name string) (*types.Transaction, error) {
func (ens *ENS) Register(name string) (*types.Transaction, error) {
parentNode, label := ensParentNode(name)
registrar, err := self.getRegistrar(parentNode)
registrar, err := ens.getRegistrar(parentNode)
if err != nil {
return nil, err
}
return registrar.Contract.Register(&self.TransactOpts, label, self.TransactOpts.From)
return registrar.Contract.Register(&ens.TransactOpts, label, ens.TransactOpts.From)
}
// SetContentHash sets the content hash associated with a name. Only works if the caller
// owns the name, and the associated resolver implements a `setContent` function.
func (self *ENS) SetContentHash(name string, hash common.Hash) (*types.Transaction, error) {
func (ens *ENS) SetContentHash(name string, hash common.Hash) (*types.Transaction, error) {
node := EnsNode(name)
resolver, err := self.getResolver(node)
resolver, err := ens.getResolver(node)
if err != nil {
return nil, err
}
opts := self.TransactOpts
opts := ens.TransactOpts
opts.GasLimit = 200000
return resolver.Contract.SetContent(&opts, node, hash)
}

85
core/blockchain.go
View file

@ -65,7 +65,13 @@ const (
triesInMemory = 128
// BlockChainVersion ensures that an incompatible database forces a resync from scratch.
BlockChainVersion uint64 = 3
//
// During the process of upgrading the database version from 3 to 4,
// the following incompatible database changes were added.
// * the `BlockNumber`, `TxHash`, `TxIndex`, `BlockHash` and `Index` fields of log are deleted
// * the `Bloom` field of receipt is deleted
// * the `BlockIndex` and `TxIndex` fields of txlookup are deleted
BlockChainVersion uint64 = 4
)
// CacheConfig contains the configuration values for the trie caching/pruning
@ -456,7 +462,7 @@ func (bc *BlockChain) repair(head **types.Block) error {
if block == nil {
return fmt.Errorf("missing block %d [%x]", (*head).NumberU64()-1, (*head).ParentHash())
}
(*head) = block
*head = block
}
}
@ -979,11 +985,16 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
triedb.Cap(limit - ethdb.IdealBatchSize)
}
// Find the next state trie we need to commit
header := bc.GetHeaderByNumber(current - triesInMemory)
chosen := header.Number.Uint64()
chosen := current - triesInMemory
// If we exceeded out time allowance, flush an entire trie to disk
if bc.gcproc > bc.cacheConfig.TrieTimeLimit {
// If the header is missing (canonical chain behind), we're reorging a low
// diff sidechain. Suspend committing until this operation is completed.
header := bc.GetHeaderByNumber(chosen)
if header == nil {
log.Warn("Reorg in progress, trie commit postponed", "number", chosen)
} else {
// If we're exceeding limits but haven't reached a large enough memory gap,
// warn the user that the system is becoming unstable.
if chosen < lastWrite+triesInMemory && bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit {
@ -994,6 +1005,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
lastWrite = chosen
bc.gcproc = 0
}
}
// Garbage collect anything below our required write retention
for !bc.triegc.Empty() {
root, number := bc.triegc.Pop()
@ -1078,15 +1090,21 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) {
if len(chain) == 0 {
return 0, nil
}
// Remove already known canon-blocks
var (
block, prev *types.Block
)
// Do a sanity check that the provided chain is actually ordered and linked
for i := 1; i < len(chain); i++ {
if chain[i].NumberU64() != chain[i-1].NumberU64()+1 || chain[i].ParentHash() != chain[i-1].Hash() {
block = chain[i]
prev = chain[i-1]
if block.NumberU64() != prev.NumberU64()+1 || block.ParentHash() != prev.Hash() {
// Chain broke ancestry, log a message (programming error) and skip insertion
log.Error("Non contiguous block insert", "number", chain[i].Number(), "hash", chain[i].Hash(),
"parent", chain[i].ParentHash(), "prevnumber", chain[i-1].Number(), "prevhash", chain[i-1].Hash())
log.Error("Non contiguous block insert", "number", block.Number(), "hash", block.Hash(),
"parent", block.ParentHash(), "prevnumber", prev.Number(), "prevhash", prev.Hash())
return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, chain[i-1].NumberU64(),
chain[i-1].Hash().Bytes()[:4], i, chain[i].NumberU64(), chain[i].Hash().Bytes()[:4], chain[i].ParentHash().Bytes()[:4])
return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, prev.NumberU64(),
prev.Hash().Bytes()[:4], i, block.NumberU64(), block.Hash().Bytes()[:4], block.ParentHash().Bytes()[:4])
}
}
// Pre-checks passed, start the full block imports
@ -1140,10 +1158,26 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, []
it := newInsertIterator(chain, results, bc.Validator())
block, err := it.next()
// Left-trim all the known blocks
if err == ErrKnownBlock {
// First block (and state) is known
// 1. We did a roll-back, and should now do a re-import
// 2. The block is stored as a sidechain, and is lying about it's stateroot, and passes a stateroot
// from the canonical chain, which has not been verified.
// Skip all known blocks that are behind us
current := bc.CurrentBlock().NumberU64()
for block != nil && err == ErrKnownBlock && current >= block.NumberU64() {
stats.ignored++
block, err = it.next()
}
// Falls through to the block import
}
switch {
// First block is pruned, insert as sidechain and reorg only if TD grows enough
case err == consensus.ErrPrunedAncestor:
return bc.insertSidechain(it)
return bc.insertSidechain(block, it)
// First block is future, shove it (and all children) to the future queue (unknown ancestor)
case err == consensus.ErrFutureBlock || (err == consensus.ErrUnknownAncestor && bc.futureBlocks.Contains(it.first().ParentHash())):
@ -1159,20 +1193,6 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, []
// If there are any still remaining, mark as ignored
return it.index, events, coalescedLogs, err
// First block (and state) is known
// 1. We did a roll-back, and should now do a re-import
// 2. The block is stored as a sidechain, and is lying about it's stateroot, and passes a stateroot
// from the canonical chain, which has not been verified.
case err == ErrKnownBlock:
// Skip all known blocks that behind us
current := bc.CurrentBlock().NumberU64()
for block != nil && err == ErrKnownBlock && current >= block.NumberU64() {
stats.ignored++
block, err = it.next()
}
// Falls through to the block import
// Some other error occurred, abort
case err != nil:
stats.ignored += len(it.chain)
@ -1253,8 +1273,8 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, []
stats.processed++
stats.usedGas += usedGas
cache, _ := bc.stateCache.TrieDB().Size()
stats.report(chain, it.index, cache)
dirty, _ := bc.stateCache.TrieDB().Size()
stats.report(chain, it.index, dirty)
}
// Any blocks remaining here? The only ones we care about are the future ones
if block != nil && err == consensus.ErrFutureBlock {
@ -1285,7 +1305,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, []
//
// The method writes all (header-and-body-valid) blocks to disk, then tries to
// switch over to the new chain if the TD exceeded the current chain.
func (bc *BlockChain) insertSidechain(it *insertIterator) (int, []interface{}, []*types.Log, error) {
func (bc *BlockChain) insertSidechain(block *types.Block, it *insertIterator) (int, []interface{}, []*types.Log, error) {
var (
externTd *big.Int
current = bc.CurrentBlock()
@ -1294,11 +1314,16 @@ func (bc *BlockChain) insertSidechain(it *insertIterator) (int, []interface{}, [
// Since we don't import them here, we expect ErrUnknownAncestor for the remaining
// ones. Any other errors means that the block is invalid, and should not be written
// to disk.
block, err := it.current(), consensus.ErrPrunedAncestor
err := consensus.ErrPrunedAncestor
for ; block != nil && (err == consensus.ErrPrunedAncestor); block, err = it.next() {
// Check the canonical state root for that number
if number := block.NumberU64(); current.NumberU64() >= number {
if canonical := bc.GetBlockByNumber(number); canonical != nil && canonical.Root() == block.Root() {
canonical := bc.GetBlockByNumber(number)
if canonical != nil && canonical.Hash() == block.Hash() {
// Not a sidechain block, this is a re-import of a canon block which has it's state pruned
continue
}
if canonical != nil && canonical.Root() == block.Root() {
// This is most likely a shadow-state attack. When a fork is imported into the
// database, and it eventually reaches a block height which is not pruned, we
// just found that the state already exist! This means that the sidechain block
@ -1324,7 +1349,7 @@ func (bc *BlockChain) insertSidechain(it *insertIterator) (int, []interface{}, [
if err := bc.WriteBlockWithoutState(block, externTd); err != nil {
return it.index, nil, nil, err
}
log.Debug("Inserted sidechain block", "number", block.Number(), "hash", block.Hash(),
log.Debug("Injected sidechain block", "number", block.Number(), "hash", block.Hash(),
"diff", block.Difficulty(), "elapsed", common.PrettyDuration(time.Since(start)),
"txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()),
"root", block.Root())

12
core/blockchain_insert.go
View file

@ -39,7 +39,7 @@ const statsReportLimit = 8 * time.Second
// report prints statistics if some number of blocks have been processed
// or more than a few seconds have passed since the last message.
func (st *insertStats) report(chain []*types.Block, index int, cache common.StorageSize) {
func (st *insertStats) report(chain []*types.Block, index int, dirty common.StorageSize) {
// Fetch the timings for the batch
var (
now = mclock.Now()
@ -63,7 +63,7 @@ func (st *insertStats) report(chain []*types.Block, index int, cache common.Stor
if timestamp := time.Unix(end.Time().Int64(), 0); time.Since(timestamp) > time.Minute {
context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...)
}
context = append(context, []interface{}{"cache", cache}...)
context = append(context, []interface{}{"dirty", dirty}...)
if st.queued > 0 {
context = append(context, []interface{}{"queued", st.queued}...)
@ -111,14 +111,6 @@ func (it *insertIterator) next() (*types.Block, error) {
return it.chain[it.index], it.validator.ValidateBody(it.chain[it.index])
}
// current returns the current block that's being processed.
func (it *insertIterator) current() *types.Block {
if it.index < 0 || it.index+1 >= len(it.chain) {
return nil
}
return it.chain[it.index]
}
// previous returns the previous block was being processed, or nil
func (it *insertIterator) previous() *types.Block {
if it.index < 1 {

156
core/blockchain_test.go
View file

@ -17,7 +17,6 @@
package core
import (
"fmt"
"math/big"
"math/rand"
"sync"
@ -126,13 +125,6 @@ func testFork(t *testing.T, blockchain *BlockChain, i, n int, full bool, compara
comparator(tdPre, tdPost)
}
func printChain(bc *BlockChain) {
for i := bc.CurrentBlock().Number().Uint64(); i > 0; i-- {
b := bc.GetBlockByNumber(uint64(i))
fmt.Printf("\t%x %v\n", b.Hash(), b.Difficulty())
}
}
// testBlockChainImport tries to process a chain of blocks, writing them into
// the database if successful.
func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error {
@ -188,15 +180,6 @@ func testHeaderChainImport(chain []*types.Header, blockchain *BlockChain) error
return nil
}
func insertChain(done chan bool, blockchain *BlockChain, chain types.Blocks, t *testing.T) {
_, err := blockchain.InsertChain(chain)
if err != nil {
fmt.Println(err)
t.FailNow()
}
done <- true
}
func TestLastBlock(t *testing.T) {
_, blockchain, err := newCanonical(ethash.NewFaker(), 0, true)
if err != nil {
@ -1429,7 +1412,7 @@ func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks in
}
b.StopTimer()
if got := chain.CurrentBlock().Transactions().Len(); got != numTxs*numBlocks {
b.Fatalf("Transactions were not included, expected %d, got %d", (numTxs * numBlocks), got)
b.Fatalf("Transactions were not included, expected %d, got %d", numTxs*numBlocks, got)
}
}
@ -1483,3 +1466,140 @@ func BenchmarkBlockChain_1x1000Executions(b *testing.B) {
benchmarkLargeNumberOfValueToNonexisting(b, numTxs, numBlocks, recipientFn, dataFn)
}
// Tests that importing a very large side fork, which is larger than the canon chain,
// but where the difficulty per block is kept low: this means that it will not
// overtake the 'canon' chain until after it's passed canon by about 200 blocks.
//
// Details at:
// - https://github.com/ethereum/go-ethereum/issues/18977
// - https://github.com/ethereum/go-ethereum/pull/18988
func TestLowDiffLongChain(t *testing.T) {
// Generate a canonical chain to act as the main dataset
engine := ethash.NewFaker()
db := ethdb.NewMemDatabase()
genesis := new(Genesis).MustCommit(db)
// We must use a pretty long chain to ensure that the fork doesn't overtake us
// until after at least 128 blocks post tip
blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 6*triesInMemory, func(i int, b *BlockGen) {
b.SetCoinbase(common.Address{1})
b.OffsetTime(-9)
})
// Import the canonical chain
diskdb := ethdb.NewMemDatabase()
new(Genesis).MustCommit(diskdb)
chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil)
if err != nil {
t.Fatalf("failed to create tester chain: %v", err)
}
if n, err := chain.InsertChain(blocks); err != nil {
t.Fatalf("block %d: failed to insert into chain: %v", n, err)
}
// Generate fork chain, starting from an early block
parent := blocks[10]
fork, _ := GenerateChain(params.TestChainConfig, parent, engine, db, 8*triesInMemory, func(i int, b *BlockGen) {
b.SetCoinbase(common.Address{2})
})
// And now import the fork
if i, err := chain.InsertChain(fork); err != nil {
t.Fatalf("block %d: failed to insert into chain: %v", i, err)
}
head := chain.CurrentBlock()
if got := fork[len(fork)-1].Hash(); got != head.Hash() {
t.Fatalf("head wrong, expected %x got %x", head.Hash(), got)
}
// Sanity check that all the canonical numbers are present
header := chain.CurrentHeader()
for number := head.NumberU64(); number > 0; number-- {
if hash := chain.GetHeaderByNumber(number).Hash(); hash != header.Hash() {
t.Fatalf("header %d: canonical hash mismatch: have %x, want %x", number, hash, header.Hash())
}
header = chain.GetHeader(header.ParentHash, number-1)
}
}
// Tests that importing a sidechain (S), where
// - S is sidechain, containing blocks [Sn...Sm]
// - C is canon chain, containing blocks [G..Cn..Cm]
// - A common ancestor is placed at prune-point + blocksBetweenCommonAncestorAndPruneblock
// - The sidechain S is prepended with numCanonBlocksInSidechain blocks from the canon chain
func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommonAncestorAndPruneblock int) {
// Generate a canonical chain to act as the main dataset
engine := ethash.NewFaker()
db := ethdb.NewMemDatabase()
genesis := new(Genesis).MustCommit(db)
// Generate and import the canonical chain
blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*triesInMemory, nil)
diskdb := ethdb.NewMemDatabase()
new(Genesis).MustCommit(diskdb)
chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil)
if err != nil {
t.Fatalf("failed to create tester chain: %v", err)
}
if n, err := chain.InsertChain(blocks); err != nil {
t.Fatalf("block %d: failed to insert into chain: %v", n, err)
}
lastPrunedIndex := len(blocks) - triesInMemory - 1
lastPrunedBlock := blocks[lastPrunedIndex]
firstNonPrunedBlock := blocks[len(blocks)-triesInMemory]
// Verify pruning of lastPrunedBlock
if chain.HasBlockAndState(lastPrunedBlock.Hash(), lastPrunedBlock.NumberU64()) {
t.Errorf("Block %d not pruned", lastPrunedBlock.NumberU64())
}
// Verify firstNonPrunedBlock is not pruned
if !chain.HasBlockAndState(firstNonPrunedBlock.Hash(), firstNonPrunedBlock.NumberU64()) {
t.Errorf("Block %d pruned", firstNonPrunedBlock.NumberU64())
}
// Generate the sidechain
// First block should be a known block, block after should be a pruned block. So
// canon(pruned), side, side...
// Generate fork chain, make it longer than canon
parentIndex := lastPrunedIndex + blocksBetweenCommonAncestorAndPruneblock
parent := blocks[parentIndex]
fork, _ := GenerateChain(params.TestChainConfig, parent, engine, db, 2*triesInMemory, func(i int, b *BlockGen) {
b.SetCoinbase(common.Address{2})
})
// Prepend the parent(s)
var sidechain []*types.Block
for i := numCanonBlocksInSidechain; i > 0; i-- {
sidechain = append(sidechain, blocks[parentIndex+1-i])
}
sidechain = append(sidechain, fork...)
_, err = chain.InsertChain(sidechain)
if err != nil {
t.Errorf("Got error, %v", err)
}
head := chain.CurrentBlock()
if got := fork[len(fork)-1].Hash(); got != head.Hash() {
t.Fatalf("head wrong, expected %x got %x", head.Hash(), got)
}
}
// Tests that importing a sidechain (S), where
// - S is sidechain, containing blocks [Sn...Sm]
// - C is canon chain, containing blocks [G..Cn..Cm]
// - The common ancestor Cc is pruned
// - The first block in S: Sn, is == Cn
// That is: the sidechain for import contains some blocks already present in canon chain.
// So the blocks are
// [ Cn, Cn+1, Cc, Sn+3 ... Sm]
// ^ ^ ^ pruned
func TestPrunedImportSide(t *testing.T) {
//glogger := log.NewGlogHandler(log.StreamHandler(os.Stdout, log.TerminalFormat(false)))
//glogger.Verbosity(3)
//log.Root().SetHandler(log.Handler(glogger))
testSideImport(t, 3, 3)
testSideImport(t, 3, -3)
testSideImport(t, 10, 0)
testSideImport(t, 1, 10)
testSideImport(t, 1, -10)
}

2
core/chain_makers.go
View file

@ -149,7 +149,7 @@ func (b *BlockGen) PrevBlock(index int) *types.Block {
// associated difficulty. It's useful to test scenarios where forking is not
// tied to chain length directly.
func (b *BlockGen) OffsetTime(seconds int64) {
b.header.Time.Add(b.header.Time, new(big.Int).SetInt64(seconds))
b.header.Time.Add(b.header.Time, big.NewInt(seconds))
if b.header.Time.Cmp(b.parent.Header().Time) <= 0 {
panic("block time out of range")
}

13
core/genesis.go
View file

@ -157,7 +157,6 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, constant
if genesis != nil && genesis.Config == nil {
return params.AllEthashProtocolChanges, common.Hash{}, errGenesisNoConfig
}
// Just commit the new block if there is no stored genesis block.
stored := rawdb.ReadCanonicalHash(db, 0)
if (stored == common.Hash{}) {
@ -340,6 +339,18 @@ func DefaultRinkebyGenesisBlock() *Genesis {
}
}
// DefaultGoerliGenesisBlock returns the Görli network genesis block.
func DefaultGoerliGenesisBlock() *Genesis {
return &Genesis{
Config: params.GoerliChainConfig,
Timestamp: 1548854791,
ExtraData: hexutil.MustDecode("0x22466c6578692069732061207468696e6722202d204166726900000000000000e0a2bd4258d2768837baa26a28fe71dc079f84c70000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"),
GasLimit: 10485760,
Difficulty: big.NewInt(1),
Alloc: decodePrealloc(goerliAllocData),
}
}
// DeveloperGenesisBlock returns the 'geth --dev' genesis block. Note, this must
// be seeded with the
func DeveloperGenesisBlock(period uint64, faucet common.Address) *Genesis {

1
core/genesis_alloc.go
View file

File diff suppressed because one or more lines are too long

10
core/rawdb/accessors_chain.go
View file

@ -294,7 +294,17 @@ func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Rece
return nil
}
receipts := make(types.Receipts, len(storageReceipts))
logIndex := uint(0)
for i, receipt := range storageReceipts {
// Assemble deriving fields for log.
for _, log := range receipt.Logs {
log.TxHash = receipt.TxHash
log.BlockHash = hash
log.BlockNumber = number
log.TxIndex = uint(i)
log.Index = logIndex
logIndex += 1
}
receipts[i] = (*types.Receipt)(receipt)
}
return receipts

2
core/rawdb/accessors_chain_test.go
View file

@ -279,6 +279,7 @@ func TestBlockReceiptStorage(t *testing.T) {
ContractAddress: common.BytesToAddress([]byte{0x01, 0x11, 0x11}),
GasUsed: 111111,
}
receipt1.Bloom = types.CreateBloom(types.Receipts{receipt1})
receipt2 := &types.Receipt{
PostState: common.Hash{2}.Bytes(),
CumulativeGasUsed: 2,
@ -290,6 +291,7 @@ func TestBlockReceiptStorage(t *testing.T) {
ContractAddress: common.BytesToAddress([]byte{0x02, 0x22, 0x22}),
GasUsed: 222222,
}
receipt2.Bloom = types.CreateBloom(types.Receipts{receipt2})
receipts := []*types.Receipt{receipt1, receipt2}
// Check that no receipt entries are in a pristine database

65
core/rawdb/accessors_indexes.go
View file

@ -25,33 +25,28 @@ import (
// ReadTxLookupEntry retrieves the positional metadata associated with a transaction
// hash to allow retrieving the transaction or receipt by hash.
func ReadTxLookupEntry(db DatabaseReader, hash common.Hash) (common.Hash, uint64, uint64) {
func ReadTxLookupEntry(db DatabaseReader, hash common.Hash) common.Hash {
data, _ := db.Get(txLookupKey(hash))
if len(data) == 0 {
return common.Hash{}, 0, 0
return common.Hash{}
}
var entry TxLookupEntry
if len(data) == common.HashLength {
return common.BytesToHash(data)
}
// Probably it's legacy txlookup entry data, try to decode it.
var entry LegacyTxLookupEntry
if err := rlp.DecodeBytes(data, &entry); err != nil {
log.Error("Invalid transaction lookup entry RLP", "hash", hash, "err", err)
return common.Hash{}, 0, 0
log.Error("Invalid transaction lookup entry RLP", "hash", hash, "blob", data, "err", err)
return common.Hash{}
}
return entry.BlockHash, entry.BlockIndex, entry.Index
return entry.BlockHash
}
// WriteTxLookupEntries stores a positional metadata for every transaction from
// a block, enabling hash based transaction and receipt lookups.
func WriteTxLookupEntries(db DatabaseWriter, block *types.Block) {
for i, tx := range block.Transactions() {
entry := TxLookupEntry{
BlockHash: block.Hash(),
BlockIndex: block.NumberU64(),
Index: uint64(i),
}
data, err := rlp.EncodeToBytes(entry)
if err != nil {
log.Crit("Failed to encode transaction lookup entry", "err", err)
}
if err := db.Put(txLookupKey(tx.Hash()), data); err != nil {
for _, tx := range block.Transactions() {
if err := db.Put(txLookupKey(tx.Hash()), block.Hash().Bytes()); err != nil {
log.Crit("Failed to store transaction lookup entry", "err", err)
}
}
@ -65,31 +60,47 @@ func DeleteTxLookupEntry(db DatabaseDeleter, hash common.Hash) {
// ReadTransaction retrieves a specific transaction from the database, along with
// its added positional metadata.
func ReadTransaction(db DatabaseReader, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) {
blockHash, blockNumber, txIndex := ReadTxLookupEntry(db, hash)
blockHash := ReadTxLookupEntry(db, hash)
if blockHash == (common.Hash{}) {
return nil, common.Hash{}, 0, 0
}
body := ReadBody(db, blockHash, blockNumber)
if body == nil || len(body.Transactions) <= int(txIndex) {
log.Error("Transaction referenced missing", "number", blockNumber, "hash", blockHash, "index", txIndex)
blockNumber := ReadHeaderNumber(db, blockHash)
if blockNumber == nil {
return nil, common.Hash{}, 0, 0
}
return body.Transactions[txIndex], blockHash, blockNumber, txIndex
body := ReadBody(db, blockHash, *blockNumber)
if body == nil {
log.Error("Transaction referenced missing", "number", blockNumber, "hash", blockHash)
return nil, common.Hash{}, 0, 0
}
for txIndex, tx := range body.Transactions {
if tx.Hash() == hash {
return tx, blockHash, *blockNumber, uint64(txIndex)
}
}
log.Error("Transaction not found", "number", blockNumber, "hash", blockHash, "txhash", hash)
return nil, common.Hash{}, 0, 0
}
// ReadReceipt retrieves a specific transaction receipt from the database, along with
// its added positional metadata.
func ReadReceipt(db DatabaseReader, hash common.Hash) (*types.Receipt, common.Hash, uint64, uint64) {
blockHash, blockNumber, receiptIndex := ReadTxLookupEntry(db, hash)
blockHash := ReadTxLookupEntry(db, hash)
if blockHash == (common.Hash{}) {
return nil, common.Hash{}, 0, 0
}
receipts := ReadReceipts(db, blockHash, blockNumber)
if len(receipts) <= int(receiptIndex) {
log.Error("Receipt refereced missing", "number", blockNumber, "hash", blockHash, "index", receiptIndex)
blockNumber := ReadHeaderNumber(db, blockHash)
if blockNumber == nil {
return nil, common.Hash{}, 0, 0
}
return receipts[receiptIndex], blockHash, blockNumber, receiptIndex
receipts := ReadReceipts(db, blockHash, *blockNumber)
for receiptIndex, receipt := range receipts {
if receipt.TxHash == hash {
return receipt, blockHash, *blockNumber, uint64(receiptIndex)
}
}
log.Error("Receipt not found", "number", blockNumber, "hash", blockHash, "txhash", hash)
return nil, common.Hash{}, 0, 0
}
// ReadBloomBits retrieves the compressed bloom bit vector belonging to the given

23
core/rawdb/accessors_indexes_test.go
View file

@ -23,6 +23,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/rlp"
)
// Tests that positional lookup metadata can be stored and retrieved.
@ -65,4 +66,26 @@ func TestLookupStorage(t *testing.T) {
t.Fatalf("tx #%d [%x]: deleted transaction returned: %v", i, tx.Hash(), txn)
}
}
// Insert legacy txlookup and verify the data retrieval
for index, tx := range block.Transactions() {
entry := LegacyTxLookupEntry{
BlockHash: block.Hash(),
BlockIndex: block.NumberU64(),
Index: uint64(index),
}
data, _ := rlp.EncodeToBytes(entry)
db.Put(txLookupKey(tx.Hash()), data)
}
for i, tx := range txs {
if txn, hash, number, index := ReadTransaction(db, tx.Hash()); txn == nil {
t.Fatalf("tx #%d [%x]: transaction not found", i, tx.Hash())
} else {
if hash != block.Hash() || number != block.NumberU64() || index != uint64(i) {
t.Fatalf("tx #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, tx.Hash(), hash, number, index, block.Hash(), block.NumberU64(), i)
}
if tx.Hash() != txn.Hash() {
t.Fatalf("tx #%d [%x]: transaction mismatch: have %v, want %v", i, tx.Hash(), txn, tx)
}
}
}
}

6
core/rawdb/schema.go
View file

@ -63,9 +63,9 @@ var (
preimageHitCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil)
)
// TxLookupEntry is a positional metadata to help looking up the data content of
// a transaction or receipt given only its hash.
type TxLookupEntry struct {
// LegacyTxLookupEntry is the legacy TxLookupEntry definition with some unnecessary
// fields.
type LegacyTxLookupEntry struct {
BlockHash common.Hash
BlockIndex uint64
Index uint64

2
core/state/statedb.go
View file

@ -502,7 +502,7 @@ func (self *StateDB) Copy() *StateDB {
refund: self.refund,
logs: make(map[common.Hash][]*types.Log, len(self.logs)),
logSize: self.logSize,
preimages: make(map[common.Hash][]byte),
preimages: make(map[common.Hash][]byte, len(self.preimages)),
journal: newJournal(),
}
// Copy the dirty states, logs, and preimages

9
core/state/statedb_test.go
View file

@ -276,6 +276,15 @@ func newTestAction(addr common.Address, r *rand.Rand) testAction {
},
args: make([]int64, 1),
},
{
name: "AddPreimage",
fn: func(a testAction, s *StateDB) {
preimage := []byte{1}
hash := common.BytesToHash(preimage)
s.AddPreimage(hash, preimage)
},
args: make([]int64, 1),
},
}
action := actions[r.Intn(len(actions))]
var nameargs []string

12
core/tx_pool_test.go
View file

@ -1274,14 +1274,14 @@ func TestTransactionPoolRepricingKeepsLocals(t *testing.T) {
}
// Create transaction (both pending and queued) with a linearly growing gasprice
for i := uint64(0); i < 500; i++ {
// Add pending
p_tx := pricedTransaction(i, 100000, big.NewInt(int64(i)), keys[2])
if err := pool.AddLocal(p_tx); err != nil {
// Add pending transaction.
pendingTx := pricedTransaction(i, 100000, big.NewInt(int64(i)), keys[2])
if err := pool.AddLocal(pendingTx); err != nil {
t.Fatal(err)
}
// Add queued
q_tx := pricedTransaction(i+501, 100000, big.NewInt(int64(i)), keys[2])
if err := pool.AddLocal(q_tx); err != nil {
// Add queued transaction.
queuedTx := pricedTransaction(i+501, 100000, big.NewInt(int64(i)), keys[2])
if err := pool.AddLocal(queuedTx); err != nil {
t.Fatal(err)
}
}

2
core/types/gen_log_json.go
View file

@ -12,6 +12,7 @@ import (
var _ = (*logMarshaling)(nil)
// MarshalJSON marshals as JSON.
func (l Log) MarshalJSON() ([]byte, error) {
type Log struct {
Address common.Address `json:"address" gencodec:"required"`
@ -37,6 +38,7 @@ func (l Log) MarshalJSON() ([]byte, error) {
return json.Marshal(&enc)
}
// UnmarshalJSON unmarshals from JSON.
func (l *Log) UnmarshalJSON(input []byte) error {
type Log struct {
Address *common.Address `json:"address" gencodec:"required"`

29
core/types/log.go
View file

@ -68,7 +68,11 @@ type rlpLog struct {
Data []byte
}
type rlpStorageLog struct {
// rlpStorageLog is the storage encoding of a log.
type rlpStorageLog rlpLog
// LegacyRlpStorageLog is the previous storage encoding of a log including some redundant fields.
type LegacyRlpStorageLog struct {
Address common.Address
Topics []common.Hash
Data []byte
@ -104,15 +108,12 @@ func (l *LogForStorage) EncodeRLP(w io.Writer) error {
Address: l.Address,
Topics: l.Topics,
Data: l.Data,
BlockNumber: l.BlockNumber,
TxHash: l.TxHash,
TxIndex: l.TxIndex,
BlockHash: l.BlockHash,
Index: l.Index,
})
}
// DecodeRLP implements rlp.Decoder.
//
// Note some redundant fields(e.g. block number, tx hash etc) will be assembled later.
func (l *LogForStorage) DecodeRLP(s *rlp.Stream) error {
var dec rlpStorageLog
err := s.Decode(&dec)
@ -121,11 +122,17 @@ func (l *LogForStorage) DecodeRLP(s *rlp.Stream) error {
Address: dec.Address,
Topics: dec.Topics,
Data: dec.Data,
BlockNumber: dec.BlockNumber,
TxHash: dec.TxHash,
TxIndex: dec.TxIndex,
BlockHash: dec.BlockHash,
Index: dec.Index,
}
} else {
// Try to decode log with previous definition.
var dec LegacyRlpStorageLog
err = s.Decode(&dec)
if err == nil {
*l = LogForStorage{
Address: dec.Address,
Topics: dec.Topics,
Data: dec.Data,
}
}
}
return err

25
core/types/receipt.go
View file

@ -72,7 +72,18 @@ type receiptRLP struct {
Logs []*Log
}
// receiptStorageRLP is the storage encoding of a receipt.
type receiptStorageRLP struct {
PostStateOrStatus []byte
CumulativeGasUsed uint64
TxHash common.Hash
ContractAddress common.Address
Logs []*LogForStorage
GasUsed uint64
}
// LegacyReceiptStorageRLP is the previous storage encoding of a receipt including some unnecessary fields.
type LegacyReceiptStorageRLP struct {
PostStateOrStatus []byte
CumulativeGasUsed uint64
Bloom Bloom
@ -159,7 +170,6 @@ func (r *ReceiptForStorage) EncodeRLP(w io.Writer) error {
enc := &receiptStorageRLP{
PostStateOrStatus: (*Receipt)(r).statusEncoding(),
CumulativeGasUsed: r.CumulativeGasUsed,
Bloom: r.Bloom,
TxHash: r.TxHash,
ContractAddress: r.ContractAddress,
Logs: make([]*LogForStorage, len(r.Logs)),
@ -176,17 +186,28 @@ func (r *ReceiptForStorage) EncodeRLP(w io.Writer) error {
func (r *ReceiptForStorage) DecodeRLP(s *rlp.Stream) error {
var dec receiptStorageRLP
if err := s.Decode(&dec); err != nil {
var sdec LegacyReceiptStorageRLP
if err := s.Decode(&sdec); err != nil {
return err
}
dec.PostStateOrStatus = common.CopyBytes(sdec.PostStateOrStatus)
dec.CumulativeGasUsed = sdec.CumulativeGasUsed
dec.TxHash = sdec.TxHash
dec.ContractAddress = sdec.ContractAddress
dec.Logs = sdec.Logs
dec.GasUsed = sdec.GasUsed
}
if err := (*Receipt)(r).setStatus(dec.PostStateOrStatus); err != nil {
return err
}
// Assign the consensus fields
r.CumulativeGasUsed, r.Bloom = dec.CumulativeGasUsed, dec.Bloom
r.CumulativeGasUsed = dec.CumulativeGasUsed
r.Logs = make([]*Log, len(dec.Logs))
for i, log := range dec.Logs {
r.Logs[i] = (*Log)(log)
}
r.Bloom = CreateBloom(Receipts{(*Receipt)(r)})
// Assign the implementation fields
r.TxHash, r.ContractAddress, r.GasUsed = dec.TxHash, dec.ContractAddress, dec.GasUsed
return nil

15
core/vm/evm.go
View file

@ -267,9 +267,8 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte,
snapshot = evm.StateDB.Snapshot()
to = AccountRef(caller.Address())
)
// initialise a new contract and set the code that is to be used by the
// EVM. The contract is a scoped environment for this execution context
// only.
// Initialise a new contract and set the code that is to be used by the EVM.
// The contract is a scoped environment for this execution context only.
contract := NewContract(caller, to, value, gas)
contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr))
@ -333,9 +332,8 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte
to = AccountRef(addr)
snapshot = evm.StateDB.Snapshot()
)
// Initialise a new contract and set the code that is to be used by the
// EVM. The contract is a scoped environment for this execution context
// only.
// Initialise a new contract and set the code that is to be used by the EVM.
// The contract is a scoped environment for this execution context only.
contract := NewContract(caller, to, new(big.Int), gas)
contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr))
@ -396,9 +394,8 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64,
}
evm.Transfer(evm.StateDB, caller.Address(), address, value)
// initialise a new contract and set the code that is to be used by the
// EVM. The contract is a scoped environment for this execution context
// only.
// Initialise a new contract and set the code that is to be used by the EVM.
// The contract is a scoped environment for this execution context only.
contract := NewContract(caller, AccountRef(address), value, gas)
contract.SetCodeOptionalHash(&address, codeAndHash)

4
core/vm/gas.go
View file

@ -30,10 +30,6 @@ const (
GasMidStep uint64 = 8
GasSlowStep uint64 = 10
GasExtStep uint64 = 20
GasReturn uint64 = 0
GasStop uint64 = 0
GasContractByte uint64 = 200
)
// calcGas returns the actual gas cost of the call.

7
core/vm/interpreter.go
View file

@ -182,6 +182,7 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) (
pcCopy uint64 // needed for the deferred Tracer
gasCopy uint64 // for Tracer to log gas remaining before execution
logged bool // deferred Tracer should ignore already logged steps
res []byte // result of the opcode execution function
)
contract.Input = input
@ -216,11 +217,11 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) (
if !operation.valid {
return nil, fmt.Errorf("invalid opcode 0x%x", int(op))
}
if err := operation.validateStack(stack); err != nil {
if err = operation.validateStack(stack); err != nil {
return nil, err
}
// If the operation is valid, enforce and write restrictions
if err := in.enforceRestrictions(op, operation, stack); err != nil {
if err = in.enforceRestrictions(op, operation, stack); err != nil {
return nil, err
}
@ -254,7 +255,7 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) (
}
// execute the operation
res, err := operation.execute(&pc, in, contract, mem, stack)
res, err = operation.execute(&pc, in, contract, mem, stack)
// verifyPool is a build flag. Pool verification makes sure the integrity
// of the integer pool by comparing values to a default value.
if verifyPool {

13
crypto/ecies/ecies_test.go
View file

@ -93,19 +93,6 @@ func cmpPublic(pub1, pub2 PublicKey) bool {
return bytes.Equal(pub1Out, pub2Out)
}
// cmpPrivate returns true if the two private keys are the same.
func cmpPrivate(prv1, prv2 *PrivateKey) bool {
if prv1 == nil || prv1.D == nil {
return false
} else if prv2 == nil || prv2.D == nil {
return false
} else if prv1.D.Cmp(prv2.D) != 0 {
return false
} else {
return cmpPublic(prv1.PublicKey, prv2.PublicKey)
}
}
// Validate the ECDH component.
func TestSharedKey(t *testing.T) {
prv1, err := GenerateKey(rand.Reader, DefaultCurve, nil)

2
crypto/signature_cgo.go
View file

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// +build !nacl,!js,!nocgo
// +build !nacl,!js,cgo
package crypto

2
crypto/signature_nocgo.go
View file

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// +build nacl js nocgo
// +build nacl js !cgo
package crypto

BIN
docs/audits/2017-04-25_Geth-audit_Truesec.pdf Normal file
View file

Binary file not shown.

BIN
docs/audits/2018-09-14_Clef-audit_NCC.pdf Normal file
View file

Binary file not shown.

31
eth/api_tracer.go
View file

@ -214,7 +214,8 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
log.Warn("Tracing failed", "hash", tx.Hash(), "block", task.block.NumberU64(), "err", err)
break
}
task.statedb.Finalise(true)
// Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect
task.statedb.Finalise(api.eth.blockchain.Config().IsEIP158(task.block.Number()))
task.results[i] = &txTraceResult{Result: res}
}
// Stream the result back to the user or abort on teardown
@ -506,7 +507,8 @@ func (api *PrivateDebugAPI) traceBlock(ctx context.Context, block *types.Block,
break
}
// Finalize the state so any modifications are written to the trie
statedb.Finalise(true)
// Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect
statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number()))
}
close(jobs)
pend.Wait()
@ -524,13 +526,7 @@ func (api *PrivateDebugAPI) traceBlock(ctx context.Context, block *types.Block,
func (api *PrivateDebugAPI) standardTraceBlockToFile(ctx context.Context, block *types.Block, config *StdTraceConfig) ([]string, error) {
// If we're tracing a single transaction, make sure it's present
if config != nil && config.TxHash != (common.Hash{}) {
var exists bool
for _, tx := range block.Transactions() {
if exists = (tx.Hash() == config.TxHash); exists {
break
}
}
if !exists {
if !containsTx(block, config.TxHash) {
return nil, fmt.Errorf("transaction %#x not found in block", config.TxHash)
}
}
@ -612,7 +608,8 @@ func (api *PrivateDebugAPI) standardTraceBlockToFile(ctx context.Context, block
return dumps, err
}
// Finalize the state so any modifications are written to the trie
statedb.Finalise(true)
// Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect
statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number()))
// If we've traced the transaction we were looking for, abort
if tx.Hash() == txHash {
@ -622,6 +619,17 @@ func (api *PrivateDebugAPI) standardTraceBlockToFile(ctx context.Context, block
return dumps, nil
}
// containsTx reports whether the transaction with a certain hash
// is contained within the specified block.
func containsTx(block *types.Block, hash common.Hash) bool {
for _, tx := range block.Transactions() {
if tx.Hash() == hash {
return true
}
}
return false
}
// computeStateDB retrieves the state database associated with a certain block.
// If no state is locally available for the given block, a number of blocks are
// attempted to be reexecuted to generate the desired state.
@ -803,7 +811,8 @@ func (api *PrivateDebugAPI) computeTxEnv(blockHash common.Hash, txIndex int, ree
return nil, vm.Context{}, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err)
}
// Ensure any modifications are committed to the state
statedb.Finalise(true)
// Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect
statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number()))
}
return nil, vm.Context{}, nil, fmt.Errorf("transaction index %d out of range for block %#x", txIndex, blockHash)
}

22
eth/backend.go
View file

@ -113,6 +113,12 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
log.Warn("Sanitizing invalid miner gas price", "provided", config.MinerGasPrice, "updated", DefaultConfig.MinerGasPrice)
config.MinerGasPrice = new(big.Int).Set(DefaultConfig.MinerGasPrice)
}
if config.NoPruning && config.TrieDirtyCache > 0 {
config.TrieCleanCache += config.TrieDirtyCache
config.TrieDirtyCache = 0
}
log.Info("Allocated trie memory caches", "clean", common.StorageSize(config.TrieCleanCache)*1024*1024, "dirty", common.StorageSize(config.TrieDirtyCache)*1024*1024)
// Assemble the Ethereum object
chainDb, err := CreateDB(ctx, config, "chaindata")
if err != nil {
@ -139,17 +145,21 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
bloomIndexer: NewBloomIndexer(chainDb, params.BloomBitsBlocks, params.BloomConfirms),
}
log.Info("Initialising Ethereum protocol", "versions", ProtocolVersions, "network", config.NetworkId)
bcVersion := rawdb.ReadDatabaseVersion(chainDb)
var dbVer = "<nil>"
if bcVersion != nil {
dbVer = fmt.Sprintf("%d", *bcVersion)
}
log.Info("Initialising Ethereum protocol", "versions", ProtocolVersions, "network", config.NetworkId, "dbversion", dbVer)
if !config.SkipBcVersionCheck {
bcVersion := rawdb.ReadDatabaseVersion(chainDb)
if bcVersion != nil && *bcVersion > core.BlockChainVersion {
return nil, fmt.Errorf("database version is v%d, Geth %s only supports v%d", *bcVersion, params.VersionWithMeta, core.BlockChainVersion)
} else if bcVersion != nil && *bcVersion < core.BlockChainVersion {
log.Warn("Upgrade blockchain database version", "from", *bcVersion, "to", core.BlockChainVersion)
}
} else if bcVersion == nil || *bcVersion < core.BlockChainVersion {
log.Warn("Upgrade blockchain database version", "from", dbVer, "to", core.BlockChainVersion)
rawdb.WriteDatabaseVersion(chainDb, core.BlockChainVersion)
}
}
var (
vmConfig = vm.Config{
EnablePreimageRecording: config.EnablePreimageRecording,
@ -434,7 +444,7 @@ func (s *Ethereum) StartMining(threads int) error {
log.Error("Etherbase account unavailable locally", "err", err)
return fmt.Errorf("signer missing: %v", err)
}
clique.Authorize(eb, wallet.SignHash)
clique.Authorize(eb, wallet.SignData)
}
// If mining is started, we can disable the transaction rejection mechanism
// introduced to speed sync times.

11
eth/config.go
View file

@ -68,8 +68,15 @@ func init() {
home = user.HomeDir
}
}
if runtime.GOOS == "windows" {
DefaultConfig.Ethash.DatasetDir = filepath.Join(home, "AppData", "Ethash")
if runtime.GOOS == "darwin" {
DefaultConfig.Ethash.DatasetDir = filepath.Join(home, "Library", "Ethash")
} else if runtime.GOOS == "windows" {
localappdata := os.Getenv("LOCALAPPDATA")
if localappdata != "" {
DefaultConfig.Ethash.DatasetDir = filepath.Join(localappdata, "Ethash")
} else {
DefaultConfig.Ethash.DatasetDir = filepath.Join(home, "AppData", "Local", "Ethash")
}
} else {
DefaultConfig.Ethash.DatasetDir = filepath.Join(home, ".ethash")
}

7
eth/downloader/downloader.go
View file

@ -415,7 +415,8 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I
if err != nil {
d.mux.Post(FailedEvent{err})
} else {
d.mux.Post(DoneEvent{})
latest := d.lightchain.CurrentHeader()
d.mux.Post(DoneEvent{latest})
}
}()
if p.version < 62 {
@ -718,7 +719,7 @@ func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header)
}
// Make sure the peer's reply conforms to the request
for i, header := range headers {
expectNumber := from + int64(i)*int64((skip+1))
expectNumber := from + int64(i)*int64(skip+1)
if number := header.Number.Int64(); number != expectNumber {
p.log.Warn("Head headers broke chain ordering", "index", i, "requested", expectNumber, "received", number)
return 0, errInvalidChain
@ -1369,7 +1370,6 @@ func (d *Downloader) processHeaders(origin uint64, pivot uint64, td *big.Int) er
}
// Otherwise split the chunk of headers into batches and process them
gotHeaders = true
for len(headers) > 0 {
// Terminate if something failed in between processing chunks
select {
@ -1432,7 +1432,6 @@ func (d *Downloader) processHeaders(origin uint64, pivot uint64, td *big.Int) er
headers = headers[limit:]
origin += uint64(limit)
}
// Update the highest block number we know if a higher one is found.
d.syncStatsLock.Lock()
if d.syncStatsChainHeight < origin {

6
eth/downloader/events.go
View file

@ -16,6 +16,10 @@
package downloader
type DoneEvent struct{}
import "github.com/ethereum/go-ethereum/core/types"
type DoneEvent struct {
Latest *types.Header
}
type StartEvent struct{}
type FailedEvent struct{ Err error }

Some files were not shown because too many files have changed in this diff Show more