forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-20 10:52:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge branch 'master' into abigen-vyper

Browse source
This commit is contained in:
Kushagra Sharma 2019-02-18 23:24:20 +00:00 committed by GitHub
commit 4fc7a53e85
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
351 changed files with 86177 additions and 15545 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.travis.yml
View file

@ -68,8 +68,11 @@ matrix:
- debhelper
- dput
- fakeroot
- python-bzrlib
- python-paramiko
script:
- go run build/ci.go debsrc -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>" -upload ppa:ethereum/ethereum
- echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
- go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
# This builder does the Linux Azure uploads
- if: type = push

19
accounts/abi/bind/backends/simulated.go
View file

@ -164,6 +164,25 @@ func (b *SimulatedBackend) TransactionReceipt(ctx context.Context, txHash common
return receipt, nil
}
// TransactionByHash checks the pool of pending transactions in addition to the
// blockchain. The isPending return value indicates whether the transaction has been
// mined yet. Note that the transaction may not be part of the canonical chain even if
// it's not pending.
func (b *SimulatedBackend) TransactionByHash(ctx context.Context, txHash common.Hash) (tx *types.Transaction, isPending bool, err error) {
tx = b.pendingBlock.Transaction(txHash)
if tx != nil {
return tx, true, nil
}
tx, _, _, _ = rawdb.ReadTransaction(b.database, txHash)
if tx != nil {
return tx, false, nil
}
return nil, false, ethereum.NotFound
}
// PendingCodeAt returns the code associated with an account in the pending state.
func (b *SimulatedBackend) PendingCodeAt(ctx context.Context, contract common.Address) ([]byte, error) {
b.mu.Lock()

66
accounts/abi/bind/backends/simulated_test.go Normal file
View file

@ -0,0 +1,66 @@
package backends_test
import (
"context"
"math/big"
"testing"
ethereum "github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/accounts/abi/bind"
"github.com/ethereum/go-ethereum/accounts/abi/bind/backends"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
)
func TestSimulatedBackend(t *testing.T) {
var gasLimit uint64 = 8000029
key, _ := crypto.GenerateKey() // nolint: gosec
auth := bind.NewKeyedTransactor(key)
genAlloc := make(core.GenesisAlloc)
genAlloc[auth.From] = core.GenesisAccount{Balance: big.NewInt(9223372036854775807)}
sim := backends.NewSimulatedBackend(genAlloc, gasLimit)
// should return an error if the tx is not found
txHash := common.HexToHash("2")
_, isPending, err := sim.TransactionByHash(context.Background(), txHash)
if isPending {
t.Fatal("transaction should not be pending")
}
if err != ethereum.NotFound {
t.Fatalf("err should be `ethereum.NotFound` but received %v", err)
}
// generate a transaction and confirm you can retrieve it
code := `6060604052600a8060106000396000f360606040526008565b00`
var gas uint64 = 3000000
tx := types.NewContractCreation(0, big.NewInt(0), gas, big.NewInt(1), common.FromHex(code))
tx, _ = types.SignTx(tx, types.HomesteadSigner{}, key)
err = sim.SendTransaction(context.Background(), tx)
if err != nil {
t.Fatal("error sending transaction")
}
txHash = tx.Hash()
_, isPending, err = sim.TransactionByHash(context.Background(), txHash)
if err != nil {
t.Fatalf("error getting transaction with hash: %v", txHash.String())
}
if !isPending {
t.Fatal("transaction should have pending status")
}
sim.Commit()
tx, isPending, err = sim.TransactionByHash(context.Background(), txHash)
if err != nil {
t.Fatalf("error getting transaction with hash: %v", txHash.String())
}
if isPending {
t.Fatal("transaction should not have pending status")
}
}

39
accounts/external/backend.go vendored
View file

@ -26,7 +26,6 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/internal/ethapi"
"github.com/ethereum/go-ethereum/log"
@ -154,13 +153,31 @@ func (api *ExternalSigner) signHash(account accounts.Account, hash []byte) ([]by
// SignData signs keccak256(data). The mimetype parameter describes the type of data being signed
func (api *ExternalSigner) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) {
// TODO! Replace this with a call to clef SignData with correct mime-type for Clique, once we
// have that in place
return api.signHash(account, crypto.Keccak256(data))
var res hexutil.Bytes
var signAddress = common.NewMixedcaseAddress(account.Address)
if err := api.client.Call(&res, "account_signData",
mimeType,
&signAddress, // Need to use the pointer here, because of how MarshalJSON is defined
hexutil.Encode(data)); err != nil {
return nil, err
}
// If V is on 27/28-form, convert to to 0/1 for Clique
if mimeType == accounts.MimetypeClique && (res[64] == 27 || res[64] == 28) {
res[64] -= 27 // Transform V from 27/28 to 0/1 for Clique use
}
return res, nil
}
func (api *ExternalSigner) SignText(account accounts.Account, text []byte) ([]byte, error) {
return api.signHash(account, accounts.TextHash(text))
var res hexutil.Bytes
var signAddress = common.NewMixedcaseAddress(account.Address)
if err := api.client.Call(&res, "account_signData",
accounts.MimetypeTextPlain,
&signAddress, // Need to use the pointer here, because of how MarshalJSON is defined
hexutil.Encode(text)); err != nil {
return nil, err
}
return res, nil
}
func (api *ExternalSigner) SignTx(account accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
@ -202,18 +219,6 @@ func (api *ExternalSigner) listAccounts() ([]common.Address, error) {
return res, nil
}
func (api *ExternalSigner) signCliqueBlock(a common.Address, rlpBlock hexutil.Bytes) (hexutil.Bytes, error) {
var sig hexutil.Bytes
if err := api.client.Call(&sig, "account_signData", core.ApplicationClique.Mime, a, rlpBlock); err != nil {
return nil, err
}
if sig[64] != 27 && sig[64] != 28 {
return nil, fmt.Errorf("invalid Ethereum signature (V is not 27 or 28)")
}
sig[64] -= 27 // Transform V from 27/28 to 0/1 for Clique use
return sig, nil
}
func (api *ExternalSigner) pingVersion() (string, error) {
var v string
if err := api.client.Call(&v, "account_version"); err != nil {

13
build/ci-notes.md
View file

@ -7,11 +7,18 @@ Canonical.
Packages of develop branch commits have suffix -unstable and cannot be installed alongside
the stable version. Switching between release streams requires user intervention.
## Launchpad
The packages are built and served by launchpad.net. We generate a Debian source package
for each distribution and upload it. Their builder picks up the source package, builds it
and installs the new version into the PPA repository. Launchpad requires a valid signature
by a team member for source package uploads. The signing key is stored in an environment
variable which Travis CI makes available to certain builds.
by a team member for source package uploads.
The signing key is stored in an environment variable which Travis CI makes available to
certain builds. Since Travis CI doesn't support FTP, SFTP is used to transfer the
packages. To set this up yourself, you need to create a Launchpad user and add a GPG key
and SSH key to it. Then encode both keys as base64 and configure 'secret' environment
variables `PPA_SIGNING_KEY` and `PPA_SSH_KEY` on Travis.
We want to build go-ethereum with the most recent version of Go, irrespective of the Go
version that is available in the main Ubuntu repository. In order to make this possible,
@ -27,7 +34,7 @@ Add the gophers PPA and install Go 1.10 and Debian packaging tools:
$ sudo apt-add-repository ppa:gophers/ubuntu/archive
$ sudo apt-get update
$ sudo apt-get install build-essential golang-1.10 devscripts debhelper
$ sudo apt-get install build-essential golang-1.10 devscripts debhelper python-bzrlib python-paramiko
Create the source packages:

67
build/ci.go
View file

@ -441,11 +441,8 @@ func archiveBasename(arch string, archiveVersion string) string {
func archiveUpload(archive string, blobstore string, signer string) error {
// If signing was requested, generate the signature files
if signer != "" {
pgpkey, err := base64.StdEncoding.DecodeString(os.Getenv(signer))
if err != nil {
return fmt.Errorf("invalid base64 %s", signer)
}
if err := build.PGPSignFile(archive, archive+".asc", string(pgpkey)); err != nil {
key := getenvBase64(signer)
if err := build.PGPSignFile(archive, archive+".asc", string(key)); err != nil {
return err
}
}
@ -488,7 +485,8 @@ func maybeSkipArchive(env build.Environment) {
func doDebianSource(cmdline []string) {
var (
signer = flag.String("signer", "", `Signing key name, also used as package author`)
upload = flag.String("upload", "", `Where to upload the source package (usually "ppa:ethereum/ethereum")`)
upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`)
sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`)
workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
now = time.Now()
)
@ -498,11 +496,7 @@ func doDebianSource(cmdline []string) {
maybeSkipArchive(env)
// Import the signing key.
if b64key := os.Getenv("PPA_SIGNING_KEY"); b64key != "" {
key, err := base64.StdEncoding.DecodeString(b64key)
if err != nil {
log.Fatal("invalid base64 PPA_SIGNING_KEY")
}
if key := getenvBase64("PPA_SIGNING_KEY"); len(key) > 0 {
gpg := exec.Command("gpg", "--import")
gpg.Stdin = bytes.NewReader(key)
build.MustRun(gpg)
@ -513,22 +507,58 @@ func doDebianSource(cmdline []string) {
for _, distro := range debDistros {
meta := newDebMetadata(distro, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables)
pkgdir := stageDebianSource(*workdir, meta)
debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d")
debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz")
debuild.Dir = pkgdir
build.MustRun(debuild)
changes := fmt.Sprintf("%s_%s_source.changes", meta.Name(), meta.VersionString())
changes = filepath.Join(*workdir, changes)
var (
basename = fmt.Sprintf("%s_%s", meta.Name(), meta.VersionString())
source = filepath.Join(*workdir, basename+".tar.xz")
dsc = filepath.Join(*workdir, basename+".dsc")
changes = filepath.Join(*workdir, basename+"_source.changes")
)
if *signer != "" {
build.MustRunCommand("debsign", changes)
}
if *upload != "" {
build.MustRunCommand("dput", "--passive", "--no-upload-log", *upload, changes)
ppaUpload(*workdir, *upload, *sshUser, []string{source, dsc, changes})
}
}
}
}
func ppaUpload(workdir, ppa, sshUser string, files []string) {
p := strings.Split(ppa, "/")
if len(p) != 2 {
log.Fatal("-upload PPA name must contain single /")
}
if sshUser == "" {
sshUser = p[0]
}
incomingDir := fmt.Sprintf("~%s/ubuntu/%s", p[0], p[1])
// Create the SSH identity file if it doesn't exist.
var idfile string
if sshkey := getenvBase64("PPA_SSH_KEY"); len(sshkey) > 0 {
idfile = filepath.Join(workdir, "sshkey")
if _, err := os.Stat(idfile); os.IsNotExist(err) {
ioutil.WriteFile(idfile, sshkey, 0600)
}
}
// Upload
dest := sshUser + "@ppa.launchpad.net"
if err := build.UploadSFTP(idfile, dest, incomingDir, files); err != nil {
log.Fatal(err)
}
}
func getenvBase64(variable string) []byte {
dec, err := base64.StdEncoding.DecodeString(os.Getenv(variable))
if err != nil {
log.Fatal("invalid base64 " + variable)
}
return []byte(dec)
}
func makeWorkdir(wdflag string) string {
var err error
if wdflag != "" {
@ -800,15 +830,10 @@ func doAndroidArchive(cmdline []string) {
os.Rename(archive, meta.Package+".aar")
if *signer != "" && *deploy != "" {
// Import the signing key into the local GPG instance
b64key := os.Getenv(*signer)
key, err := base64.StdEncoding.DecodeString(b64key)
if err != nil {
log.Fatalf("invalid base64 %s", *signer)
}
key := getenvBase64(*signer)
gpg := exec.Command("gpg", "--import")
gpg.Stdin = bytes.NewReader(key)
build.MustRun(gpg)
keyID, err := build.PGPKeyID(string(key))
if err != nil {
log.Fatal(err)

16
cmd/abigen/main.go
View file

@ -136,14 +136,14 @@ func main() {
}
abis = append(abis, string(abi))
bin := []byte{}
if *binFlag != "" {
if bin, err = ioutil.ReadFile(*binFlag); err != nil {
fmt.Printf("Failed to read input bytecode: %v\n", err)
os.Exit(-1)
}
}
bins = append(bins, string(bin))
var bin []byte
if *binFlag != "" {
if bin, err = ioutil.ReadFile(*binFlag); err != nil {
fmt.Printf("Failed to read input bytecode: %v\n", err)
os.Exit(-1)
}
}
bins = append(bins, string(bin))
kind := *typFlag
if kind == "" {

25
cmd/clef/README.md
View file

@ -16,7 +16,8 @@ Check out
* the [tutorial](tutorial.md) for some concrete examples on how the signer works.
* the [setup docs](docs/setup.md) for some information on how to configure it to work on QubesOS or USBArmory.
* the [data types](datatypes.md) for detailed information on the json types used in the communication between
clef and an external UI
## Command line flags
Clef accepts the following command line options:
@ -24,28 +25,34 @@ Clef accepts the following command line options:
COMMANDS:
init Initialize the signer, generate secret storage
attest Attest that a js-file is to be used
addpw Store a credential for a keystore file
setpw Store a credential for a keystore file
gendoc Generate documentation about json-rpc format
help Shows a list of commands or help for one command
GLOBAL OPTIONS:
--loglevel value log level to emit to the screen (default: 4)
--keystore value Directory for the keystore (default: "$HOME/.ethereum/keystore")
--configdir value Directory for clef configuration (default: "$HOME/.clef")
--networkid value Network identifier (integer, 1=Frontier, 2=Morden (disused), 3=Ropsten, 4=Rinkeby) (default: 1)
--configdir value Directory for Clef configuration (default: "$HOME/.clef")
--chainid value Chain id to use for signing (1=mainnet, 3=ropsten, 4=rinkeby, 5=Goerli) (default: 1)
--lightkdf Reduce key-derivation RAM & CPU usage at some expense of KDF strength
--nousb Disables monitoring for and managing USB hardware wallets
--rpcaddr value HTTP-RPC server listening interface (default: "localhost")
--rpcvhosts value Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard. (default: "localhost")
--ipcdisable Disable the IPC-RPC server
--ipcpath Filename for IPC socket/pipe within the datadir (explicit paths escape it)
--rpc Enable the HTTP-RPC server
--rpcport value HTTP-RPC server listening port (default: 8550)
--signersecret value A file containing the password used to encrypt signer credentials, e.g. keystore credentials and ruleset hash
--signersecret value A file containing the (encrypted) master seed to encrypt Clef data, e.g. keystore credentials and ruleset hash
--4bytedb value File containing 4byte-identifiers (default: "./4byte.json")
--4bytedb-custom value File used for writing new 4byte-identifiers submitted via API (default: "./4byte-custom.json")
--auditlog value File used to emit audit logs. Set to "" to disable (default: "audit.log")
--rules value Enable rule-engine (default: "rules.json")
--stdio-ui Use STDIN/STDOUT as a channel for an external UI. This means that an STDIN/STDOUT is used for RPC-communication with a e.g. a graphical user interface, and can be used when the signer is started by an external process.
--stdio-ui-test Mechanism to test interface between signer and UI. Requires 'stdio-ui'.
--stdio-ui Use STDIN/STDOUT as a channel for an external UI. This means that an STDIN/STDOUT is used for RPC-communication with a e.g. a graphical user interface, and can be used when Clef is started by an external process.
--stdio-ui-test Mechanism to test interface between Clef and UI. Requires 'stdio-ui'.
--advanced If enabled, issues warnings instead of rejections for suspicious requests. Default off
--help, -h show help
--version, -v print the version
```

219
cmd/clef/datatypes.md Normal file
View file

@ -0,0 +1,219 @@
## UI Client interface
These data types are defined in the channel between clef and the UI
### SignDataRequest
SignDataRequest contains information about a pending request to sign some data. The data to be signed can be of various types, defined by content-type. Clef has done most of the work in canonicalizing and making sense of the data, and it's up to the UI to presentthe user with the contents of the `message`
Example:
```json
{
"content_type": "text/plain",
"address": "0xDEADbEeF000000000000000000000000DeaDbeEf",
"raw_data": "GUV0aGVyZXVtIFNpZ25lZCBNZXNzYWdlOgoxMWhlbGxvIHdvcmxk",
"message": [
{
"name": "message",
"value": "\u0019Ethereum Signed Message:\n11hello world",
"type": "text/plain"
}
],
"hash": "0xd9eba16ed0ecae432b71fe008c98cc872bb4cc214d3220a36f365326cf807d68",
"meta": {
"remote": "localhost:9999",
"local": "localhost:8545",
"scheme": "http",
"User-Agent": "Firefox 3.2",
"Origin": "www.malicious.ru"
}
}
```
### SignDataResponse - approve
Response to SignDataRequest
Example:
```json
{
"approved": true,
"Password": "apassword"
}
```
### SignDataResponse - deny
Response to SignDataRequest
Example:
```json
{
"approved": false,
"Password": ""
}
```
### SignTxRequest
SignTxRequest contains information about a pending request to sign a transaction. Aside from the transaction itself, there is also a `call_info`-struct. That struct contains messages of various types, that the user should be informed of.
As in any request, it's important to consider that the `meta` info also contains untrusted data.
The `transaction` (on input into clef) can have either `data` or `input` -- if both are set, they must be identical, otherwise an error is generated. However, Clef will always use `data` when passing this struct on (if Clef does otherwise, please file a ticket)
Example:
```json
{
"transaction": {
"from": "0xDEADbEeF000000000000000000000000DeaDbeEf",
"to": null,
"gas": "0x3e8",
"gasPrice": "0x5",
"value": "0x6",
"nonce": "0x1",
"data": "0x01020304"
},
"call_info": [
{
"type": "Warning",
"message": "Something looks odd, show this message as a warning"
},
{
"type": "Info",
"message": "User should see this aswell"
}
],
"meta": {
"remote": "localhost:9999",
"local": "localhost:8545",
"scheme": "http",
"User-Agent": "Firefox 3.2",
"Origin": "www.malicious.ru"
}
}
```
### SignDataResponse - approve
Response to SignDataRequest. This response needs to contain the `transaction`, because the UI is free to make modifications to the transaction.
Example:
```json
{
"transaction": {
"from": "0xDEADbEeF000000000000000000000000DeaDbeEf",
"to": null,
"gas": "0x3e8",
"gasPrice": "0x5",
"value": "0x6",
"nonce": "0x4",
"data": "0x04030201"
},
"approved": true,
"password": "apassword"
}
```
### SignDataResponse - deny
Response to SignDataRequest. When denying a request, there's no need to provide the transaction in return
Example:
```json
{
"approved": false,
"Password": ""
}
```
### OnApproved - SignTransactionResult
SignTransactionResult is used in the call `clef` -> `OnApprovedTx(result)`
This occurs _after_ successful completion of the entire signing procedure, but right before the signed transaction is passed to the external caller. This method (and data) can be used by the UI to signal to the user that the transaction was signed, but it is primarily useful for ruleset implementations.
A ruleset that implements a rate limitation needs to know what transactions are sent out to the external interface. By hooking into this methods, the ruleset can maintain track of that count.
**OBS:** Note that if an attacker can restore your `clef` data to a previous point in time (e.g through a backup), the attacker can reset such windows, even if he/she is unable to decrypt the content.
The `OnApproved` method cannot be responded to, it's purely informative
Example:
```json
{
"raw": "0xf85d640101948a8eafb1cf62bfbeb1741769dae1a9dd47996192018026a0716bd90515acb1e68e5ac5867aa11a1e65399c3349d479f5fb698554ebc6f293a04e8a4ebfff434e971e0ef12c5bf3a881b06fd04fc3f8b8a7291fb67a26a1d4ed",
"tx": {
"nonce": "0x64",
"gasPrice": "0x1",
"gas": "0x1",
"to": "0x8a8eafb1cf62bfbeb1741769dae1a9dd47996192",
"value": "0x1",
"input": "0x",
"v": "0x26",
"r": "0x716bd90515acb1e68e5ac5867aa11a1e65399c3349d479f5fb698554ebc6f293",
"s": "0x4e8a4ebfff434e971e0ef12c5bf3a881b06fd04fc3f8b8a7291fb67a26a1d4ed",
"hash": "0x662f6d772692dd692f1b5e8baa77a9ff95bbd909362df3fc3d301aafebde5441"
}
}
```
### UserInputRequest
Sent when clef needs the user to provide data. If 'password' is true, the input field should be treated accordingly (echo-free)
Example:
```json
{
"prompt": "The question to ask the user",
"title": "The title here",
"isPassword": true
}
```
### UserInputResponse
Response to SignDataRequest
Example:
```json
{
"text": "The textual response from user"
}
```
### ListRequest
Sent when a request has been made to list addresses. The UI is provided with the full `account`s, including local directory names. Note: this information is not passed back to the external caller, who only sees the `address`es.
Example:
```json
{
"accounts": [
{
"address": "0xdeadbeef000000000000000000000000deadbeef",
"url": "keystore:///path/to/keyfile/a"
},
{
"address": "0x1111111122222222222233333333334444444444",
"url": "keystore:///path/to/keyfile/b"
}
],
"meta": {
"remote": "localhost:9999",
"local": "localhost:8545",
"scheme": "http",
"User-Agent": "Firefox 3.2",
"Origin": "www.malicious.ru"
}
}
```
### UserInputResponse
Response to list request. The response contains a list of all addresses to show to the caller. Note: the UI is free to respond with any address the caller, regardless of whether it exists or not
Example:
```json
{
"accounts": [
{
"address": "0x0000000000000000000000000000000000000000",
"url": ".. ignored .."
},
{
"address": "0xffffffffffffffffffffffffffffffffffffffff",
"url": ""
}
]
}
```

BIN
cmd/clef/docs/clef_architecture_pt1.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 68 KiB

BIN
cmd/clef/docs/clef_architecture_pt2.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 80 KiB

BIN
cmd/clef/docs/clef_architecture_pt3.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 99 KiB

BIN
cmd/clef/docs/clef_architecture_pt4.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 115 KiB

5
cmd/clef/extapi_changelog.md
View file

@ -1,5 +1,10 @@
### Changelog for external API
### 6.0.0
* `New` was changed to deliver only an address, not the full `Account` data
* `Export` was moved from External API to the UI Server API
#### 5.0.0
* The external `account_EcRecover`-method was reimplemented.

33
cmd/clef/intapi_changelog.md
View file

@ -1,8 +1,39 @@
### Changelog for internal API (ui-api)
### 4.0.0
* Bidirectional communication implemented, so the UI can query `clef` via the stdin/stdout RPC channel. Methods implemented are:
- `clef_listWallets`
- `clef_listAccounts`
- `clef_listWallets`
- `clef_deriveAccount`
- `clef_importRawKey`
- `clef_openWallet`
- `clef_chainId`
- `clef_setChainId`
- `clef_export`
- `clef_import`
* The type `Account` was modified (the json-field `type` was removed), to consist of
```golang
type Account struct {
Address common.Address `json:"address"` // Ethereum account address derived from the key
URL URL `json:"url"` // Optional resource locator within a backend
}
```
### 3.2.0
* Make `ShowError`, `OnApprovedTx`, `OnSignerStartup` be json-rpc [notifications](https://www.jsonrpc.org/specification#notification):
> A Notification is a Request object without an "id" member. A Request object that is a Notification signifies the Client's lack of interest in the corresponding Response object, and as such no Response object needs to be returned to the client. The Server MUST NOT reply to a Notification, including those that are within a batch request.
>
> Notifications are not confirmable by definition, since they do not have a Response object to be returned. As such, the Client would not be aware of any errors (like e.g. "Invalid params","Internal error"
### 3.1.0
* Add `ContentType string` to `SignDataRequest` to accommodate the latest EIP-191 and EIP-712 implementations.
* Add `ContentType` `string` to `SignDataRequest` to accommodate the latest EIP-191 and EIP-712 implementations.
### 3.0.0

202
cmd/clef/main.go
View file

@ -36,14 +36,18 @@ import (
"runtime"
"strings"
"github.com/ethereum/go-ethereum/accounts"
"github.com/ethereum/go-ethereum/accounts/keystore"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/console"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/internal/ethapi"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/signer/core"
@ -83,6 +87,11 @@ var (
Value: DefaultConfigDir(),
Usage: "Directory for Clef configuration",
}
chainIdFlag = cli.Int64Flag{
Name: "chainid",
Value: params.MainnetChainConfig.ChainID.Int64(),
Usage: "Chain id to use for signing (1=mainnet, 3=ropsten, 4=rinkeby, 5=Goerli)",
}
rpcPortFlag = cli.IntFlag{
Name: "rpcport",
Usage: "HTTP-RPC server listening port",
@ -165,10 +174,16 @@ Clef that the file is 'safe' to execute.`,
signerSecretFlag,
},
Description: `
The setpw command stores a password for a given address (keyfile). If you enter a blank passphrase, it will
The setpw command stores a password for a given address (keyfile). If you enter a blank passphrase, it will
remove any stored credential for that address (keyfile)
`,
}
`}
gendocCommand = cli.Command{
Action: GenDoc,
Name: "gendoc",
Usage: "Generate documentation about json-rpc format",
Description: `
The gendoc generates example structures of the json-rpc communication types.
`}
)
func init() {
@ -178,7 +193,7 @@ func init() {
logLevelFlag,
keystoreFlag,
configdirFlag,
utils.NetworkIdFlag,
chainIdFlag,
utils.LightKDFFlag,
utils.NoUSBFlag,
utils.RPCListenAddrFlag,
@ -197,7 +212,7 @@ func init() {
advancedMode,
}
app.Action = signer
app.Commands = []cli.Command{initCommand, attestCommand, setCredentialCommand}
app.Commands = []cli.Command{initCommand, attestCommand, setCredentialCommand, gendocCommand}
}
func main() {
@ -338,7 +353,7 @@ func signer(c *cli.Context) error {
return err
}
var (
ui core.SignerUI
ui core.UIClientAPI
)
if c.GlobalBool(stdiouiFlag.Name) {
log.Info("Using stdin/stdout as UI-channel")
@ -402,14 +417,21 @@ func signer(c *cli.Context) error {
}
}
}
var (
chainId = c.GlobalInt64(chainIdFlag.Name)
ksLoc = c.GlobalString(keystoreFlag.Name)
lightKdf = c.GlobalBool(utils.LightKDFFlag.Name)
advanced = c.GlobalBool(advancedMode.Name)
nousb = c.GlobalBool(utils.NoUSBFlag.Name)
)
log.Info("Starting signer", "chainid", chainId, "keystore", ksLoc,
"light-kdf", lightKdf, "advanced", advanced)
am := core.StartClefAccountManager(ksLoc, nousb, lightKdf)
apiImpl := core.NewSignerAPI(am, chainId, nousb, ui, db, advanced)
apiImpl := core.NewSignerAPI(
c.GlobalInt64(utils.NetworkIdFlag.Name),
c.GlobalString(keystoreFlag.Name),
c.GlobalBool(utils.NoUSBFlag.Name),
ui, db,
c.GlobalBool(utils.LightKDFFlag.Name),
c.GlobalBool(advancedMode.Name))
// Establish the bidirectional communication, by creating a new UI backend and registering
// it with the UI.
ui.RegisterUIServer(core.NewUIServerAPI(apiImpl))
api = apiImpl
// Audit logging
if logfile := c.GlobalString(auditLogFlag.Name); logfile != "" {
@ -529,7 +551,7 @@ func homeDir() string {
}
return ""
}
func readMasterKey(ctx *cli.Context, ui core.SignerUI) ([]byte, error) {
func readMasterKey(ctx *cli.Context, ui core.UIClientAPI) ([]byte, error) {
var (
file string
configDir = ctx.GlobalString(configdirFlag.Name)
@ -664,10 +686,6 @@ func testExternalUI(api *core.SignerAPI) {
checkErr("List", err)
_, err = api.New(ctx)
checkErr("New", err)
_, err = api.Export(ctx, common.Address{})
checkErr("Export", err)
_, err = api.Import(ctx, json.RawMessage{})
checkErr("Import", err)
api.UI.ShowInfo("Tests completed")
@ -734,6 +752,154 @@ func decryptSeed(keyjson []byte, auth string) ([]byte, error) {
return seed, err
}
// GenDoc outputs examples of all structures used in json-rpc communication
func GenDoc(ctx *cli.Context) {
var (
a = common.HexToAddress("0xdeadbeef000000000000000000000000deadbeef")
b = common.HexToAddress("0x1111111122222222222233333333334444444444")
meta = core.Metadata{
Scheme: "http",
Local: "localhost:8545",
Origin: "www.malicious.ru",
Remote: "localhost:9999",
UserAgent: "Firefox 3.2",
}
output []string
add = func(name, desc string, v interface{}) {
if data, err := json.MarshalIndent(v, "", " "); err == nil {
output = append(output, fmt.Sprintf("### %s\n\n%s\n\nExample:\n```json\n%s\n```", name, desc, data))
} else {
log.Error("Error generating output", err)
}
}
)
{ // Sign plain text request
desc := "SignDataRequest contains information about a pending request to sign some data. " +
"The data to be signed can be of various types, defined by content-type. Clef has done most " +
"of the work in canonicalizing and making sense of the data, and it's up to the UI to present" +
"the user with the contents of the `message`"
sighash, msg := accounts.TextAndHash([]byte("hello world"))
message := []*core.NameValueType{{"message", msg, accounts.MimetypeTextPlain}}
add("SignDataRequest", desc, &core.SignDataRequest{
Address: common.NewMixedcaseAddress(a),
Meta: meta,
ContentType: accounts.MimetypeTextPlain,
Rawdata: []byte(msg),
Message: message,
Hash: sighash})
}
{ // Sign plain text response
add("SignDataResponse - approve", "Response to SignDataRequest",
&core.SignDataResponse{Password: "apassword", Approved: true})
add("SignDataResponse - deny", "Response to SignDataRequest",
&core.SignDataResponse{})
}
{ // Sign transaction request
desc := "SignTxRequest contains information about a pending request to sign a transaction. " +
"Aside from the transaction itself, there is also a `call_info`-struct. That struct contains " +
"messages of various types, that the user should be informed of." +
"\n\n" +
"As in any request, it's important to consider that the `meta` info also contains untrusted data." +
"\n\n" +
"The `transaction` (on input into clef) can have either `data` or `input` -- if both are set, " +
"they must be identical, otherwise an error is generated. " +
"However, Clef will always use `data` when passing this struct on (if Clef does otherwise, please file a ticket)"
data := hexutil.Bytes([]byte{0x01, 0x02, 0x03, 0x04})
add("SignTxRequest", desc, &core.SignTxRequest{
Meta: meta,
Callinfo: []core.ValidationInfo{
{"Warning", "Something looks odd, show this message as a warning"},
{"Info", "User should see this aswell"},
},
Transaction: core.SendTxArgs{
Data: &data,
Nonce: 0x1,
Value: hexutil.Big(*big.NewInt(6)),
From: common.NewMixedcaseAddress(a),
To: nil,
GasPrice: hexutil.Big(*big.NewInt(5)),
Gas: 1000,
Input: nil,
}})
}
{ // Sign tx response
data := hexutil.Bytes([]byte{0x04, 0x03, 0x02, 0x01})
add("SignDataResponse - approve", "Response to SignDataRequest. This response needs to contain the `transaction`"+
", because the UI is free to make modifications to the transaction.",
&core.SignTxResponse{Password: "apassword", Approved: true,
Transaction: core.SendTxArgs{
Data: &data,
Nonce: 0x4,
Value: hexutil.Big(*big.NewInt(6)),
From: common.NewMixedcaseAddress(a),
To: nil,
GasPrice: hexutil.Big(*big.NewInt(5)),
Gas: 1000,
Input: nil,
}})
add("SignDataResponse - deny", "Response to SignDataRequest. When denying a request, there's no need to "+
"provide the transaction in return",
&core.SignDataResponse{})
}
{ // WHen a signed tx is ready to go out
desc := "SignTransactionResult is used in the call `clef` -> `OnApprovedTx(result)`" +
"\n\n" +
"This occurs _after_ successful completion of the entire signing procedure, but right before the signed " +
"transaction is passed to the external caller. This method (and data) can be used by the UI to signal " +
"to the user that the transaction was signed, but it is primarily useful for ruleset implementations." +
"\n\n" +
"A ruleset that implements a rate limitation needs to know what transactions are sent out to the external " +
"interface. By hooking into this methods, the ruleset can maintain track of that count." +
"\n\n" +
"**OBS:** Note that if an attacker can restore your `clef` data to a previous point in time" +
" (e.g through a backup), the attacker can reset such windows, even if he/she is unable to decrypt the content. " +
"\n\n" +
"The `OnApproved` method cannot be responded to, it's purely informative"
rlpdata := common.FromHex("0xf85d640101948a8eafb1cf62bfbeb1741769dae1a9dd47996192018026a0716bd90515acb1e68e5ac5867aa11a1e65399c3349d479f5fb698554ebc6f293a04e8a4ebfff434e971e0ef12c5bf3a881b06fd04fc3f8b8a7291fb67a26a1d4ed")
var tx types.Transaction
rlp.DecodeBytes(rlpdata, &tx)
add("OnApproved - SignTransactionResult", desc, &ethapi.SignTransactionResult{Raw: rlpdata, Tx: &tx})
}
{ // User input
add("UserInputRequest", "Sent when clef needs the user to provide data. If 'password' is true, the input field should be treated accordingly (echo-free)",
&core.UserInputRequest{IsPassword: true, Title: "The title here", Prompt: "The question to ask the user"})
add("UserInputResponse", "Response to SignDataRequest",
&core.UserInputResponse{Text: "The textual response from user"})
}
{ // List request
add("ListRequest", "Sent when a request has been made to list addresses. The UI is provided with the "+
"full `account`s, including local directory names. Note: this information is not passed back to the external caller, "+
"who only sees the `address`es. ",
&core.ListRequest{
Meta: meta,
Accounts: []accounts.Account{
{a, accounts.URL{Scheme: "keystore", Path: "/path/to/keyfile/a"}},
{b, accounts.URL{Scheme: "keystore", Path: "/path/to/keyfile/b"}}},
})
add("UserInputResponse", "Response to list request. The response contains a list of all addresses to show to the caller. "+
"Note: the UI is free to respond with any address the caller, regardless of whether it exists or not",
&core.ListResponse{
Accounts: []accounts.Account{
{common.HexToAddress("0xcowbeef000000cowbeef00000000000000000c0w"), accounts.URL{Path: ".. ignored .."}},
{common.HexToAddress("0xffffffffffffffffffffffffffffffffffffffff"), accounts.URL{}},
}})
}
fmt.Println(`## UI Client interface
These data types are defined in the channel between clef and the UI`)
for _, elem := range output {
fmt.Println(elem)
}
}
/**
//Create Account

73
cmd/clef/tests/testsigner.js Normal file
View file

@ -0,0 +1,73 @@
// This file is a test-utility for testing clef-functionality
//
// Start clef with
//
// build/bin/clef --4bytedb=./cmd/clef/4byte.json --rpc
//
// Start geth with
//
// build/bin/geth --nodiscover --maxpeers 0 --signer http://localhost:8550 console --preload=cmd/clef/tests/testsigner.js
//
// and in the console simply invoke
//
// > test()
//
// You can reload the file via `reload()`
function reload(){
loadScript("./cmd/clef/tests/testsigner.js");
}
function init(){
if (typeof accts == 'undefined' || accts.length == 0){
accts = eth.accounts
console.log("Got accounts ", accts);
}
}
init()
function testTx(){
if( accts && accts.length > 0) {
var a = accts[0]
var txdata = eth.signTransaction({from: a, to: a, value: 1, nonce: 1, gas: 1, gasPrice: 1})
var v = parseInt(txdata.tx.v)
console.log("V value: ", v)
if (v == 37 || v == 38){
console.log("Mainnet 155-protected chainid was used")
}
if (v == 27 || v == 28){
throw new Error("Mainnet chainid was used, but without replay protection!")
}
}
}
function testSignText(){
if( accts && accts.length > 0){
var a = accts[0]
var r = eth.sign(a, "0x68656c6c6f20776f726c64"); //hello world
console.log("signing response", r)
}
}
function testClique(){
if( accts && accts.length > 0){
var a = accts[0]
var r = debug.testSignCliqueBlock(a, 0); // Sign genesis
console.log("signing response", r)
if( a != r){
throw new Error("Requested signing by "+a+ " but got sealer "+r)
}
}
}
function test(){
var tests = [
testTx,
testSignText,
testClique,
]
for( i in tests){
try{
tests[i]()
}catch(err){
console.log(err)
}
}
}

6
cmd/geth/monitorcmd.go
View file

@ -169,7 +169,7 @@ func retrieveMetrics(client *rpc.Client) (map[string]interface{}, error) {
// resolveMetrics takes a list of input metric patterns, and resolves each to one
// or more canonical metric names.
func resolveMetrics(metrics map[string]interface{}, patterns []string) []string {
res := []string{}
var res []string
for _, pattern := range patterns {
res = append(res, resolveMetric(metrics, pattern, "")...)
}
@ -179,7 +179,7 @@ func resolveMetrics(metrics map[string]interface{}, patterns []string) []string
// resolveMetrics takes a single of input metric pattern, and resolves it to one
// or more canonical metric names.
func resolveMetric(metrics map[string]interface{}, pattern string, path string) []string {
results := []string{}
var results []string
// If a nested metric was requested, recurse optionally branching (via comma)
parts := strings.SplitN(pattern, "/", 2)
@ -215,7 +215,7 @@ func resolveMetric(metrics map[string]interface{}, pattern string, path string)
// expandMetrics expands the entire tree of metrics into a flat list of paths.
func expandMetrics(metrics map[string]interface{}, path string) []string {
// Iterate over all fields and expand individually
list := []string{}
var list []string
for name, metric := range metrics {
switch metric := metric.(type) {
case float64:

2
cmd/geth/usage.go
View file

@ -308,7 +308,7 @@ func init() {
categorized[flag.String()] = struct{}{}
}
}
uncategorized := []cli.Flag{}
var uncategorized []cli.Flag
for _, flag := range data.(*cli.App).Flags {
if _, ok := categorized[flag.String()]; !ok {
if strings.HasPrefix(flag.GetName(), "dashboard") {

4
cmd/swarm/access.go
View file

@ -178,8 +178,8 @@ func accessNewACT(ctx *cli.Context) {
accessKey []byte
err error
ref = args[0]
pkGrantees = []string{}
passGrantees = []string{}
pkGrantees []string
passGrantees []string
pkGranteesFilename = ctx.String(SwarmAccessGrantKeysFlag.Name)
passGranteesFilename = ctx.String(utils.PasswordFileFlag.Name)
privateKey = getPrivKey(ctx)

4
cmd/swarm/access_test.go
View file

@ -397,7 +397,7 @@ func testACT(t *testing.T, bogusEntries int) {
}
ref := matches[0]
grantees := []string{}
var grantees []string
for i, v := range cluster.Nodes {
if i == nodeToSkip {
continue
@ -408,7 +408,7 @@ func testACT(t *testing.T, bogusEntries int) {
}
if bogusEntries > 0 {
bogusGrantees := []string{}
var bogusGrantees []string
for i := 0; i < bogusEntries; i++ {
prv, err := ecies.GenerateKey(rand.Reader, DefaultCurve, nil)

2
cmd/swarm/fs.go
View file

@ -123,7 +123,7 @@ func listMounts(cliContext *cli.Context) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
mf := []fuse.MountInfo{}
var mf []fuse.MountInfo
err = client.CallContext(ctx, &mf, "swarmfs_listmounts")
if err != nil {
utils.Fatalf("encountered an error calling the RPC endpoint while listing mounts: %v", err)

25
cmd/swarm/swarm-smoke/main.go
View file

@ -37,15 +37,16 @@ var (
)
var (
allhosts string
hosts []string
filesize int
syncDelay int
httpPort int
wsPort int
verbosity int
timeout int
single bool
allhosts string
hosts []string
filesize int
syncDelay int
httpPort int
wsPort int
verbosity int
timeout int
single bool
trackTimeout int
)
func main() {
@ -102,6 +103,12 @@ func main() {
Usage: "whether to fetch content from a single node or from all nodes",
Destination: &single,
},
cli.IntFlag{
Name: "track-timeout",
Value: 5,
Usage: "timeout in seconds to wait for GetAllReferences to return",
Destination: &trackTimeout,
},
}
app.Flags = append(app.Flags, []cli.Flag{

2
cmd/swarm/swarm-smoke/sliding_window.go
View file

@ -56,7 +56,7 @@ func slidingWindowCmd(ctx *cli.Context, tuid string) error {
}
func slidingWindow(ctx *cli.Context, tuid string) error {
hashes := []uploadResult{} //swarm hashes of the uploads
var hashes []uploadResult //swarm hashes of the uploads
nodes := len(hosts)
const iterationTimeout = 30 * time.Second
log.Info("sliding window test started", "tuid", tuid, "nodes", nodes, "filesize(kb)", filesize, "timeout", timeout)

71
cmd/swarm/swarm-smoke/upload_and_sync.go
View file

@ -18,13 +18,19 @@ package main
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"math/rand"
"os"
"sync"
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/swarm/api"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/testutil"
"github.com/pborman/uuid"
@ -49,12 +55,75 @@ func uploadAndSyncCmd(ctx *cli.Context, tuid string) error {
case <-time.After(time.Duration(timeout) * time.Second):
metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1)
e := fmt.Errorf("timeout after %v sec", timeout)
// trigger debug functionality on randomBytes
err := trackChunks(randomBytes[:])
if err != nil {
e = fmt.Errorf("%v; triggerChunkDebug failed: %v", e, err)
}
return fmt.Errorf("timeout after %v sec", timeout)
return e
}
}
func trackChunks(testData []byte) error {
log.Warn("Test timed out; running chunk debug sequence")
addrs, err := getAllRefs(testData)
if err != nil {
return err
}
log.Trace("All references retrieved")
// has-chunks
for _, host := range hosts {
httpHost := fmt.Sprintf("ws://%s:%d", host, 8546)
log.Trace("Calling `Has` on host", "httpHost", httpHost)
rpcClient, err := rpc.Dial(httpHost)
if err != nil {
log.Trace("Error dialing host", "err", err)
return err
}
log.Trace("rpc dial ok")
var hasInfo []api.HasInfo
err = rpcClient.Call(&hasInfo, "bzz_has", addrs)
if err != nil {
log.Trace("Error calling host", "err", err)
return err
}
log.Trace("rpc call ok")
count := 0
for _, info := range hasInfo {
if !info.Has {
count++
log.Error("Host does not have chunk", "host", httpHost, "chunk", info.Addr)
}
}
if count == 0 {
log.Info("Host reported to have all chunks", "host", httpHost)
}
}
return nil
}
func getAllRefs(testData []byte) (storage.AddressCollection, error) {
log.Trace("Getting all references for given root hash")
datadir, err := ioutil.TempDir("", "chunk-debug")
if err != nil {
return nil, fmt.Errorf("unable to create temp dir: %v", err)
}
defer os.RemoveAll(datadir)
fileStore, err := storage.NewLocalFileStore(datadir, make([]byte, 32))
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(trackTimeout)*time.Second)
defer cancel()
reader := bytes.NewReader(testData)
return fileStore.GetAllReferences(ctx, reader, false)
}
func uplaodAndSync(c *cli.Context, randomBytes []byte, tuid string) error {
log.Info("uploading to "+httpEndpoint(hosts[0])+" and syncing", "tuid", tuid, "seed", seed)

7
cmd/utils/flags.go
View file

@ -950,10 +950,11 @@ func makeDatabaseHandles() int {
if err != nil {
Fatalf("Failed to retrieve file descriptor allowance: %v", err)
}
if err := fdlimit.Raise(uint64(limit)); err != nil {
raised, err := fdlimit.Raise(uint64(limit))
if err != nil {
Fatalf("Failed to raise file descriptor allowance: %v", err)
}
return limit / 2 // Leave half for networking and other stuff
return int(raised / 2) // Leave half for networking and other stuff
}
// MakeAddress converts an account specified directly as a hex encoded string or
@ -1612,7 +1613,7 @@ func MakeConsolePreloads(ctx *cli.Context) []string {
return nil
}
// Otherwise resolve absolute paths and return them
preloads := []string{}
var preloads []string
assets := ctx.GlobalString(JSpathFlag.Name)
for _, file := range strings.Split(ctx.GlobalString(PreloadJSFlag.Name), ",") {

11
common/fdlimit/fdlimit_freebsd.go
View file

@ -26,11 +26,11 @@ import "syscall"
// Raise tries to maximize the file descriptor allowance of this process
// to the maximum hard-limit allowed by the OS.
func Raise(max uint64) error {
func Raise(max uint64) (uint64, error) {
// Get the current limit
var limit syscall.Rlimit
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
return err
return 0, err
}
// Try to update the limit to the max allowance
limit.Cur = limit.Max
@ -38,9 +38,12 @@ func Raise(max uint64) error {
limit.Cur = int64(max)
}
if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
return err
return 0, err
}
return nil
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
return 0, err
}
return limit.Cur, nil
}
// Current retrieves the number of file descriptors allowed to be opened by this

2
common/fdlimit/fdlimit_test.go
View file

@ -36,7 +36,7 @@ func TestFileDescriptorLimits(t *testing.T) {
if limit, err := Current(); err != nil || limit <= 0 {
t.Fatalf("failed to retrieve file descriptor limit (%d): %v", limit, err)
}
if err := Raise(uint64(target)); err != nil {
if _, err := Raise(uint64(target)); err != nil {
t.Fatalf("failed to raise file allowance")
}
if limit, err := Current(); err != nil || limit < target {

13
common/fdlimit/fdlimit_unix.go
View file

@ -22,11 +22,12 @@ import "syscall"
// Raise tries to maximize the file descriptor allowance of this process
// to the maximum hard-limit allowed by the OS.
func Raise(max uint64) error {
// Returns the size it was set to (may differ from the desired 'max')
func Raise(max uint64) (uint64, error) {
// Get the current limit
var limit syscall.Rlimit
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
return err
return 0, err
}
// Try to update the limit to the max allowance
limit.Cur = limit.Max
@ -34,9 +35,13 @@ func Raise(max uint64) error {
limit.Cur = max
}
if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
return err
return 0, err
}
return nil
// MacOS can silently apply further caps, so retrieve the actually set limit
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
return 0, err
}
return limit.Cur, nil
}
// Current retrieves the number of file descriptors allowed to be opened by this

14
common/fdlimit/fdlimit_windows.go
View file

@ -16,28 +16,30 @@
package fdlimit
import "errors"
import "fmt"
const hardlimit = 16384
// Raise tries to maximize the file descriptor allowance of this process
// to the maximum hard-limit allowed by the OS.
func Raise(max uint64) error {
func Raise(max uint64) (uint64, error) {
// This method is NOP by design:
// * Linux/Darwin counterparts need to manually increase per process limits
// * On Windows Go uses the CreateFile API, which is limited to 16K files, non
// changeable from within a running process
// This way we can always "request" raising the limits, which will either have
// or not have effect based on the platform we're running on.
if max > 16384 {
return errors.New("file descriptor limit (16384) reached")
if max > hardlimit {
return hardlimit, fmt.Errorf("file descriptor limit (%d) reached", hardlimit)
}
return nil
return max, nil
}
// Current retrieves the number of file descriptors allowed to be opened by this
// process.
func Current() (int, error) {
// Please see Raise for the reason why we use hard coded 16K as the limit
return 16384, nil
return hardlimit, nil
}
// Maximum retrieves the maximum number of file descriptors this process is

6
console/console.go
View file

@ -120,10 +120,10 @@ func (c *Console) init(preload []string) error {
consoleObj.Object().Set("error", c.consoleOutput)
// Load all the internal utility JavaScript libraries
if err := c.jsre.Compile("bignumber.js", jsre.BigNumber_JS); err != nil {
if err := c.jsre.Compile("bignumber.js", jsre.BignumberJs); err != nil {
return fmt.Errorf("bignumber.js: %v", err)
}
if err := c.jsre.Compile("web3.js", jsre.Web3_JS); err != nil {
if err := c.jsre.Compile("web3.js", jsre.Web3Js); err != nil {
return fmt.Errorf("web3.js: %v", err)
}
if _, err := c.jsre.Run("var Web3 = require('web3');"); err != nil {
@ -236,7 +236,7 @@ func (c *Console) clearHistory() {
// consoleOutput is an override for the console.log and console.error methods to
// stream the output into the configured output stream instead of stdout.
func (c *Console) consoleOutput(call otto.FunctionCall) otto.Value {
output := []string{}
var output []string
for _, argument := range call.ArgumentList {
output = append(output, fmt.Sprintf("%v", argument))
}

17
core/blockchain_test.go
View file

@ -17,7 +17,6 @@
package core
import (
"fmt"
"math/big"
"math/rand"
"sync"
@ -126,13 +125,6 @@ func testFork(t *testing.T, blockchain *BlockChain, i, n int, full bool, compara
comparator(tdPre, tdPost)
}
func printChain(bc *BlockChain) {
for i := bc.CurrentBlock().Number().Uint64(); i > 0; i-- {
b := bc.GetBlockByNumber(uint64(i))
fmt.Printf("\t%x %v\n", b.Hash(), b.Difficulty())
}
}
// testBlockChainImport tries to process a chain of blocks, writing them into
// the database if successful.
func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error {
@ -188,15 +180,6 @@ func testHeaderChainImport(chain []*types.Header, blockchain *BlockChain) error
return nil
}
func insertChain(done chan bool, blockchain *BlockChain, chain types.Blocks, t *testing.T) {
_, err := blockchain.InsertChain(chain)
if err != nil {
fmt.Println(err)
t.FailNow()
}
done <- true
}
func TestLastBlock(t *testing.T) {
_, blockchain, err := newCanonical(ethash.NewFaker(), 0, true)
if err != nil {

12
core/tx_pool_test.go
View file

@ -1274,14 +1274,14 @@ func TestTransactionPoolRepricingKeepsLocals(t *testing.T) {
}
// Create transaction (both pending and queued) with a linearly growing gasprice
for i := uint64(0); i < 500; i++ {
// Add pending
p_tx := pricedTransaction(i, 100000, big.NewInt(int64(i)), keys[2])
if err := pool.AddLocal(p_tx); err != nil {
// Add pending transaction.
pendingTx := pricedTransaction(i, 100000, big.NewInt(int64(i)), keys[2])
if err := pool.AddLocal(pendingTx); err != nil {
t.Fatal(err)
}
// Add queued
q_tx := pricedTransaction(i+501, 100000, big.NewInt(int64(i)), keys[2])
if err := pool.AddLocal(q_tx); err != nil {
// Add queued transaction.
queuedTx := pricedTransaction(i+501, 100000, big.NewInt(int64(i)), keys[2])
if err := pool.AddLocal(queuedTx); err != nil {
t.Fatal(err)
}
}

15
core/vm/evm.go
View file

@ -267,9 +267,8 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte,
snapshot = evm.StateDB.Snapshot()
to = AccountRef(caller.Address())
)
// initialise a new contract and set the code that is to be used by the
// EVM. The contract is a scoped environment for this execution context
// only.
// Initialise a new contract and set the code that is to be used by the EVM.
// The contract is a scoped environment for this execution context only.
contract := NewContract(caller, to, value, gas)
contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr))
@ -333,9 +332,8 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte
to = AccountRef(addr)
snapshot = evm.StateDB.Snapshot()
)
// Initialise a new contract and set the code that is to be used by the
// EVM. The contract is a scoped environment for this execution context
// only.
// Initialise a new contract and set the code that is to be used by the EVM.
// The contract is a scoped environment for this execution context only.
contract := NewContract(caller, to, new(big.Int), gas)
contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr))
@ -396,9 +394,8 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64,
}
evm.Transfer(evm.StateDB, caller.Address(), address, value)
// initialise a new contract and set the code that is to be used by the
// EVM. The contract is a scoped environment for this execution context
// only.
// Initialise a new contract and set the code that is to be used by the EVM.
// The contract is a scoped environment for this execution context only.
contract := NewContract(caller, AccountRef(address), value, gas)
contract.SetCodeOptionalHash(&address, codeAndHash)

13
crypto/ecies/ecies_test.go
View file

@ -93,19 +93,6 @@ func cmpPublic(pub1, pub2 PublicKey) bool {
return bytes.Equal(pub1Out, pub2Out)
}
// cmpPrivate returns true if the two private keys are the same.
func cmpPrivate(prv1, prv2 *PrivateKey) bool {
if prv1 == nil || prv1.D == nil {
return false
} else if prv2 == nil || prv2.D == nil {
return false
} else if prv1.D.Cmp(prv2.D) != 0 {
return false
} else {
return cmpPublic(prv1.PublicKey, prv2.PublicKey)
}
}
// Validate the ECDH component.
func TestSharedKey(t *testing.T) {
prv1, err := GenerateKey(rand.Reader, DefaultCurve, nil)

2
eth/downloader/downloader.go
View file

@ -719,7 +719,7 @@ func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header)
}
// Make sure the peer's reply conforms to the request
for i, header := range headers {
expectNumber := from + int64(i)*int64((skip+1))
expectNumber := from + int64(i)*int64(skip+1)
if number := header.Number.Int64(); number != expectNumber {
p.log.Warn("Head headers broke chain ordering", "index", i, "requested", expectNumber, "received", number)
return 0, errInvalidChain

4
eth/tracers/tracers_test.go
View file

@ -121,7 +121,7 @@ type callTracerTest struct {
}
func TestPrestateTracerCreate2(t *testing.T) {
unsigned_tx := types.NewTransaction(1, common.HexToAddress("0x00000000000000000000000000000000deadbeef"),
unsignedTx := types.NewTransaction(1, common.HexToAddress("0x00000000000000000000000000000000deadbeef"),
new(big.Int), 5000000, big.NewInt(1), []byte{})
privateKeyECDSA, err := ecdsa.GenerateKey(crypto.S256(), rand.Reader)
@ -129,7 +129,7 @@ func TestPrestateTracerCreate2(t *testing.T) {
t.Fatalf("err %v", err)
}
signer := types.NewEIP155Signer(big.NewInt(1))
tx, err := types.SignTx(unsigned_tx, signer, privateKeyECDSA)
tx, err := types.SignTx(unsignedTx, signer, privateKeyECDSA)
if err != nil {
t.Fatalf("err %v", err)
}

31
internal/build/util.go
View file

@ -177,3 +177,34 @@ func ExpandPackagesNoVendor(patterns []string) []string {
}
return patterns
}
// UploadSFTP uploads files to a remote host using the sftp command line tool.
// The destination host may be specified either as [user@]host[: or as a URI in
// the form sftp://[user@]host[:port].
func UploadSFTP(identityFile, host, dir string, files []string) error {
sftp := exec.Command("sftp")
sftp.Stdout = nil
sftp.Stderr = os.Stderr
if identityFile != "" {
sftp.Args = append(sftp.Args, "-i", identityFile)
}
sftp.Args = append(sftp.Args, host)
fmt.Println(">>>", strings.Join(sftp.Args, " "))
if *DryRunFlag {
return nil
}
stdin, err := sftp.StdinPipe()
if err != nil {
return fmt.Errorf("can't create stdin pipe for sftp: %v", err)
}
if err := sftp.Start(); err != nil {
return err
}
in := io.MultiWriter(stdin, os.Stdout)
for _, f := range files {
fmt.Fprintln(in, "put", f, path.Join(dir, filepath.Base(f)))
}
stdin.Close()
return sftp.Wait()
}

40
internal/ethapi/api.go
View file

@ -31,6 +31,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/common/math"
"github.com/ethereum/go-ethereum/consensus/clique"
"github.com/ethereum/go-ethereum/consensus/ethash"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
@ -1469,6 +1470,45 @@ func (api *PublicDebugAPI) GetBlockRlp(ctx context.Context, number uint64) (stri
return fmt.Sprintf("%x", encoded), nil
}
// TestSignCliqueBlock fetches the given block number, and attempts to sign it as a clique header with the
// given address, returning the address of the recovered signature
//
// This is a temporary method to debug the externalsigner integration,
// TODO: Remove this method when the integration is mature
func (api *PublicDebugAPI) TestSignCliqueBlock(ctx context.Context, address common.Address, number uint64) (common.Address, error) {
block, _ := api.b.BlockByNumber(ctx, rpc.BlockNumber(number))
if block == nil {
return common.Address{}, fmt.Errorf("block #%d not found", number)
}
header := block.Header()
header.Extra = make([]byte, 32+65)
encoded := clique.CliqueRLP(header)
// Look up the wallet containing the requested signer
account := accounts.Account{Address: address}
wallet, err := api.b.AccountManager().Find(account)
if err != nil {
return common.Address{}, err
}
signature, err := wallet.SignData(account, accounts.MimetypeClique, encoded)
if err != nil {
return common.Address{}, err
}
sealHash := clique.SealHash(header).Bytes()
log.Info("test signing of clique block",
"Sealhash", fmt.Sprintf("%x", sealHash),
"signature", fmt.Sprintf("%x", signature))
pubkey, err := crypto.Ecrecover(sealHash, signature)
if err != nil {
return common.Address{}, err
}
var signer common.Address
copy(signer[:], crypto.Keccak256(pubkey[1:])[12:])
return signer, nil
}
// PrintBlock retrieves a block and returns its pretty printed form.
func (api *PublicDebugAPI) PrintBlock(ctx context.Context, number uint64) (string, error) {
block, _ := api.b.BlockByNumber(ctx, rpc.BlockNumber(number))

4
internal/jsre/jsre.go
View file

@ -32,8 +32,8 @@ import (
)
var (
BigNumber_JS = deps.MustAsset("bignumber.js")
Web3_JS = deps.MustAsset("web3.js")
BignumberJs = deps.MustAsset("bignumber.js")
Web3Js = deps.MustAsset("web3.js")
)
/*

62
internal/web3ext/web3ext.go
View file

@ -18,23 +18,23 @@
package web3ext
var Modules = map[string]string{
"accounting": Accounting_JS,
"admin": Admin_JS,
"chequebook": Chequebook_JS,
"clique": Clique_JS,
"ethash": Ethash_JS,
"debug": Debug_JS,
"eth": Eth_JS,
"miner": Miner_JS,
"net": Net_JS,
"personal": Personal_JS,
"rpc": RPC_JS,
"shh": Shh_JS,
"swarmfs": SWARMFS_JS,
"txpool": TxPool_JS,
"accounting": AccountingJs,
"admin": AdminJs,
"chequebook": ChequebookJs,
"clique": CliqueJs,
"ethash": EthashJs,
"debug": DebugJs,
"eth": EthJs,
"miner": MinerJs,
"net": NetJs,
"personal": PersonalJs,
"rpc": RpcJs,
"shh": ShhJs,
"swarmfs": SwarmfsJs,
"txpool": TxpoolJs,
}
const Chequebook_JS = `
const ChequebookJs = `
web3._extend({
property: 'chequebook',
methods: [
@ -65,7 +65,7 @@ web3._extend({
});
`
const Clique_JS = `
const CliqueJs = `
web3._extend({
property: 'clique',
methods: [
@ -111,7 +111,7 @@ web3._extend({
});
`
const Ethash_JS = `
const EthashJs = `
web3._extend({
property: 'ethash',
methods: [
@ -139,7 +139,7 @@ web3._extend({
});
`
const Admin_JS = `
const AdminJs = `
web3._extend({
property: 'admin',
methods: [
@ -217,7 +217,7 @@ web3._extend({
});
`
const Debug_JS = `
const DebugJs = `
web3._extend({
property: 'debug',
methods: [
@ -231,6 +231,12 @@ web3._extend({
call: 'debug_getBlockRlp',
params: 1
}),
new web3._extend.Method({
name: 'testSignCliqueBlock',
call: 'debug_testSignCliqueBlock',
params: 2,
inputFormatters: [web3._extend.formatters.inputAddressFormatter, null],
}),
new web3._extend.Method({
name: 'setHead',
call: 'debug_setHead',
@ -448,7 +454,7 @@ web3._extend({
});
`
const Eth_JS = `
const EthJs = `
web3._extend({
property: 'eth',
methods: [
@ -518,7 +524,7 @@ web3._extend({
});
`
const Miner_JS = `
const MinerJs = `
web3._extend({
property: 'miner',
methods: [
@ -563,7 +569,7 @@ web3._extend({
});
`
const Net_JS = `
const NetJs = `
web3._extend({
property: 'net',
methods: [],
@ -576,7 +582,7 @@ web3._extend({
});
`
const Personal_JS = `
const PersonalJs = `
web3._extend({
property: 'personal',
methods: [
@ -622,7 +628,7 @@ web3._extend({
})
`
const RPC_JS = `
const RpcJs = `
web3._extend({
property: 'rpc',
methods: [],
@ -635,7 +641,7 @@ web3._extend({
});
`
const Shh_JS = `
const ShhJs = `
web3._extend({
property: 'shh',
methods: [
@ -655,7 +661,7 @@ web3._extend({
});
`
const SWARMFS_JS = `
const SwarmfsJs = `
web3._extend({
property: 'swarmfs',
methods:
@ -679,7 +685,7 @@ web3._extend({
});
`
const TxPool_JS = `
const TxpoolJs = `
web3._extend({
property: 'txpool',
methods: [],
@ -706,7 +712,7 @@ web3._extend({
});
`
const Accounting_JS = `
const AccountingJs = `
web3._extend({
property: 'accounting',
methods: [

2
light/odr_util.go
View file

@ -28,7 +28,7 @@ import (
"github.com/ethereum/go-ethereum/rlp"
)
var sha3_nil = crypto.Keccak256Hash(nil)
var sha3Nil = crypto.Keccak256Hash(nil)
func GetHeaderByNumber(ctx context.Context, odr OdrBackend, number uint64) (*types.Header, error) {
db := odr.Database()

2
light/trie.go
View file

@ -67,7 +67,7 @@ func (db *odrDatabase) CopyTrie(t state.Trie) state.Trie {
}
func (db *odrDatabase) ContractCode(addrHash, codeHash common.Hash) ([]byte, error) {
if codeHash == sha3_nil {
if codeHash == sha3Nil {
return nil, nil
}
if code, err := db.backend.Database().Get(codeHash[:]); err == nil {

2
metrics/metrics.go
View file

@ -19,7 +19,7 @@ import (
//
// This global kill-switch helps quantify the observer effect and makes
// for less cluttered pprof profiles.
var Enabled bool = false
var Enabled = false
// MetricsEnabledFlag is the CLI flag name to use to enable metrics collections.
const MetricsEnabledFlag = "metrics"

2
metrics/opentsdb.go
View file

@ -10,7 +10,7 @@ import (
"time"
)
var shortHostName string = ""
var shortHostName = ""
// OpenTSDBConfig provides a container with configuration parameters for
// the OpenTSDB exporter

2
node/config.go
View file

@ -466,7 +466,7 @@ func makeAccountManager(conf *Config) (*accounts.Manager, string, error) {
return nil, "", err
}
// Assemble the account manager and supported backends
backends := []accounts.Backend{}
var backends []accounts.Backend
if len(conf.ExternalSigner) > 0 {
log.Info("Using external signer", "url", conf.ExternalSigner)
if extapi, err := external.NewExternalBackend(conf.ExternalSigner); err == nil {

2
node/node.go
View file

@ -220,7 +220,7 @@ func (n *Node) Start() error {
return convertFileLockError(err)
}
// Start each of the services
started := []reflect.Type{}
var started []reflect.Type
for kind, service := range services {
// Start the next service, stopping all previous upon failure
if err := service.Start(running); err != nil {

43
p2p/simulations/connect.go
View file

@ -32,6 +32,9 @@ var (
// It is useful when constructing a chain network topology
// when Network adds and removes nodes dynamically.
func (net *Network) ConnectToLastNode(id enode.ID) (err error) {
net.lock.Lock()
defer net.lock.Unlock()
ids := net.getUpNodeIDs()
l := len(ids)
if l < 2 {
@ -41,29 +44,35 @@ func (net *Network) ConnectToLastNode(id enode.ID) (err error) {
if last == id {
last = ids[l-2]
}
return net.connect(last, id)
return net.connectNotConnected(last, id)
}
// ConnectToRandomNode connects the node with provided NodeID
// to a random node that is up.
func (net *Network) ConnectToRandomNode(id enode.ID) (err error) {
selected := net.GetRandomUpNode(id)
net.lock.Lock()
defer net.lock.Unlock()
selected := net.getRandomUpNode(id)
if selected == nil {
return ErrNodeNotFound
}
return net.connect(selected.ID(), id)
return net.connectNotConnected(selected.ID(), id)
}
// ConnectNodesFull connects all nodes one to another.
// It provides a complete connectivity in the network
// which should be rarely needed.
func (net *Network) ConnectNodesFull(ids []enode.ID) (err error) {
net.lock.Lock()
defer net.lock.Unlock()
if ids == nil {
ids = net.getUpNodeIDs()
}
for i, lid := range ids {
for _, rid := range ids[i+1:] {
if err = net.connect(lid, rid); err != nil {
if err = net.connectNotConnected(lid, rid); err != nil {
return err
}
}
@ -74,12 +83,19 @@ func (net *Network) ConnectNodesFull(ids []enode.ID) (err error) {
// ConnectNodesChain connects all nodes in a chain topology.
// If ids argument is nil, all nodes that are up will be connected.
func (net *Network) ConnectNodesChain(ids []enode.ID) (err error) {
net.lock.Lock()
defer net.lock.Unlock()
return net.connectNodesChain(ids)
}
func (net *Network) connectNodesChain(ids []enode.ID) (err error) {
if ids == nil {
ids = net.getUpNodeIDs()
}
l := len(ids)
for i := 0; i < l-1; i++ {
if err := net.connect(ids[i], ids[i+1]); err != nil {
if err := net.connectNotConnected(ids[i], ids[i+1]); err != nil {
return err
}
}
@ -89,6 +105,9 @@ func (net *Network) ConnectNodesChain(ids []enode.ID) (err error) {
// ConnectNodesRing connects all nodes in a ring topology.
// If ids argument is nil, all nodes that are up will be connected.
func (net *Network) ConnectNodesRing(ids []enode.ID) (err error) {
net.lock.Lock()
defer net.lock.Unlock()
if ids == nil {
ids = net.getUpNodeIDs()
}
@ -96,15 +115,18 @@ func (net *Network) ConnectNodesRing(ids []enode.ID) (err error) {
if l < 2 {
return nil
}
if err := net.ConnectNodesChain(ids); err != nil {
if err := net.connectNodesChain(ids); err != nil {
return err
}
return net.connect(ids[l-1], ids[0])
return net.connectNotConnected(ids[l-1], ids[0])
}
// ConnectNodesStar connects all nodes into a star topology
// If ids argument is nil, all nodes that are up will be connected.
func (net *Network) ConnectNodesStar(ids []enode.ID, center enode.ID) (err error) {
net.lock.Lock()
defer net.lock.Unlock()
if ids == nil {
ids = net.getUpNodeIDs()
}
@ -112,16 +134,15 @@ func (net *Network) ConnectNodesStar(ids []enode.ID, center enode.ID) (err error
if center == id {
continue
}
if err := net.connect(center, id); err != nil {
if err := net.connectNotConnected(center, id); err != nil {
return err
}
}
return nil
}
// connect connects two nodes but ignores already connected error.
func (net *Network) connect(oneID, otherID enode.ID) error {
return ignoreAlreadyConnectedErr(net.Connect(oneID, otherID))
func (net *Network) connectNotConnected(oneID, otherID enode.ID) error {
return ignoreAlreadyConnectedErr(net.connect(oneID, otherID))
}
func ignoreAlreadyConnectedErr(err error) error {

2
p2p/simulations/events.go
View file

@ -100,7 +100,7 @@ func ControlEvent(v interface{}) *Event {
func (e *Event) String() string {
switch e.Type {
case EventTypeNode:
return fmt.Sprintf("<node-event> id: %s up: %t", e.Node.ID().TerminalString(), e.Node.Up)
return fmt.Sprintf("<node-event> id: %s up: %t", e.Node.ID().TerminalString(), e.Node.Up())
case EventTypeConn:
return fmt.Sprintf("<conn-event> nodes: %s->%s up: %t", e.Conn.One.TerminalString(), e.Conn.Other.TerminalString(), e.Conn.Up)
case EventTypeMsg:

18
p2p/simulations/http_test.go
View file

@ -421,14 +421,15 @@ type expectEvents struct {
}
func (t *expectEvents) nodeEvent(id string, up bool) *Event {
node := Node{
Config: &adapters.NodeConfig{
ID: enode.HexID(id),
},
up: up,
}
return &Event{
Type: EventTypeNode,
Node: &Node{
Config: &adapters.NodeConfig{
ID: enode.HexID(id),
},
Up: up,
},
Node: &node,
}
}
@ -480,6 +481,7 @@ loop:
}
func (t *expectEvents) expect(events ...*Event) {
t.Helper()
timeout := time.After(10 * time.Second)
i := 0
for {
@ -501,8 +503,8 @@ func (t *expectEvents) expect(events ...*Event) {
if event.Node.ID() != expected.Node.ID() {
t.Fatalf("expected node event %d to have id %q, got %q", i, expected.Node.ID().TerminalString(), event.Node.ID().TerminalString())
}
if event.Node.Up != expected.Node.Up {
t.Fatalf("expected node event %d to have up=%t, got up=%t", i, expected.Node.Up, event.Node.Up)
if event.Node.Up() != expected.Node.Up() {
t.Fatalf("expected node event %d to have up=%t, got up=%t", i, expected.Node.Up(), event.Node.Up())
}
case EventTypeConn:

9
p2p/simulations/mocker_test.go
View file

@ -90,15 +90,12 @@ func TestMocker(t *testing.T) {
for {
select {
case event := <-events:
//if the event is a node Up event only
if event.Node != nil && event.Node.Up {
if isNodeUp(event) {
//add the correspondent node ID to the map
nodemap[event.Node.Config.ID] = true
//this means all nodes got a nodeUp event, so we can continue the test
if len(nodemap) == nodeCount {
nodesComplete = true
//wait for 3s as the mocker will need time to connect the nodes
//time.Sleep( 3 *time.Second)
}
} else if event.Conn != nil && nodesComplete {
connCount += 1
@ -169,3 +166,7 @@ func TestMocker(t *testing.T) {
t.Fatalf("Expected empty list of nodes, got: %d", len(nodesInfo))
}
}
func isNodeUp(event *Event) bool {
return event.Node != nil && event.Node.Up()
}

159
p2p/simulations/network.go
View file

@ -137,7 +137,7 @@ func (net *Network) Config() *NetworkConfig {
// StartAll starts all nodes in the network
func (net *Network) StartAll() error {
for _, node := range net.Nodes {
if node.Up {
if node.Up() {
continue
}
if err := net.Start(node.ID()); err != nil {
@ -150,7 +150,7 @@ func (net *Network) StartAll() error {
// StopAll stops all nodes in the network
func (net *Network) StopAll() error {
for _, node := range net.Nodes {
if !node.Up {
if !node.Up() {
continue
}
if err := net.Stop(node.ID()); err != nil {
@ -169,27 +169,23 @@ func (net *Network) Start(id enode.ID) error {
// snapshots
func (net *Network) startWithSnapshots(id enode.ID, snapshots map[string][]byte) error {
net.lock.Lock()
defer net.lock.Unlock()
node := net.getNode(id)
if node == nil {
net.lock.Unlock()
return fmt.Errorf("node %v does not exist", id)
}
if node.Up {
net.lock.Unlock()
if node.Up() {
return fmt.Errorf("node %v already up", id)
}
log.Trace("Starting node", "id", id, "adapter", net.nodeAdapter.Name())
if err := node.Start(snapshots); err != nil {
net.lock.Unlock()
log.Warn("Node startup failed", "id", id, "err", err)
return err
}
node.Up = true
node.SetUp(true)
log.Info("Started node", "id", id)
ev := NewEvent(node)
net.lock.Unlock()
net.events.Send(ev)
// subscribe to peer events
@ -220,7 +216,7 @@ func (net *Network) watchPeerEvents(id enode.ID, events chan *p2p.PeerEvent, sub
if node == nil {
return
}
node.Up = false
node.SetUp(false)
ev := NewEvent(node)
net.events.Send(ev)
}()
@ -258,30 +254,42 @@ func (net *Network) watchPeerEvents(id enode.ID, events chan *p2p.PeerEvent, sub
// Stop stops the node with the given ID
func (net *Network) Stop(id enode.ID) error {
net.lock.Lock()
node := net.getNode(id)
if node == nil {
net.lock.Unlock()
return fmt.Errorf("node %v does not exist", id)
}
if !node.Up {
net.lock.Unlock()
return fmt.Errorf("node %v already down", id)
}
node.Up = false
net.lock.Unlock()
// IMPORTANT: node.Stop() must NOT be called under net.lock as
// node.Reachable() closure has a reference to the network and
// calls net.InitConn() what also locks the network. => DEADLOCK
// That holds until the following ticket is not resolved:
err := node.Stop()
if err != nil {
var err error
node, err := func() (*Node, error) {
net.lock.Lock()
node.Up = true
net.lock.Unlock()
defer net.lock.Unlock()
node := net.getNode(id)
if node == nil {
return nil, fmt.Errorf("node %v does not exist", id)
}
if !node.Up() {
return nil, fmt.Errorf("node %v already down", id)
}
node.SetUp(false)
return node, nil
}()
if err != nil {
return err
}
err = node.Stop() // must be called without net.lock
net.lock.Lock()
defer net.lock.Unlock()
if err != nil {
node.SetUp(true)
return err
}
log.Info("Stopped node", "id", id, "err", err)
net.lock.Lock()
ev := ControlEvent(node)
net.lock.Unlock()
net.events.Send(ev)
return nil
}
@ -289,8 +297,14 @@ func (net *Network) Stop(id enode.ID) error {
// Connect connects two nodes together by calling the "admin_addPeer" RPC
// method on the "one" node so that it connects to the "other" node
func (net *Network) Connect(oneID, otherID enode.ID) error {
net.lock.Lock()
defer net.lock.Unlock()
return net.connect(oneID, otherID)
}
func (net *Network) connect(oneID, otherID enode.ID) error {
log.Debug("Connecting nodes with addPeer", "id", oneID, "other", otherID)
conn, err := net.InitConn(oneID, otherID)
conn, err := net.initConn(oneID, otherID)
if err != nil {
return err
}
@ -388,6 +402,14 @@ func (net *Network) GetNode(id enode.ID) *Node {
return net.getNode(id)
}
func (net *Network) getNode(id enode.ID) *Node {
i, found := net.nodeMap[id]
if !found {
return nil
}
return net.Nodes[i]
}
// GetNode gets the node with the given name, returning nil if the node does
// not exist
func (net *Network) GetNodeByName(name string) *Node {
@ -410,28 +432,29 @@ func (net *Network) GetNodes() (nodes []*Node) {
net.lock.RLock()
defer net.lock.RUnlock()
nodes = append(nodes, net.Nodes...)
return nodes
return net.getNodes()
}
func (net *Network) getNode(id enode.ID) *Node {
i, found := net.nodeMap[id]
if !found {
return nil
}
return net.Nodes[i]
func (net *Network) getNodes() (nodes []*Node) {
nodes = append(nodes, net.Nodes...)
return nodes
}
// GetRandomUpNode returns a random node on the network, which is running.
func (net *Network) GetRandomUpNode(excludeIDs ...enode.ID) *Node {
net.lock.RLock()
defer net.lock.RUnlock()
return net.getRandomUpNode(excludeIDs...)
}
// GetRandomUpNode returns a random node on the network, which is running.
func (net *Network) getRandomUpNode(excludeIDs ...enode.ID) *Node {
return net.getRandomNode(net.getUpNodeIDs(), excludeIDs)
}
func (net *Network) getUpNodeIDs() (ids []enode.ID) {
for _, node := range net.Nodes {
if node.Up {
if node.Up() {
ids = append(ids, node.ID())
}
}
@ -446,8 +469,8 @@ func (net *Network) GetRandomDownNode(excludeIDs ...enode.ID) *Node {
}
func (net *Network) getDownNodeIDs() (ids []enode.ID) {
for _, node := range net.GetNodes() {
if !node.Up {
for _, node := range net.getNodes() {
if !node.Up() {
ids = append(ids, node.ID())
}
}
@ -461,7 +484,7 @@ func (net *Network) getRandomNode(ids []enode.ID, excludeIDs []enode.ID) *Node {
if l == 0 {
return nil
}
return net.GetNode(filtered[rand.Intn(l)])
return net.getNode(filtered[rand.Intn(l)])
}
func filterIDs(ids []enode.ID, excludeIDs []enode.ID) []enode.ID {
@ -539,6 +562,10 @@ func (net *Network) getConn(oneID, otherID enode.ID) *Conn {
func (net *Network) InitConn(oneID, otherID enode.ID) (*Conn, error) {
net.lock.Lock()
defer net.lock.Unlock()
return net.initConn(oneID, otherID)
}
func (net *Network) initConn(oneID, otherID enode.ID) (*Conn, error) {
if oneID == otherID {
return nil, fmt.Errorf("refusing to connect to self %v", oneID)
}
@ -602,8 +629,21 @@ type Node struct {
// Config if the config used to created the node
Config *adapters.NodeConfig `json:"config"`
// Up tracks whether or not the node is running
Up bool `json:"up"`
// up tracks whether or not the node is running
up bool
upMu sync.RWMutex
}
func (n *Node) Up() bool {
n.upMu.RLock()
defer n.upMu.RUnlock()
return n.up
}
func (n *Node) SetUp(up bool) {
n.upMu.Lock()
defer n.upMu.Unlock()
n.up = up
}
// ID returns the ID of the node
@ -637,10 +677,29 @@ func (n *Node) MarshalJSON() ([]byte, error) {
}{
Info: n.NodeInfo(),
Config: n.Config,
Up: n.Up,
Up: n.Up(),
})
}
// UnmarshalJSON implements json.Unmarshaler interface so that we don't lose
// Node.up status. IMPORTANT: The implementation is incomplete; we lose
// p2p.NodeInfo.
func (n *Node) UnmarshalJSON(raw []byte) error {
// TODO: How should we turn back NodeInfo into n.Node?
// Ticket: https://github.com/ethersphere/go-ethereum/issues/1177
node := struct {
Config *adapters.NodeConfig `json:"config,omitempty"`
Up bool `json:"up"`
}{}
if err := json.Unmarshal(raw, &node); err != nil {
return err
}
n.SetUp(node.Up)
n.Config = node.Config
return nil
}
// Conn represents a connection between two nodes in the network
type Conn struct {
// One is the node which initiated the connection
@ -660,10 +719,10 @@ type Conn struct {
// nodesUp returns whether both nodes are currently up
func (c *Conn) nodesUp() error {
if !c.one.Up {
if !c.one.Up() {
return fmt.Errorf("one %v is not up", c.One)
}
if !c.other.Up {
if !c.other.Up() {
return fmt.Errorf("other %v is not up", c.Other)
}
return nil
@ -735,7 +794,7 @@ func (net *Network) snapshot(addServices []string, removeServices []string) (*Sn
}
for i, node := range net.Nodes {
snap.Nodes[i] = NodeSnapshot{Node: *node}
if !node.Up {
if !node.Up() {
continue
}
snapshots, err := node.Snapshots()
@ -790,7 +849,7 @@ func (net *Network) Load(snap *Snapshot) error {
if _, err := net.NewNodeWithConfig(n.Node.Config); err != nil {
return err
}
if !n.Node.Up {
if !n.Node.Up() {
continue
}
if err := net.startWithSnapshots(n.Node.Config.ID, n.Snapshots); err != nil {
@ -862,7 +921,7 @@ func (net *Network) Load(snap *Snapshot) error {
// Start connecting.
for _, conn := range snap.Conns {
if !net.GetNode(conn.One).Up || !net.GetNode(conn.Other).Up {
if !net.GetNode(conn.One).Up() || !net.GetNode(conn.Other).Up() {
//in this case, at least one of the nodes of a connection is not up,
//so it would result in the snapshot `Load` to fail
continue
@ -916,7 +975,7 @@ func (net *Network) executeControlEvent(event *Event) {
}
func (net *Network) executeNodeEvent(e *Event) error {
if !e.Node.Up {
if !e.Node.Up() {
return net.Stop(e.Node.ID())
}

135
p2p/simulations/network_test.go
View file

@ -20,6 +20,7 @@ import (
"context"
"encoding/json"
"fmt"
"reflect"
"strconv"
"strings"
"testing"
@ -485,3 +486,137 @@ func benchmarkMinimalServiceTmp(b *testing.B) {
}
}
}
func TestNode_UnmarshalJSON(t *testing.T) {
t.Run(
"test unmarshal of Node up field",
func(t *testing.T) {
runNodeUnmarshalJSON(t, casesNodeUnmarshalJSONUpField())
},
)
t.Run(
"test unmarshal of Node Config field",
func(t *testing.T) {
runNodeUnmarshalJSON(t, casesNodeUnmarshalJSONConfigField())
},
)
}
func runNodeUnmarshalJSON(t *testing.T, tests []nodeUnmarshalTestCase) {
t.Helper()
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var got Node
if err := got.UnmarshalJSON([]byte(tt.marshaled)); err != nil {
expectErrorMessageToContain(t, err, tt.wantErr)
}
expectNodeEquality(t, got, tt.want)
})
}
}
type nodeUnmarshalTestCase struct {
name string
marshaled string
want Node
wantErr string
}
func expectErrorMessageToContain(t *testing.T, got error, want string) {
t.Helper()
if got == nil && want == "" {
return
}
if got == nil && want != "" {
t.Errorf("error was expected, got: nil, want: %v", want)
return
}
if !strings.Contains(got.Error(), want) {
t.Errorf(
"unexpected error message, got %v, want: %v",
want,
got,
)
}
}
func expectNodeEquality(t *testing.T, got Node, want Node) {
t.Helper()
if !reflect.DeepEqual(got, want) {
t.Errorf("Node.UnmarshalJSON() = %v, want %v", got, want)
}
}
func casesNodeUnmarshalJSONUpField() []nodeUnmarshalTestCase {
return []nodeUnmarshalTestCase{
{
name: "empty json",
marshaled: "{}",
want: Node{
up: false,
},
},
{
name: "a stopped node",
marshaled: "{\"up\": false}",
want: Node{
up: false,
},
},
{
name: "a running node",
marshaled: "{\"up\": true}",
want: Node{
up: true,
},
},
{
name: "invalid JSON value on valid key",
marshaled: "{\"up\": foo}",
wantErr: "invalid character",
},
{
name: "invalid JSON key and value",
marshaled: "{foo: bar}",
wantErr: "invalid character",
},
{
name: "bool value expected but got something else (string)",
marshaled: "{\"up\": \"true\"}",
wantErr: "cannot unmarshal string into Go struct",
},
}
}
func casesNodeUnmarshalJSONConfigField() []nodeUnmarshalTestCase {
// Don't do a big fuss around testing, as adapters.NodeConfig should
// handle it's own serialization. Just do a sanity check.
return []nodeUnmarshalTestCase{
{
name: "Config field is omitted",
marshaled: "{}",
want: Node{
Config: nil,
},
},
{
name: "Config field is nil",
marshaled: "{\"config\": nil}",
want: Node{
Config: nil,
},
},
{
name: "a non default Config field",
marshaled: "{\"config\":{\"name\":\"node_ecdd0\",\"port\":44665}}",
want: Node{
Config: &adapters.NodeConfig{
Name: "node_ecdd0",
Port: 44665,
},
},
},
}
}

20
rpc/stdio.go
View file

@ -19,6 +19,7 @@ package rpc
import (
"context"
"errors"
"io"
"net"
"os"
"time"
@ -26,19 +27,30 @@ import (
// DialStdIO creates a client on stdin/stdout.
func DialStdIO(ctx context.Context) (*Client, error) {
return DialIO(ctx, os.Stdin, os.Stdout)
}
// DialIO creates a client which uses the given IO channels
func DialIO(ctx context.Context, in io.Reader, out io.Writer) (*Client, error) {
return newClient(ctx, func(_ context.Context) (ServerCodec, error) {
return NewJSONCodec(stdioConn{}), nil
return NewJSONCodec(stdioConn{
in: in,
out: out,
}), nil
})
}
type stdioConn struct{}
type stdioConn struct {
in io.Reader
out io.Writer
}
func (io stdioConn) Read(b []byte) (n int, err error) {
return os.Stdin.Read(b)
return io.in.Read(b)
}
func (io stdioConn) Write(b []byte) (n int, err error) {
return os.Stdout.Write(b)
return io.out.Write(b)
}
func (io stdioConn) Close() error {

169
signer/core/api.go
View file

@ -21,7 +21,6 @@ import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"math/big"
"reflect"
@ -39,9 +38,9 @@ const (
// numberOfAccountsToDerive For hardware wallets, the number of accounts to derive
numberOfAccountsToDerive = 10
// ExternalAPIVersion -- see extapi_changelog.md
ExternalAPIVersion = "5.0.0"
ExternalAPIVersion = "6.0.0"
// InternalAPIVersion -- see intapi_changelog.md
InternalAPIVersion = "3.1.0"
InternalAPIVersion = "4.0.0"
)
// ExternalAPI defines the external API through which signing requests are made.
@ -49,7 +48,7 @@ type ExternalAPI interface {
// List available accounts
List(ctx context.Context) ([]common.Address, error)
// New request to create a new account
New(ctx context.Context) (accounts.Account, error)
New(ctx context.Context) (common.Address, error)
// SignTransaction request to sign the specified transaction
SignTransaction(ctx context.Context, args SendTxArgs, methodSelector *string) (*ethapi.SignTransactionResult, error)
// SignData - request to sign the given data (plus prefix)
@ -58,17 +57,13 @@ type ExternalAPI interface {
SignTypedData(ctx context.Context, addr common.MixedcaseAddress, data TypedData) (hexutil.Bytes, error)
// EcRecover - recover public key from given message and signature
EcRecover(ctx context.Context, data hexutil.Bytes, sig hexutil.Bytes) (common.Address, error)
// Export - request to export an account
Export(ctx context.Context, addr common.Address) (json.RawMessage, error)
// Import - request to import an account
// Should be moved to Internal API, in next phase when we have
// bi-directional communication
//Import(ctx context.Context, keyJSON json.RawMessage) (Account, error)
// Version info about the APIs
Version(ctx context.Context) (string, error)
}
// SignerUI specifies what method a UI needs to implement to be able to be used as a UI for the signer
type SignerUI interface {
// UIClientAPI specifies what method a UI needs to implement to be able to be used as a
// UI for the signer
type UIClientAPI interface {
// ApproveTx prompt the user for confirmation to request to sign Transaction
ApproveTx(request *SignTxRequest) (SignTxResponse, error)
// ApproveSignData prompt the user for confirmation to request to sign data
@ -95,13 +90,15 @@ type SignerUI interface {
// OnInputRequired is invoked when clef requires user input, for example master password or
// pin-code for unlocking hardware wallets
OnInputRequired(info UserInputRequest) (UserInputResponse, error)
// RegisterUIServer tells the UI to use the given UIServerAPI for ui->clef communication
RegisterUIServer(api *UIServerAPI)
}
// SignerAPI defines the actual implementation of ExternalAPI
type SignerAPI struct {
chainID *big.Int
am *accounts.Manager
UI SignerUI
UI UIClientAPI
validator *Validator
rejectMode bool
}
@ -115,6 +112,37 @@ type Metadata struct {
Origin string `json:"Origin"`
}
func StartClefAccountManager(ksLocation string, nousb, lightKDF bool) *accounts.Manager {
var (
backends []accounts.Backend
n, p = keystore.StandardScryptN, keystore.StandardScryptP
)
if lightKDF {
n, p = keystore.LightScryptN, keystore.LightScryptP
}
// support password based accounts
if len(ksLocation) > 0 {
backends = append(backends, keystore.NewKeyStore(ksLocation, n, p))
}
if !nousb {
// Start a USB hub for Ledger hardware wallets
if ledgerhub, err := usbwallet.NewLedgerHub(); err != nil {
log.Warn(fmt.Sprintf("Failed to start Ledger hub, disabling: %v", err))
} else {
backends = append(backends, ledgerhub)
log.Debug("Ledger support enabled")
}
// Start a USB hub for Trezor hardware wallets
if trezorhub, err := usbwallet.NewTrezorHub(); err != nil {
log.Warn(fmt.Sprintf("Failed to start Trezor hub, disabling: %v", err))
} else {
backends = append(backends, trezorhub)
log.Debug("Trezor support enabled")
}
}
return accounts.NewManager(backends...)
}
// MetadataFromContext extracts Metadata from a given context.Context
func MetadataFromContext(ctx context.Context) Metadata {
m := Metadata{"NA", "NA", "NA", "", ""} // batman
@ -199,21 +227,15 @@ type (
Password string `json:"password"`
}
ListRequest struct {
Accounts []Account `json:"accounts"`
Meta Metadata `json:"meta"`
Accounts []accounts.Account `json:"accounts"`
Meta Metadata `json:"meta"`
}
ListResponse struct {
Accounts []Account `json:"accounts"`
Accounts []accounts.Account `json:"accounts"`
}
Message struct {
Text string `json:"text"`
}
PasswordRequest struct {
Prompt string `json:"prompt"`
}
PasswordResponse struct {
Password string `json:"password"`
}
StartupInfo struct {
Info map[string]interface{} `json:"info"`
}
@ -234,38 +256,11 @@ var ErrRequestDenied = errors.New("Request denied")
// key that is generated when a new Account is created.
// noUSB disables USB support that is required to support hardware devices such as
// ledger and trezor.
func NewSignerAPI(chainID int64, ksLocation string, noUSB bool, ui SignerUI, abidb *AbiDb, lightKDF bool, advancedMode bool) *SignerAPI {
var (
backends []accounts.Backend
n, p = keystore.StandardScryptN, keystore.StandardScryptP
)
if lightKDF {
n, p = keystore.LightScryptN, keystore.LightScryptP
}
// support password based accounts
if len(ksLocation) > 0 {
backends = append(backends, keystore.NewKeyStore(ksLocation, n, p))
}
func NewSignerAPI(am *accounts.Manager, chainID int64, noUSB bool, ui UIClientAPI, abidb *AbiDb, advancedMode bool) *SignerAPI {
if advancedMode {
log.Info("Clef is in advanced mode: will warn instead of reject")
}
if !noUSB {
// Start a USB hub for Ledger hardware wallets
if ledgerhub, err := usbwallet.NewLedgerHub(); err != nil {
log.Warn(fmt.Sprintf("Failed to start Ledger hub, disabling: %v", err))
} else {
backends = append(backends, ledgerhub)
log.Debug("Ledger support enabled")
}
// Start a USB hub for Trezor hardware wallets
if trezorhub, err := usbwallet.NewTrezorHub(); err != nil {
log.Warn(fmt.Sprintf("Failed to start Trezor hub, disabling: %v", err))
} else {
backends = append(backends, trezorhub)
log.Debug("Trezor support enabled")
}
}
signer := &SignerAPI{big.NewInt(chainID), accounts.NewManager(backends...), ui, NewValidator(abidb), !advancedMode}
signer := &SignerAPI{big.NewInt(chainID), am, ui, NewValidator(abidb), !advancedMode}
if !noUSB {
signer.startUSBListener()
}
@ -358,12 +353,9 @@ func (api *SignerAPI) startUSBListener() {
// List returns the set of wallet this signer manages. Each wallet can contain
// multiple accounts.
func (api *SignerAPI) List(ctx context.Context) ([]common.Address, error) {
var accs []Account
var accs []accounts.Account
for _, wallet := range api.am.Wallets() {
for _, acc := range wallet.Accounts() {
acc := Account{Typ: "Account", URL: wallet.URL(), Address: acc.Address}
accs = append(accs, acc)
}
accs = append(accs, wallet.Accounts()...)
}
result, err := api.UI.ApproveListing(&ListRequest{Accounts: accs, Meta: MetadataFromContext(ctx)})
if err != nil {
@ -373,7 +365,6 @@ func (api *SignerAPI) List(ctx context.Context) ([]common.Address, error) {
return nil, ErrRequestDenied
}
addresses := make([]common.Address, 0)
for _, acc := range result.Accounts {
addresses = append(addresses, acc.Address)
@ -385,10 +376,10 @@ func (api *SignerAPI) List(ctx context.Context) ([]common.Address, error) {
// New creates a new password protected Account. The private key is protected with
// the given password. Users are responsible to backup the private key that is stored
// in the keystore location thas was specified when this API was created.
func (api *SignerAPI) New(ctx context.Context) (accounts.Account, error) {
func (api *SignerAPI) New(ctx context.Context) (common.Address, error) {
be := api.am.Backends(keystore.KeyStoreType)
if len(be) == 0 {
return accounts.Account{}, errors.New("password based accounts not supported")
return common.Address{}, errors.New("password based accounts not supported")
}
var (
resp NewAccountResponse
@ -398,20 +389,21 @@ func (api *SignerAPI) New(ctx context.Context) (accounts.Account, error) {
for i := 0; i < 3; i++ {
resp, err = api.UI.ApproveNewAccount(&NewAccountRequest{MetadataFromContext(ctx)})
if err != nil {
return accounts.Account{}, err
return common.Address{}, err
}
if !resp.Approved {
return accounts.Account{}, ErrRequestDenied
return common.Address{}, ErrRequestDenied
}
if pwErr := ValidatePasswordFormat(resp.Password); pwErr != nil {
api.UI.ShowError(fmt.Sprintf("Account creation attempt #%d failed due to password requirements: %v", (i + 1), pwErr))
} else {
// No error
return be[0].(*keystore.KeyStore).NewAccount(resp.Password)
acc, err := be[0].(*keystore.KeyStore).NewAccount(resp.Password)
return acc.Address, err
}
}
// Otherwise fail, with generic error message
return accounts.Account{}, errors.New("account creation failed")
return common.Address{}, errors.New("account creation failed")
}
// logDiff logs the difference between the incoming (original) transaction and the one returned from the signer.
@ -521,57 +513,6 @@ func (api *SignerAPI) SignTransaction(ctx context.Context, args SendTxArgs, meth
}
// Export returns encrypted private key associated with the given address in web3 keystore format.
func (api *SignerAPI) Export(ctx context.Context, addr common.Address) (json.RawMessage, error) {
res, err := api.UI.ApproveExport(&ExportRequest{Address: addr, Meta: MetadataFromContext(ctx)})
if err != nil {
return nil, err
}
if !res.Approved {
return nil, ErrRequestDenied
}
// Look up the wallet containing the requested signer
wallet, err := api.am.Find(accounts.Account{Address: addr})
if err != nil {
return nil, err
}
if wallet.URL().Scheme != keystore.KeyStoreScheme {
return nil, fmt.Errorf("Account is not a keystore-account")
}
return ioutil.ReadFile(wallet.URL().Path)
}
// Import tries to import the given keyJSON in the local keystore. The keyJSON data is expected to be
// in web3 keystore format. It will decrypt the keyJSON with the given passphrase and on successful
// decryption it will encrypt the key with the given newPassphrase and store it in the keystore.
// OBS! This method is removed from the public API. It should not be exposed on the external API
// for a couple of reasons:
// 1. Even though it is encrypted, it should still be seen as sensitive data
// 2. It can be used to DoS clef, by using malicious data with e.g. extreme large
// values for the kdfparams.
func (api *SignerAPI) Import(ctx context.Context, keyJSON json.RawMessage) (Account, error) {
be := api.am.Backends(keystore.KeyStoreType)
if len(be) == 0 {
return Account{}, errors.New("password based accounts not supported")
}
res, err := api.UI.ApproveImport(&ImportRequest{Meta: MetadataFromContext(ctx)})
if err != nil {
return Account{}, err
}
if !res.Approved {
return Account{}, ErrRequestDenied
}
acc, err := be[0].(*keystore.KeyStore).Import(keyJSON, res.OldPassword, res.NewPassword)
if err != nil {
api.UI.ShowError(err.Error())
return Account{}, err
}
return Account{Typ: "Account", URL: acc.URL, Address: acc.Address}, nil
}
// Returns the external api version. This method does not require user acceptance. Available methods are
// available via enumeration anyway, and this info does not contain user-specific data
func (api *SignerAPI) Version(ctx context.Context) (string, error) {

22
signer/core/api_test.go
View file

@ -28,6 +28,7 @@ import (
"testing"
"time"
"github.com/ethereum/go-ethereum/accounts"
"github.com/ethereum/go-ethereum/accounts/keystore"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
@ -47,6 +48,8 @@ func (ui *HeadlessUI) OnInputRequired(info UserInputRequest) (UserInputResponse,
func (ui *HeadlessUI) OnSignerStartup(info StartupInfo) {
}
func (ui *HeadlessUI) RegisterUIServer(api *UIServerAPI) {
}
func (ui *HeadlessUI) OnApprovedTx(tx ethapi.SignTransactionResult) {
fmt.Printf("OnApproved()\n")
@ -91,7 +94,7 @@ func (ui *HeadlessUI) ApproveListing(request *ListRequest) (ListResponse, error)
case "A":
return ListResponse{request.Accounts}, nil
case "1":
l := make([]Account, 1)
l := make([]accounts.Account, 1)
l[0] = request.Accounts[1]
return ListResponse{l}, nil
default:
@ -138,13 +141,8 @@ func setup(t *testing.T) (*SignerAPI, chan string) {
}
var (
ui = &HeadlessUI{controller}
api = NewSignerAPI(
1,
tmpDirName(t),
true,
ui,
db,
true, true)
am = StartClefAccountManager(tmpDirName(t), true, true)
api = NewSignerAPI(am, 1337, true, ui, db, true)
)
return api, controller
}
@ -169,22 +167,22 @@ func failCreateAccountWithPassword(control chan string, api *SignerAPI, password
control <- "Y"
control <- password
acc, err := api.New(context.Background())
addr, err := api.New(context.Background())
if err == nil {
t.Fatal("Should have returned an error")
}
if acc.Address != (common.Address{}) {
if addr != (common.Address{}) {
t.Fatal("Empty address should be returned")
}
}
func failCreateAccount(control chan string, api *SignerAPI, t *testing.T) {
control <- "N"
acc, err := api.New(context.Background())
addr, err := api.New(context.Background())
if err != ErrRequestDenied {
t.Fatal(err)
}
if acc.Address != (common.Address{}) {
if addr != (common.Address{}) {
t.Fatal("Empty address should be returned")
}
}

13
signer/core/auditlog.go
View file

@ -18,9 +18,7 @@ package core
import (
"context"
"encoding/json"
"github.com/ethereum/go-ethereum/accounts"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/internal/ethapi"
@ -40,7 +38,7 @@ func (l *AuditLogger) List(ctx context.Context) ([]common.Address, error) {
return res, e
}
func (l *AuditLogger) New(ctx context.Context) (accounts.Account, error) {
func (l *AuditLogger) New(ctx context.Context) (common.Address, error) {
return l.api.New(ctx)
}
@ -86,15 +84,6 @@ func (l *AuditLogger) EcRecover(ctx context.Context, data hexutil.Bytes, sig hex
return b, e
}
func (l *AuditLogger) Export(ctx context.Context, addr common.Address) (json.RawMessage, error) {
l.log.Info("Export", "type", "request", "metadata", MetadataFromContext(ctx).String(),
"addr", addr.Hex())
j, e := l.api.Export(ctx, addr)
// In this case, we don't actually log the json-response, which may be extra sensitive
l.log.Info("Export", "type", "response", "json response size", len(j), "error", e)
return j, e
}
func (l *AuditLogger) Version(ctx context.Context) (string, error) {
l.log.Info("Version", "type", "request", "metadata", MetadataFromContext(ctx).String())
data, err := l.api.Version(ctx)

13
signer/core/cliui.go
View file

@ -18,12 +18,12 @@ package core
import (
"bufio"
"encoding/json"
"fmt"
"os"
"strings"
"sync"
"github.com/davecgh/go-spew/spew"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/internal/ethapi"
"github.com/ethereum/go-ethereum/log"
@ -39,6 +39,10 @@ func NewCommandlineUI() *CommandlineUI {
return &CommandlineUI{in: bufio.NewReader(os.Stdin)}
}
func (ui *CommandlineUI) RegisterUIServer(api *UIServerAPI) {
// noop
}
// readString reads a single line from stdin, trimming if from spaces, enforcing
// non-emptyness.
func (ui *CommandlineUI) readString() string {
@ -223,7 +227,6 @@ func (ui *CommandlineUI) ApproveListing(request *ListRequest) (ListResponse, err
for _, account := range request.Accounts {
fmt.Printf(" [x] %v\n", account.Address.Hex())
fmt.Printf(" URL: %v\n", account.URL)
fmt.Printf(" Type: %v\n", account.Typ)
}
fmt.Printf("-------------------------------------------\n")
showMetadata(request.Meta)
@ -264,7 +267,11 @@ func (ui *CommandlineUI) ShowInfo(message string) {
func (ui *CommandlineUI) OnApprovedTx(tx ethapi.SignTransactionResult) {
fmt.Printf("Transaction signed:\n ")
spew.Dump(tx.Tx)
if jsn, err := json.MarshalIndent(tx.Tx, " ", " "); err != nil {
fmt.Printf("WARN: marshalling error %v\n", err)
} else {
fmt.Println(string(jsn))
}
}
func (ui *CommandlineUI) OnSignerStartup(info StartupInfo) {

104
signer/core/signed_data.go
View file

@ -121,8 +121,8 @@ var typedDataReferenceTypeRegexp = regexp.MustCompile(`^[A-Z](\w*)(\[\])?$`)
// sign receives a request and produces a signature
// Note, the produced signature conforms to the secp256k1 curve R, S and V values,
// where the V value will be 27 or 28 for legacy reasons.
func (api *SignerAPI) sign(addr common.MixedcaseAddress, req *SignDataRequest) (hexutil.Bytes, error) {
// where the V value will be 27 or 28 for legacy reasons, if legacyV==true.
func (api *SignerAPI) sign(addr common.MixedcaseAddress, req *SignDataRequest, legacyV bool) (hexutil.Bytes, error) {
// We make the request prior to looking up if we actually have the account, to prevent
// account-enumeration via the API
@ -140,11 +140,13 @@ func (api *SignerAPI) sign(addr common.MixedcaseAddress, req *SignDataRequest) (
return nil, err
}
// Sign the data with the wallet
signature, err := wallet.SignDataWithPassphrase(account, res.Password, req.ContentType, req.Hash)
signature, err := wallet.SignDataWithPassphrase(account, res.Password, req.ContentType, req.Rawdata)
if err != nil {
return nil, err
}
signature[64] += 27 // Transform V from 0/1 to 27/28 according to the yellow paper
if legacyV {
signature[64] += 27 // Transform V from 0/1 to 27/28 according to the yellow paper
}
return signature, nil
}
@ -153,17 +155,16 @@ func (api *SignerAPI) sign(addr common.MixedcaseAddress, req *SignDataRequest) (
//
// Different types of validation occur.
func (api *SignerAPI) SignData(ctx context.Context, contentType string, addr common.MixedcaseAddress, data interface{}) (hexutil.Bytes, error) {
var req, err = api.determineSignatureFormat(ctx, contentType, addr, data)
var req, transformV, err = api.determineSignatureFormat(ctx, contentType, addr, data)
if err != nil {
return nil, err
}
signature, err := api.sign(addr, req)
signature, err := api.sign(addr, req, transformV)
if err != nil {
api.UI.ShowError(err.Error())
return nil, err
}
return signature, nil
}
@ -173,12 +174,14 @@ func (api *SignerAPI) SignData(ctx context.Context, contentType string, addr com
// charset, ok := params["charset"]
// As it is now, we accept any charset and just treat it as 'raw'.
// This method returns the mimetype for signing along with the request
func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType string, addr common.MixedcaseAddress, data interface{}) (*SignDataRequest, error) {
var req *SignDataRequest
func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType string, addr common.MixedcaseAddress, data interface{}) (*SignDataRequest, bool, error) {
var (
req *SignDataRequest
useEthereumV = true // Default to use V = 27 or 28, the legacy Ethereum format
)
mediaType, _, err := mime.ParseMediaType(contentType)
if err != nil {
return nil, err
return nil, useEthereumV, err
}
switch mediaType {
@ -186,7 +189,7 @@ func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType
// Data with an intended validator
validatorData, err := UnmarshalValidatorData(data)
if err != nil {
return nil, err
return nil, useEthereumV, err
}
sighash, msg := SignTextValidator(validatorData)
message := []*NameValueType{
@ -201,55 +204,63 @@ func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType
// Clique is the Ethereum PoA standard
stringData, ok := data.(string)
if !ok {
return nil, fmt.Errorf("input for %v plain must be an hex-encoded string", ApplicationClique.Mime)
return nil, useEthereumV, fmt.Errorf("input for %v must be an hex-encoded string", ApplicationClique.Mime)
}
cliqueData, err := hexutil.Decode(stringData)
if err != nil {
return nil, err
return nil, useEthereumV, err
}
header := &types.Header{}
if err := rlp.DecodeBytes(cliqueData, header); err != nil {
return nil, err
return nil, useEthereumV, err
}
// The incoming clique header is already truncated, sent to us with a extradata already shortened
if len(header.Extra) < 65 {
// Need to add it back, to get a suitable length for hashing
newExtra := make([]byte, len(header.Extra)+65)
copy(newExtra, header.Extra)
header.Extra = newExtra
}
// Get back the rlp data, encoded by us
cliqueData = clique.CliqueRLP(header)
sighash, err := SignCliqueHeader(header)
sighash, cliqueRlp, err := cliqueHeaderHashAndRlp(header)
if err != nil {
return nil, err
return nil, useEthereumV, err
}
message := []*NameValueType{
{
Name: "Clique block",
Name: "Clique header",
Typ: "clique",
Value: fmt.Sprintf("clique block %d [0x%x]", header.Number, header.Hash()),
Value: fmt.Sprintf("clique header %d [0x%x]", header.Number, header.Hash()),
},
}
req = &SignDataRequest{ContentType: mediaType, Rawdata: cliqueData, Message: message, Hash: sighash}
// Clique uses V on the form 0 or 1
useEthereumV = false
req = &SignDataRequest{ContentType: mediaType, Rawdata: cliqueRlp, Message: message, Hash: sighash}
default: // also case TextPlain.Mime:
// Calculates an Ethereum ECDSA signature for:
// hash = keccak256("\x19${byteVersion}Ethereum Signed Message:\n${message length}${message}")
// We expect it to be a string
stringData, ok := data.(string)
if !ok {
return nil, fmt.Errorf("input for text/plain must be a string")
if stringData, ok := data.(string); !ok {
return nil, useEthereumV, fmt.Errorf("input for text/plain must be an hex-encoded string")
} else {
if textData, err := hexutil.Decode(stringData); err != nil {
return nil, useEthereumV, err
} else {
sighash, msg := accounts.TextAndHash(textData)
message := []*NameValueType{
{
Name: "message",
Typ: accounts.MimetypeTextPlain,
Value: msg,
},
}
req = &SignDataRequest{ContentType: mediaType, Rawdata: []byte(msg), Message: message, Hash: sighash}
}
}
//plainData, err := hexutil.Decode(stringdata)
//if err != nil {
// return nil, err
//}
sighash, msg := accounts.TextAndHash([]byte(stringData))
message := []*NameValueType{
{
Name: "message",
Typ: "text/plain",
Value: msg,
},
}
req = &SignDataRequest{ContentType: mediaType, Rawdata: []byte(msg), Message: message, Hash: sighash}
}
req.Address = addr
req.Meta = MetadataFromContext(ctx)
return req, nil
return req, useEthereumV, nil
}
@ -262,20 +273,21 @@ func SignTextValidator(validatorData ValidatorData) (hexutil.Bytes, string) {
return crypto.Keccak256([]byte(msg)), msg
}
// SignCliqueHeader returns the hash which is used as input for the proof-of-authority
// cliqueHeaderHashAndRlp returns the hash which is used as input for the proof-of-authority
// signing. It is the hash of the entire header apart from the 65 byte signature
// contained at the end of the extra data.
//
// The method requires the extra data to be at least 65 bytes -- the original implementation
// in clique.go panics if this is the case, thus it's been reimplemented here to avoid the panic
// and simply return an error instead
func SignCliqueHeader(header *types.Header) (hexutil.Bytes, error) {
//hash := common.Hash{}
func cliqueHeaderHashAndRlp(header *types.Header) (hash, rlp []byte, err error) {
if len(header.Extra) < 65 {
return nil, fmt.Errorf("clique header extradata too short, %d < 65", len(header.Extra))
err = fmt.Errorf("clique header extradata too short, %d < 65", len(header.Extra))
return
}
hash := clique.SealHash(header)
return hash.Bytes(), nil
rlp = clique.CliqueRLP(header)
hash = clique.SealHash(header).Bytes()
return hash, rlp, err
}
// SignTypedData signs EIP-712 conformant typed data
@ -293,7 +305,7 @@ func (api *SignerAPI) SignTypedData(ctx context.Context, addr common.MixedcaseAd
sighash := crypto.Keccak256(rawData)
message := typedData.Format()
req := &SignDataRequest{ContentType: DataTyped.Mime, Rawdata: rawData, Message: message, Hash: sighash}
signature, err := api.sign(addr, req)
signature, err := api.sign(addr, req, true)
if err != nil {
api.UI.ShowError(err.Error())
return nil, err
@ -722,7 +734,7 @@ func (nvt *NameValueType) Pprint(depth int) string {
output.WriteString(sublevel)
}
} else {
output.WriteString(fmt.Sprintf("%s\n", nvt.Value))
output.WriteString(fmt.Sprintf("%q\n", nvt.Value))
}
return output.String()
}

26
signer/core/stdioui.go
View file

@ -32,12 +32,16 @@ type StdIOUI struct {
}
func NewStdIOUI() *StdIOUI {
log.Info("NewStdIOUI")
client, err := rpc.DialContext(context.Background(), "stdio://")
if err != nil {
log.Crit("Could not create stdio client", "err", err)
}
return &StdIOUI{client: *client}
ui := &StdIOUI{client: *client}
return ui
}
func (ui *StdIOUI) RegisterUIServer(api *UIServerAPI) {
ui.client.RegisterName("clef", api)
}
// dispatch sends a request over the stdio
@ -49,6 +53,16 @@ func (ui *StdIOUI) dispatch(serviceMethod string, args interface{}, reply interf
return err
}
// notify sends a request over the stdio, and does not listen for a response
func (ui *StdIOUI) notify(serviceMethod string, args interface{}) error {
ctx := context.Background()
err := ui.client.Notify(ctx, serviceMethod, args)
if err != nil {
log.Info("Error", "exc", err.Error())
}
return err
}
func (ui *StdIOUI) ApproveTx(request *SignTxRequest) (SignTxResponse, error) {
var result SignTxResponse
err := ui.dispatch("ApproveTx", request, &result)
@ -86,27 +100,27 @@ func (ui *StdIOUI) ApproveNewAccount(request *NewAccountRequest) (NewAccountResp
}
func (ui *StdIOUI) ShowError(message string) {
err := ui.dispatch("ShowError", &Message{message}, nil)
err := ui.notify("ShowError", &Message{message})
if err != nil {
log.Info("Error calling 'ShowError'", "exc", err.Error(), "msg", message)
}
}
func (ui *StdIOUI) ShowInfo(message string) {
err := ui.dispatch("ShowInfo", Message{message}, nil)
err := ui.notify("ShowInfo", Message{message})
if err != nil {
log.Info("Error calling 'ShowInfo'", "exc", err.Error(), "msg", message)
}
}
func (ui *StdIOUI) OnApprovedTx(tx ethapi.SignTransactionResult) {
err := ui.dispatch("OnApprovedTx", tx, nil)
err := ui.notify("OnApprovedTx", tx)
if err != nil {
log.Info("Error calling 'OnApprovedTx'", "exc", err.Error(), "tx", tx)
}
}
func (ui *StdIOUI) OnSignerStartup(info StartupInfo) {
err := ui.dispatch("OnSignerStartup", info, nil)
err := ui.notify("OnSignerStartup", info)
if err != nil {
log.Info("Error calling 'OnSignerStartup'", "exc", err.Error(), "info", info)
}

27
signer/core/types.go
View file

@ -22,36 +22,11 @@ import (
"math/big"
"strings"
"github.com/ethereum/go-ethereum/accounts"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core/types"
)
type Accounts []Account
func (as Accounts) String() string {
var output []string
for _, a := range as {
output = append(output, a.String())
}
return strings.Join(output, "\n")
}
type Account struct {
Typ string `json:"type"`
URL accounts.URL `json:"url"`
Address common.Address `json:"address"`
}
func (a Account) String() string {
s, err := json.Marshal(a)
if err == nil {
return string(s)
}
return err.Error()
}
type ValidationInfo struct {
Typ string `json:"type"`
Message string `json:"message"`
@ -100,7 +75,7 @@ type SendTxArgs struct {
Nonce hexutil.Uint64 `json:"nonce"`
// We accept "data" and "input" for backwards-compatibility reasons.
Data *hexutil.Bytes `json:"data"`
Input *hexutil.Bytes `json:"input"`
Input *hexutil.Bytes `json:"input,omitempty"`
}
func (args SendTxArgs) String() string {

201
signer/core/uiapi.go Normal file
View file

@ -0,0 +1,201 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
//
package core
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"math/big"
"github.com/ethereum/go-ethereum/accounts"
"github.com/ethereum/go-ethereum/accounts/keystore"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/math"
"github.com/ethereum/go-ethereum/crypto"
)
// SignerUIAPI implements methods Clef provides for a UI to query, in the bidirectional communication
// channel.
// This API is considered secure, since a request can only
// ever arrive from the UI -- and the UI is capable of approving any action, thus we can consider these
// requests pre-approved.
// NB: It's very important that these methods are not ever exposed on the external service
// registry.
type UIServerAPI struct {
extApi *SignerAPI
am *accounts.Manager
}
// NewUIServerAPI creates a new UIServerAPI
func NewUIServerAPI(extapi *SignerAPI) *UIServerAPI {
return &UIServerAPI{extapi, extapi.am}
}
// List available accounts. As opposed to the external API definition, this method delivers
// the full Account object and not only Address.
// Example call
// {"jsonrpc":"2.0","method":"clef_listAccounts","params":[], "id":4}
func (s *UIServerAPI) ListAccounts(ctx context.Context) ([]accounts.Account, error) {
var accs []accounts.Account
for _, wallet := range s.am.Wallets() {
accs = append(accs, wallet.Accounts()...)
}
return accs, nil
}
// rawWallet is a JSON representation of an accounts.Wallet interface, with its
// data contents extracted into plain fields.
type rawWallet struct {
URL string `json:"url"`
Status string `json:"status"`
Failure string `json:"failure,omitempty"`
Accounts []accounts.Account `json:"accounts,omitempty"`
}
// ListWallets will return a list of wallets that clef manages
// Example call
// {"jsonrpc":"2.0","method":"clef_listWallets","params":[], "id":5}
func (s *UIServerAPI) ListWallets() []rawWallet {
wallets := make([]rawWallet, 0) // return [] instead of nil if empty
for _, wallet := range s.am.Wallets() {
status, failure := wallet.Status()
raw := rawWallet{
URL: wallet.URL().String(),
Status: status,
Accounts: wallet.Accounts(),
}
if failure != nil {
raw.Failure = failure.Error()
}
wallets = append(wallets, raw)
}
return wallets
}
// DeriveAccount requests a HD wallet to derive a new account, optionally pinning
// it for later reuse.
// Example call
// {"jsonrpc":"2.0","method":"clef_deriveAccount","params":["ledger://","m/44'/60'/0'", false], "id":6}
func (s *UIServerAPI) DeriveAccount(url string, path string, pin *bool) (accounts.Account, error) {
wallet, err := s.am.Wallet(url)
if err != nil {
return accounts.Account{}, err
}
derivPath, err := accounts.ParseDerivationPath(path)
if err != nil {
return accounts.Account{}, err
}
if pin == nil {
pin = new(bool)
}
return wallet.Derive(derivPath, *pin)
}
// fetchKeystore retrives the encrypted keystore from the account manager.
func fetchKeystore(am *accounts.Manager) *keystore.KeyStore {
return am.Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
}
// ImportRawKey stores the given hex encoded ECDSA key into the key directory,
// encrypting it with the passphrase.
// Example call (should fail on password too short)
// {"jsonrpc":"2.0","method":"clef_importRawKey","params":["1111111111111111111111111111111111111111111111111111111111111111","test"], "id":6}
func (s *UIServerAPI) ImportRawKey(privkey string, password string) (accounts.Account, error) {
key, err := crypto.HexToECDSA(privkey)
if err != nil {
return accounts.Account{}, err
}
if err := ValidatePasswordFormat(password); err != nil {
return accounts.Account{}, fmt.Errorf("password requirements not met: %v", err)
}
// No error
return fetchKeystore(s.am).ImportECDSA(key, password)
}
// OpenWallet initiates a hardware wallet opening procedure, establishing a USB
// connection and attempting to authenticate via the provided passphrase. Note,
// the method may return an extra challenge requiring a second open (e.g. the
// Trezor PIN matrix challenge).
// Example
// {"jsonrpc":"2.0","method":"clef_openWallet","params":["ledger://",""], "id":6}
func (s *UIServerAPI) OpenWallet(url string, passphrase *string) error {
wallet, err := s.am.Wallet(url)
if err != nil {
return err
}
pass := ""
if passphrase != nil {
pass = *passphrase
}
return wallet.Open(pass)
}
// ChainId returns the chainid in use for Eip-155 replay protection
// Example call
// {"jsonrpc":"2.0","method":"clef_chainId","params":[], "id":8}
func (s *UIServerAPI) ChainId() math.HexOrDecimal64 {
return (math.HexOrDecimal64)(s.extApi.chainID.Uint64())
}
// SetChainId sets the chain id to use when signing transactions.
// Example call to set Ropsten:
// {"jsonrpc":"2.0","method":"clef_setChainId","params":["3"], "id":8}
func (s *UIServerAPI) SetChainId(id math.HexOrDecimal64) math.HexOrDecimal64 {
s.extApi.chainID = new(big.Int).SetUint64(uint64(id))
return s.ChainId()
}
// Export returns encrypted private key associated with the given address in web3 keystore format.
// Example
// {"jsonrpc":"2.0","method":"clef_export","params":["0x19e7e376e7c213b7e7e7e46cc70a5dd086daff2a"], "id":4}
func (s *UIServerAPI) Export(ctx context.Context, addr common.Address) (json.RawMessage, error) {
// Look up the wallet containing the requested signer
wallet, err := s.am.Find(accounts.Account{Address: addr})
if err != nil {
return nil, err
}
if wallet.URL().Scheme != keystore.KeyStoreScheme {
return nil, fmt.Errorf("Account is not a keystore-account")
}
return ioutil.ReadFile(wallet.URL().Path)
}
// Import tries to import the given keyJSON in the local keystore. The keyJSON data is expected to be
// in web3 keystore format. It will decrypt the keyJSON with the given passphrase and on successful
// decryption it will encrypt the key with the given newPassphrase and store it in the keystore.
// Example (the address in question has privkey `11...11`):
// {"jsonrpc":"2.0","method":"clef_import","params":[{"address":"19e7e376e7c213b7e7e7e46cc70a5dd086daff2a","crypto":{"cipher":"aes-128-ctr","ciphertext":"33e4cd3756091d037862bb7295e9552424a391a6e003272180a455ca2a9fb332","cipherparams":{"iv":"b54b263e8f89c42bb219b6279fba5cce"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"e4ca94644fd30569c1b1afbbc851729953c92637b7fe4bb9840bbb31ffbc64a5"},"mac":"f4092a445c2b21c0ef34f17c9cd0d873702b2869ec5df4439a0c2505823217e7"},"id":"216c7eac-e8c1-49af-a215-fa0036f29141","version":3},"test","yaddayadda"], "id":4}
func (api *UIServerAPI) Import(ctx context.Context, keyJSON json.RawMessage, oldPassphrase, newPassphrase string) (accounts.Account, error) {
be := api.am.Backends(keystore.KeyStoreType)
if len(be) == 0 {
return accounts.Account{}, errors.New("password based accounts not supported")
}
if err := ValidatePasswordFormat(newPassphrase); err != nil {
return accounts.Account{}, fmt.Errorf("password requirements not met: %v", err)
}
return be[0].(*keystore.KeyStore).Import(keyJSON, oldPassphrase, newPassphrase)
}
// Other methods to be added, not yet implemented are:
// - Ruleset interaction: add rules, attest rulefiles
// - Store metadata about accounts, e.g. naming of accounts

1
signer/core/validation_test.go
View file

@ -25,7 +25,6 @@ import (
"github.com/ethereum/go-ethereum/common/hexutil"
)
func hexAddr(a string) common.Address { return common.BytesToAddress(common.FromHex(a)) }
func mixAddr(a string) (*common.MixedcaseAddress, error) {
return common.NewMixedcaseAddressFromString(a)
}

9
signer/rules/rules.go
View file

@ -46,16 +46,16 @@ func consoleOutput(call otto.FunctionCall) otto.Value {
return otto.Value{}
}
// rulesetUI provides an implementation of SignerUI that evaluates a javascript
// rulesetUI provides an implementation of UIClientAPI that evaluates a javascript
// file for each defined UI-method
type rulesetUI struct {
next core.SignerUI // The next handler, for manual processing
next core.UIClientAPI // The next handler, for manual processing
storage storage.Storage
credentials storage.Storage
jsRules string // The rules to use
}
func NewRuleEvaluator(next core.SignerUI, jsbackend, credentialsBackend storage.Storage) (*rulesetUI, error) {
func NewRuleEvaluator(next core.UIClientAPI, jsbackend, credentialsBackend storage.Storage) (*rulesetUI, error) {
c := &rulesetUI{
next: next,
storage: jsbackend,
@ -65,6 +65,9 @@ func NewRuleEvaluator(next core.SignerUI, jsbackend, credentialsBackend storage.
return c, nil
}
func (r *rulesetUI) RegisterUIServer(api *core.UIServerAPI) {
// TODO, make it possible to query from js
}
func (r *rulesetUI) Init(javascriptRules string) error {
r.jsRules = javascriptRules

24
signer/rules/rules_test.go
View file

@ -77,12 +77,10 @@ type alwaysDenyUI struct{}
func (alwaysDenyUI) OnInputRequired(info core.UserInputRequest) (core.UserInputResponse, error) {
return core.UserInputResponse{}, nil
}
func (alwaysDenyUI) OnSignerStartup(info core.StartupInfo) {
func (alwaysDenyUI) RegisterUIServer(api *core.UIServerAPI) {
}
func (alwaysDenyUI) OnMasterPassword(request *core.PasswordRequest) (core.PasswordResponse, error) {
return core.PasswordResponse{}, nil
func (alwaysDenyUI) OnSignerStartup(info core.StartupInfo) {
}
func (alwaysDenyUI) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, error) {
@ -133,11 +131,11 @@ func initRuleEngine(js string) (*rulesetUI, error) {
}
func TestListRequest(t *testing.T) {
accs := make([]core.Account, 5)
accs := make([]accounts.Account, 5)
for i := range accs {
addr := fmt.Sprintf("000000000000000000000000000000000000000%x", i)
acc := core.Account{
acc := accounts.Account{
Address: common.BytesToAddress(common.Hex2Bytes(addr)),
URL: accounts.URL{Scheme: "test", Path: fmt.Sprintf("acc-%d", i)},
}
@ -208,6 +206,10 @@ type dummyUI struct {
calls []string
}
func (d *dummyUI) RegisterUIServer(api *core.UIServerAPI) {
panic("implement me")
}
func (d *dummyUI) OnInputRequired(info core.UserInputRequest) (core.UserInputResponse, error) {
d.calls = append(d.calls, "OnInputRequired")
return core.UserInputResponse{}, nil
@ -255,10 +257,6 @@ func (d *dummyUI) OnApprovedTx(tx ethapi.SignTransactionResult) {
d.calls = append(d.calls, "OnApprovedTx")
}
func (d *dummyUI) OnMasterPassword(request *core.PasswordRequest) (core.PasswordResponse, error) {
return core.PasswordResponse{}, nil
}
func (d *dummyUI) OnSignerStartup(info core.StartupInfo) {
}
@ -531,12 +529,10 @@ func (d *dontCallMe) OnInputRequired(info core.UserInputRequest) (core.UserInput
d.t.Fatalf("Did not expect next-handler to be called")
return core.UserInputResponse{}, nil
}
func (d *dontCallMe) OnSignerStartup(info core.StartupInfo) {
func (d *dontCallMe) RegisterUIServer(api *core.UIServerAPI) {
}
func (d *dontCallMe) OnMasterPassword(request *core.PasswordRequest) (core.PasswordResponse, error) {
return core.PasswordResponse{}, nil
func (d *dontCallMe) OnSignerStartup(info core.StartupInfo) {
}
func (d *dontCallMe) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, error) {

21
swarm/docker/Dockerfile
View file

@ -10,14 +10,23 @@ RUN mkdir -p $GOPATH/src/github.com/ethereum && \
git checkout ${VERSION} && \
go install -ldflags "-X main.gitCommit=${VERSION}" ./cmd/swarm && \
go install -ldflags "-X main.gitCommit=${VERSION}" ./cmd/swarm/swarm-smoke && \
go install -ldflags "-X main.gitCommit=${VERSION}" ./cmd/geth && \
cp $GOPATH/bin/swarm /swarm && cp $GOPATH/bin/geth /geth && cp $GOPATH/bin/swarm-smoke /swarm-smoke
go install -ldflags "-X main.gitCommit=${VERSION}" ./cmd/swarm/global-store && \
go install -ldflags "-X main.gitCommit=${VERSION}" ./cmd/geth
# Release image with the required binaries and scripts
FROM alpine:3.8
FROM alpine:3.8 as swarm-smoke
WORKDIR /
COPY --from=builder /swarm /geth /swarm-smoke /
ADD run.sh /run.sh
COPY --from=builder /go/bin/swarm-smoke /
ADD run-smoke.sh /run-smoke.sh
ENTRYPOINT ["/run-smoke.sh"]
FROM alpine:3.8 as swarm-global-store
WORKDIR /
COPY --from=builder /go/bin/global-store /
ENTRYPOINT ["/global-store"]
FROM alpine:3.8 as swarm
WORKDIR /
COPY --from=builder /go/bin/swarm /go/bin/geth /
ADD run.sh /run.sh
ENTRYPOINT ["/run.sh"]

77
swarm/network/kademlia.go
View file

@ -353,6 +353,9 @@ func (k *Kademlia) sendNeighbourhoodDepthChange() {
// Not receiving from the returned channel will block Register function
// when address count value changes.
func (k *Kademlia) AddrCountC() <-chan int {
k.lock.Lock()
defer k.lock.Unlock()
if k.addrCountC == nil {
k.addrCountC = make(chan int)
}
@ -625,7 +628,8 @@ func (k *Kademlia) string() string {
// used for testing only
// TODO move to separate testing tools file
type PeerPot struct {
NNSet [][]byte
NNSet [][]byte
PeersPerBin []int
}
// NewPeerPotMap creates a map of pot record of *BzzAddr with keys
@ -651,6 +655,7 @@ func NewPeerPotMap(neighbourhoodSize int, addrs [][]byte) map[string]*PeerPot {
// all nn-peers
var nns [][]byte
peersPerBin := make([]int, depth)
// iterate through the neighbours, going from the deepest to the shallowest
np.EachNeighbour(a, Pof, func(val pot.Val, po int) bool {
@ -664,14 +669,18 @@ func NewPeerPotMap(neighbourhoodSize int, addrs [][]byte) map[string]*PeerPot {
// a neighbor is any peer in or deeper than the depth
if po >= depth {
nns = append(nns, addr)
return true
} else {
// for peers < depth, we just count the number in each bin
// the bin is the index of the slice
peersPerBin[po]++
}
return false
return true
})
log.Trace(fmt.Sprintf("%x PeerPotMap NNS: %s", addrs[i][:4], LogAddrs(nns)))
log.Trace(fmt.Sprintf("%x PeerPotMap NNS: %s, peersPerBin", addrs[i][:4], LogAddrs(nns)))
ppmap[common.Bytes2Hex(a)] = &PeerPot{
NNSet: nns,
NNSet: nns,
PeersPerBin: peersPerBin,
}
}
return ppmap
@ -695,6 +704,39 @@ func (k *Kademlia) saturation() int {
return prev
}
// isSaturated returns true if the kademlia is considered saturated, or false if not.
// It checks this by checking an array of ints called unsaturatedBins; each item in that array corresponds
// to the bin which is unsaturated (number of connections < k.MinBinSize).
// The bin is considered unsaturated only if there are actual peers in that PeerPot's bin (peersPerBin)
// (if there is no peer for a given bin, then no connection could ever be established;
// in a God's view this is relevant as no more peers will ever appear on that bin)
func (k *Kademlia) isSaturated(peersPerBin []int, depth int) bool {
// depth could be calculated from k but as this is called from `GetHealthInfo()`,
// the depth has already been calculated so we can require it as a parameter
// early check for depth
if depth != len(peersPerBin) {
return false
}
unsaturatedBins := make([]int, 0)
k.conns.EachBin(k.base, Pof, 0, func(po, size int, f func(func(val pot.Val) bool) bool) bool {
if po >= depth {
return false
}
log.Trace("peers per bin", "peersPerBin[po]", peersPerBin[po], "po", po)
// if there are actually peers in the PeerPot who can fulfill k.MinBinSize
if size < k.MinBinSize && size < peersPerBin[po] {
log.Trace("connections for po", "po", po, "size", size)
unsaturatedBins = append(unsaturatedBins, po)
}
return true
})
log.Trace("list of unsaturated bins", "unsaturatedBins", unsaturatedBins)
return len(unsaturatedBins) == 0
}
// knowNeighbours tests if all neighbours in the peerpot
// are found among the peers known to the kademlia
// It is used in Healthy function for testing only
@ -777,11 +819,13 @@ type Health struct {
ConnectNN bool // whether node is connected to all its neighbours
CountConnectNN int // amount of neighbours connected to
MissingConnectNN [][]byte // which neighbours we should have been connected to but we're not
Saturated bool // whether we are connected to all the peers we would have liked to
Hive string
// Saturated: if in all bins < depth number of connections >= MinBinsize or,
// if number of connections < MinBinSize, to the number of available peers in that bin
Saturated bool
Hive string
}
// Healthy reports the health state of the kademlia connectivity
// GetHealthInfo reports the health state of the kademlia connectivity
//
// The PeerPot argument provides an all-knowing view of the network
// The resulting Health object is a result of comparisons between
@ -789,7 +833,7 @@ type Health struct {
// what SHOULD it have been when we take all we know about the network into consideration.
//
// used for testing only
func (k *Kademlia) Healthy(pp *PeerPot) *Health {
func (k *Kademlia) GetHealthInfo(pp *PeerPot) *Health {
k.lock.RLock()
defer k.lock.RUnlock()
if len(pp.NNSet) < k.NeighbourhoodSize {
@ -798,7 +842,10 @@ func (k *Kademlia) Healthy(pp *PeerPot) *Health {
gotnn, countgotnn, culpritsgotnn := k.connectedNeighbours(pp.NNSet)
knownn, countknownn, culpritsknownn := k.knowNeighbours(pp.NNSet)
depth := depthForPot(k.conns, k.NeighbourhoodSize, k.base)
saturated := k.saturation() < depth
// check saturation
saturated := k.isSaturated(pp.PeersPerBin, depth)
log.Trace(fmt.Sprintf("%08x: healthy: knowNNs: %v, gotNNs: %v, saturated: %v\n", k.base, knownn, gotnn, saturated))
return &Health{
KnowNN: knownn,
@ -811,3 +858,13 @@ func (k *Kademlia) Healthy(pp *PeerPot) *Health {
Hive: k.string(),
}
}
// Healthy return the strict interpretation of `Healthy` given a `Health` struct
// definition of strict health: all conditions must be true:
// - we at least know one peer
// - we know all neighbors
// - we are connected to all known neighbors
// - it is saturated
func (h *Health) Healthy() bool {
return h.KnowNN && h.ConnectNN && h.CountKnowNN > 0 && h.Saturated
}

126
swarm/network/kademlia_test.go
View file

@ -168,6 +168,46 @@ func TestNeighbourhoodDepth(t *testing.T) {
testNum++
}
// TestHighMinBinSize tests that the saturation function also works
// if MinBinSize is > 2, the connection count is < k.MinBinSize
// and there are more peers available than connected
func TestHighMinBinSize(t *testing.T) {
// a function to test for different MinBinSize values
testKad := func(minBinSize int) {
// create a test kademlia
tk := newTestKademlia(t, "11111111")
// set its MinBinSize to desired value
tk.KadParams.MinBinSize = minBinSize
// add a couple of peers (so we have NN and depth)
tk.On("00000000") // bin 0
tk.On("11100000") // bin 3
tk.On("11110000") // bin 4
first := "10000000" // add a first peer at bin 1
tk.Register(first) // register it
// we now have one registered peer at bin 1;
// iterate and connect one peer at each iteration;
// should be unhealthy until at minBinSize - 1
// we connect the unconnected but registered peer
for i := 1; i < minBinSize; i++ {
peer := fmt.Sprintf("1000%b", 8|i)
tk.On(peer)
if i == minBinSize-1 {
tk.On(first)
tk.checkHealth(true)
return
}
tk.checkHealth(false)
}
}
// test MinBinSizes of 3 to 5
testMinBinSizes := []int{3, 4, 5}
for _, k := range testMinBinSizes {
testKad(k)
}
}
// TestHealthStrict tests the simplest definition of health
// Which means whether we are connected to all neighbors we know of
func TestHealthStrict(t *testing.T) {
@ -176,60 +216,116 @@ func TestHealthStrict(t *testing.T) {
// no peers
// unhealthy (and lonely)
tk := newTestKademlia(t, "11111111")
tk.checkHealth(false, false)
tk.checkHealth(false)
// know one peer but not connected
// unhealthy
tk.Register("11100000")
tk.checkHealth(false, false)
tk.checkHealth(false)
// know one peer and connected
// healthy
// unhealthy: not saturated
tk.On("11100000")
tk.checkHealth(true, false)
tk.checkHealth(true)
// know two peers, only one connected
// unhealthy
tk.Register("11111100")
tk.checkHealth(false, false)
tk.checkHealth(false)
// know two peers and connected to both
// healthy
tk.On("11111100")
tk.checkHealth(true, false)
tk.checkHealth(true)
// know three peers, connected to the two deepest
// healthy
tk.Register("00000000")
tk.checkHealth(true, false)
tk.checkHealth(false)
// know three peers, connected to all three
// healthy
tk.On("00000000")
tk.checkHealth(true, false)
tk.checkHealth(true)
// add fourth peer deeper than current depth
// unhealthy
tk.Register("11110000")
tk.checkHealth(false, false)
tk.checkHealth(false)
// connected to three deepest peers
// healthy
tk.On("11110000")
tk.checkHealth(true, false)
tk.checkHealth(true)
// add additional peer in same bin as deepest peer
// unhealthy
tk.Register("11111101")
tk.checkHealth(false, false)
tk.checkHealth(false)
// four deepest of five peers connected
// healthy
tk.On("11111101")
tk.checkHealth(true, false)
tk.checkHealth(true)
// add additional peer in bin 0
// unhealthy: unsaturated bin 0, 2 known but 1 connected
tk.Register("00000001")
tk.checkHealth(false)
// Connect second in bin 0
// healthy
tk.On("00000001")
tk.checkHealth(true)
// add peer in bin 1
// unhealthy, as it is known but not connected
tk.Register("10000000")
tk.checkHealth(false)
// connect peer in bin 1
// depth change, is now 1
// healthy, 1 peer in bin 1 known and connected
tk.On("10000000")
tk.checkHealth(true)
// add second peer in bin 1
// unhealthy, as it is known but not connected
tk.Register("10000001")
tk.checkHealth(false)
// connect second peer in bin 1
// healthy,
tk.On("10000001")
tk.checkHealth(true)
// connect third peer in bin 1
// healthy,
tk.On("10000011")
tk.checkHealth(true)
// add peer in bin 2
// unhealthy, no depth change
tk.Register("11000000")
tk.checkHealth(false)
// connect peer in bin 2
// depth change - as we already have peers in bin 3 and 4,
// we have contiguous bins, no bin < po 5 is empty -> depth 5
// healthy, every bin < depth has the max available peers,
// even if they are < MinBinSize
tk.On("11000000")
tk.checkHealth(true)
// add peer in bin 2
// unhealthy, peer bin is below depth 5 but
// has more available peers (2) than connected ones (1)
// --> unsaturated
tk.Register("11000011")
tk.checkHealth(false)
}
func (tk *testKademlia) checkHealth(expectHealthy bool, expectSaturation bool) {
func (tk *testKademlia) checkHealth(expectHealthy bool) {
tk.t.Helper()
kid := common.Bytes2Hex(tk.BaseAddr())
addrs := [][]byte{tk.BaseAddr()}
@ -239,13 +335,13 @@ func (tk *testKademlia) checkHealth(expectHealthy bool, expectSaturation bool) {
})
pp := NewPeerPotMap(tk.NeighbourhoodSize, addrs)
healthParams := tk.Healthy(pp[kid])
healthParams := tk.GetHealthInfo(pp[kid])
// definition of health, all conditions but be true:
// - we at least know one peer
// - we know all neighbors
// - we are connected to all known neighbors
health := healthParams.KnowNN && healthParams.ConnectNN && healthParams.CountKnowNN > 0
health := healthParams.Healthy()
if expectHealthy != health {
tk.t.Fatalf("expected kademlia health %v, is %v\n%v", expectHealthy, health, tk.String())
}

2
swarm/network/simulation/kademlia.go
View file

@ -64,7 +64,7 @@ func (s *Simulation) WaitTillHealthy(ctx context.Context) (ill map[enode.ID]*net
addr := common.Bytes2Hex(k.BaseAddr())
pp := ppmap[addr]
//call Healthy RPC
h := k.Healthy(pp)
h := k.GetHealthInfo(pp)
//print info
log.Debug(k.String())
log.Debug("kademlia", "connectNN", h.ConnectNN, "knowNN", h.KnowNN)

4
swarm/network/simulation/node.go
View file

@ -44,7 +44,7 @@ func (s *Simulation) NodeIDs() (ids []enode.ID) {
func (s *Simulation) UpNodeIDs() (ids []enode.ID) {
nodes := s.Net.GetNodes()
for _, node := range nodes {
if node.Up {
if node.Up() {
ids = append(ids, node.ID())
}
}
@ -55,7 +55,7 @@ func (s *Simulation) UpNodeIDs() (ids []enode.ID) {
func (s *Simulation) DownNodeIDs() (ids []enode.ID) {
nodes := s.Net.GetNodes()
for _, node := range nodes {
if !node.Up {
if !node.Up() {
ids = append(ids, node.ID())
}
}

63
swarm/network/simulation/node_test.go
View file

@ -54,7 +54,7 @@ func TestUpDownNodeIDs(t *testing.T) {
gotIDs = sim.UpNodeIDs()
for _, id := range gotIDs {
if !sim.Net.GetNode(id).Up {
if !sim.Net.GetNode(id).Up() {
t.Errorf("node %s should not be down", id)
}
}
@ -66,7 +66,7 @@ func TestUpDownNodeIDs(t *testing.T) {
gotIDs = sim.DownNodeIDs()
for _, id := range gotIDs {
if sim.Net.GetNode(id).Up {
if sim.Net.GetNode(id).Up() {
t.Errorf("node %s should not be up", id)
}
}
@ -112,7 +112,7 @@ func TestAddNode(t *testing.T) {
t.Fatal("node not found")
}
if !n.Up {
if !n.Up() {
t.Error("node not started")
}
}
@ -327,7 +327,7 @@ func TestStartStopNode(t *testing.T) {
if n == nil {
t.Fatal("node not found")
}
if !n.Up {
if !n.Up() {
t.Error("node not started")
}
@ -335,26 +335,17 @@ func TestStartStopNode(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if n.Up {
if n.Up() {
t.Error("node not stopped")
}
// Sleep here to ensure that Network.watchPeerEvents defer function
// has set the `node.Up = false` before we start the node again.
// p2p/simulations/network.go:215
//
// The same node is stopped and started again, and upon start
// watchPeerEvents is started in a goroutine. If the node is stopped
// and then very quickly started, that goroutine may be scheduled later
// then start and force `node.Up = false` in its defer function.
// This will make this test unreliable.
time.Sleep(time.Second)
waitForPeerEventPropagation()
err = sim.StartNode(id)
if err != nil {
t.Fatal(err)
}
if !n.Up {
if !n.Up() {
t.Error("node not started")
}
}
@ -377,7 +368,7 @@ func TestStartStopRandomNode(t *testing.T) {
if n == nil {
t.Fatal("node not found")
}
if n.Up {
if n.Up() {
t.Error("node not stopped")
}
@ -386,16 +377,7 @@ func TestStartStopRandomNode(t *testing.T) {
t.Fatal(err)
}
// Sleep here to ensure that Network.watchPeerEvents defer function
// has set the `node.Up = false` before we start the node again.
// p2p/simulations/network.go:215
//
// The same node is stopped and started again, and upon start
// watchPeerEvents is started in a goroutine. If the node is stopped
// and then very quickly started, that goroutine may be scheduled later
// then start and force `node.Up = false` in its defer function.
// This will make this test unreliable.
time.Sleep(time.Second)
waitForPeerEventPropagation()
idStarted, err := sim.StartRandomNode()
if err != nil {
@ -426,21 +408,12 @@ func TestStartStopRandomNodes(t *testing.T) {
if n == nil {
t.Fatal("node not found")
}
if n.Up {
if n.Up() {
t.Error("node not stopped")
}
}
// Sleep here to ensure that Network.watchPeerEvents defer function
// has set the `node.Up = false` before we start the node again.
// p2p/simulations/network.go:215
//
// The same node is stopped and started again, and upon start
// watchPeerEvents is started in a goroutine. If the node is stopped
// and then very quickly started, that goroutine may be scheduled later
// then start and force `node.Up = false` in its defer function.
// This will make this test unreliable.
time.Sleep(time.Second)
waitForPeerEventPropagation()
ids, err = sim.StartRandomNodes(2)
if err != nil {
@ -452,8 +425,20 @@ func TestStartStopRandomNodes(t *testing.T) {
if n == nil {
t.Fatal("node not found")
}
if !n.Up {
if !n.Up() {
t.Error("node not started")
}
}
}
func waitForPeerEventPropagation() {
// Sleep here to ensure that Network.watchPeerEvents defer function
// has set the `node.Up() = false` before we start the node again.
//
// The same node is stopped and started again, and upon start
// watchPeerEvents is started in a goroutine. If the node is stopped
// and then very quickly started, that goroutine may be scheduled later
// then start and force `node.Up() = false` in its defer function.
// This will make this test unreliable.
time.Sleep(1 * time.Second)
}

2
swarm/network/simulation/service.go
View file

@ -52,7 +52,7 @@ func (s *Simulation) Services(name string) (services map[enode.ID]node.Service)
nodes := s.Net.GetNodes()
services = make(map[enode.ID]node.Service)
for _, node := range nodes {
if !node.Up {
if !node.Up() {
continue
}
simNode, ok := node.Node.(*adapters.SimNode)

4
swarm/network/simulation/simulation_test.go
View file

@ -124,7 +124,7 @@ func TestClose(t *testing.T) {
var upNodeCount int
for _, n := range sim.Net.GetNodes() {
if n.Up {
if n.Up() {
upNodeCount++
}
}
@ -140,7 +140,7 @@ func TestClose(t *testing.T) {
upNodeCount = 0
for _, n := range sim.Net.GetNodes() {
if n.Up {
if n.Up() {
upNodeCount++
}
}

6
swarm/network/simulations/discovery/discovery_test.go
View file

@ -267,7 +267,7 @@ func discoverySimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simul
}
healthy := &network.Health{}
if err := client.Call(&healthy, "hive_healthy", ppmap[common.Bytes2Hex(id.Bytes())]); err != nil {
if err := client.Call(&healthy, "hive_getHealthInfo", ppmap[common.Bytes2Hex(id.Bytes())]); err != nil {
return false, fmt.Errorf("error getting node health: %s", err)
}
log.Debug(fmt.Sprintf("node %4s healthy: connected nearest neighbours: %v, know nearest neighbours: %v,\n\n%v", id, healthy.ConnectNN, healthy.KnowNN, healthy.Hive))
@ -352,7 +352,7 @@ func discoveryPersistenceSimulation(nodes, conns int, adapter adapters.NodeAdapt
healthy := &network.Health{}
addr := id.String()
ppmap := network.NewPeerPotMap(network.NewKadParams().NeighbourhoodSize, addrs)
if err := client.Call(&healthy, "hive_healthy", ppmap[common.Bytes2Hex(id.Bytes())]); err != nil {
if err := client.Call(&healthy, "hive_getHealthInfo", ppmap[common.Bytes2Hex(id.Bytes())]); err != nil {
return fmt.Errorf("error getting node health: %s", err)
}
@ -422,7 +422,7 @@ func discoveryPersistenceSimulation(nodes, conns int, adapter adapters.NodeAdapt
healthy := &network.Health{}
ppmap := network.NewPeerPotMap(network.NewKadParams().NeighbourhoodSize, addrs)
if err := client.Call(&healthy, "hive_healthy", ppmap[common.Bytes2Hex(id.Bytes())]); err != nil {
if err := client.Call(&healthy, "hive_getHealthInfo", ppmap[common.Bytes2Hex(id.Bytes())]); err != nil {
return false, fmt.Errorf("error getting node health: %s", err)
}
log.Info(fmt.Sprintf("node %4s healthy: got nearest neighbours: %v, know nearest neighbours: %v", id, healthy.ConnectNN, healthy.KnowNN))

2
swarm/network/simulations/overlay_test.go
View file

@ -178,7 +178,7 @@ func watchSimEvents(net *simulations.Network, ctx context.Context, trigger chan
case ev := <-events:
//only catch node up events
if ev.Type == simulations.EventTypeNode {
if ev.Node.Up {
if ev.Node.Up() {
log.Debug("got node up event", "event", ev, "node", ev.Node.Config.ID)
select {
case trigger <- ev.Node.Config.ID:

62
swarm/network/stream/common_test.go
View file

@ -151,7 +151,7 @@ func newStreamerTester(registryOptions *RegistryOptions) (*p2ptest.ProtocolTeste
// temp datadir
datadir, err := ioutil.TempDir("", "streamer")
if err != nil {
return nil, nil, nil, func() {}, err
return nil, nil, nil, nil, err
}
removeDataDir := func() {
os.RemoveAll(datadir)
@ -163,12 +163,14 @@ func newStreamerTester(registryOptions *RegistryOptions) (*p2ptest.ProtocolTeste
localStore, err := storage.NewTestLocalStoreForAddr(params)
if err != nil {
return nil, nil, nil, removeDataDir, err
removeDataDir()
return nil, nil, nil, nil, err
}
netStore, err := storage.NewNetStore(localStore, nil)
if err != nil {
return nil, nil, nil, removeDataDir, err
removeDataDir()
return nil, nil, nil, nil, err
}
delivery := NewDelivery(to, netStore)
@ -180,8 +182,9 @@ func newStreamerTester(registryOptions *RegistryOptions) (*p2ptest.ProtocolTeste
}
protocolTester := p2ptest.NewProtocolTester(addr.ID(), 1, streamer.runProtocol)
err = waitForPeers(streamer, 1*time.Second, 1)
err = waitForPeers(streamer, 10*time.Second, 1)
if err != nil {
teardown()
return nil, nil, nil, nil, errors.New("timeout: peer is not created")
}
@ -317,3 +320,54 @@ func createTestLocalStorageForID(id enode.ID, addr *network.BzzAddr) (storage.Ch
}
return store, datadir, nil
}
// watchDisconnections receives simulation peer events in a new goroutine and sets atomic value
// disconnected to true in case of a disconnect event.
func watchDisconnections(ctx context.Context, sim *simulation.Simulation) (disconnected *boolean) {
log.Debug("Watching for disconnections")
disconnections := sim.PeerEvents(
ctx,
sim.NodeIDs(),
simulation.NewPeerEventsFilter().Drop(),
)
disconnected = new(boolean)
go func() {
for {
select {
case <-ctx.Done():
return
case d := <-disconnections:
if d.Error != nil {
log.Error("peer drop event error", "node", d.NodeID, "peer", d.PeerID, "err", d.Error)
} else {
log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
}
disconnected.set(true)
}
}
}()
return disconnected
}
// boolean is used to concurrently set
// and read a boolean value.
type boolean struct {
v bool
mu sync.RWMutex
}
// set sets the value.
func (b *boolean) set(v bool) {
b.mu.Lock()
defer b.mu.Unlock()
b.v = v
}
// bool reads the value.
func (b *boolean) bool() bool {
b.mu.RLock()
defer b.mu.RUnlock()
return b.v
}

302
swarm/network/stream/delivery_test.go
View file

@ -22,7 +22,6 @@ import (
"errors"
"fmt"
"sync"
"sync/atomic"
"testing"
"time"
@ -48,10 +47,10 @@ func TestStreamerRetrieveRequest(t *testing.T) {
Syncing: SyncingDisabled,
}
tester, streamer, _, teardown, err := newStreamerTester(regOpts)
defer teardown()
if err != nil {
t.Fatal(err)
}
defer teardown()
node := tester.Nodes[0]
@ -100,10 +99,10 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) {
Retrieval: RetrievalEnabled,
Syncing: SyncingDisabled, //do no syncing
})
defer teardown()
if err != nil {
t.Fatal(err)
}
defer teardown()
node := tester.Nodes[0]
@ -172,10 +171,10 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) {
Retrieval: RetrievalEnabled,
Syncing: SyncingDisabled,
})
defer teardown()
if err != nil {
t.Fatal(err)
}
defer teardown()
node := tester.Nodes[0]
@ -362,10 +361,10 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) {
Retrieval: RetrievalDisabled,
Syncing: SyncingDisabled,
})
defer teardown()
if err != nil {
t.Fatal(err)
}
defer teardown()
streamer.RegisterClientFunc("foo", func(p *Peer, t string, live bool) (Client, error) {
return &testClient{
@ -454,149 +453,136 @@ func TestDeliveryFromNodes(t *testing.T) {
}
func testDeliveryFromNodes(t *testing.T, nodes, chunkCount int, skipCheck bool) {
sim := simulation.New(map[string]simulation.ServiceFunc{
"streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) {
addr, netStore, delivery, clean, err := newNetStoreAndDelivery(ctx, bucket)
if err != nil {
return nil, nil, err
}
r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{
SkipCheck: skipCheck,
Syncing: SyncingDisabled,
Retrieval: RetrievalEnabled,
}, nil)
bucket.Store(bucketKeyRegistry, r)
cleanup = func() {
r.Close()
clean()
}
return r, cleanup, nil
},
})
defer sim.Close()
log.Info("Adding nodes to simulation")
_, err := sim.AddNodesAndConnectChain(nodes)
if err != nil {
t.Fatal(err)
}
log.Info("Starting simulation")
ctx := context.Background()
result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) (err error) {
nodeIDs := sim.UpNodeIDs()
//determine the pivot node to be the first node of the simulation
pivot := nodeIDs[0]
//distribute chunks of a random file into Stores of nodes 1 to nodes
//we will do this by creating a file store with an underlying round-robin store:
//the file store will create a hash for the uploaded file, but every chunk will be
//distributed to different nodes via round-robin scheduling
log.Debug("Writing file to round-robin file store")
//to do this, we create an array for chunkstores (length minus one, the pivot node)
stores := make([]storage.ChunkStore, len(nodeIDs)-1)
//we then need to get all stores from the sim....
lStores := sim.NodesItems(bucketKeyStore)
i := 0
//...iterate the buckets...
for id, bucketVal := range lStores {
//...and remove the one which is the pivot node
if id == pivot {
continue
}
//the other ones are added to the array...
stores[i] = bucketVal.(storage.ChunkStore)
i++
}
//...which then gets passed to the round-robin file store
roundRobinFileStore := storage.NewFileStore(newRoundRobinStore(stores...), storage.NewFileStoreParams())
//now we can actually upload a (random) file to the round-robin store
size := chunkCount * chunkSize
log.Debug("Storing data to file store")
fileHash, wait, err := roundRobinFileStore.Store(ctx, testutil.RandomReader(1, size), int64(size), false)
// wait until all chunks stored
if err != nil {
return err
}
err = wait(ctx)
if err != nil {
return err
}
log.Debug("Waiting for kademlia")
// TODO this does not seem to be correct usage of the function, as the simulation may have no kademlias
if _, err := sim.WaitTillHealthy(ctx); err != nil {
return err
}
//get the pivot node's filestore
item, ok := sim.NodeItem(pivot, bucketKeyFileStore)
if !ok {
return fmt.Errorf("No filestore")
}
pivotFileStore := item.(*storage.FileStore)
log.Debug("Starting retrieval routine")
retErrC := make(chan error)
go func() {
// start the retrieval on the pivot node - this will spawn retrieve requests for missing chunks
// we must wait for the peer connections to have started before requesting
n, err := readAll(pivotFileStore, fileHash)
log.Info(fmt.Sprintf("retrieved %v", fileHash), "read", n, "err", err)
retErrC <- err
}()
log.Debug("Watching for disconnections")
disconnections := sim.PeerEvents(
context.Background(),
sim.NodeIDs(),
simulation.NewPeerEventsFilter().Drop(),
)
var disconnected atomic.Value
go func() {
for d := range disconnections {
if d.Error != nil {
log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
disconnected.Store(true)
t.Helper()
t.Run(fmt.Sprintf("testDeliveryFromNodes_%d_%d_skipCheck_%t", nodes, chunkCount, skipCheck), func(t *testing.T) {
sim := simulation.New(map[string]simulation.ServiceFunc{
"streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) {
addr, netStore, delivery, clean, err := newNetStoreAndDelivery(ctx, bucket)
if err != nil {
return nil, nil, err
}
r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{
SkipCheck: skipCheck,
Syncing: SyncingDisabled,
Retrieval: RetrievalEnabled,
}, nil)
bucket.Store(bucketKeyRegistry, r)
cleanup = func() {
r.Close()
clean()
}
return r, cleanup, nil
},
})
defer sim.Close()
log.Info("Adding nodes to simulation")
_, err := sim.AddNodesAndConnectChain(nodes)
if err != nil {
t.Fatal(err)
}
log.Info("Starting simulation")
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) (err error) {
nodeIDs := sim.UpNodeIDs()
//determine the pivot node to be the first node of the simulation
pivot := nodeIDs[0]
//distribute chunks of a random file into Stores of nodes 1 to nodes
//we will do this by creating a file store with an underlying round-robin store:
//the file store will create a hash for the uploaded file, but every chunk will be
//distributed to different nodes via round-robin scheduling
log.Debug("Writing file to round-robin file store")
//to do this, we create an array for chunkstores (length minus one, the pivot node)
stores := make([]storage.ChunkStore, len(nodeIDs)-1)
//we then need to get all stores from the sim....
lStores := sim.NodesItems(bucketKeyStore)
i := 0
//...iterate the buckets...
for id, bucketVal := range lStores {
//...and remove the one which is the pivot node
if id == pivot {
continue
}
//the other ones are added to the array...
stores[i] = bucketVal.(storage.ChunkStore)
i++
}
}()
defer func() {
//...which then gets passed to the round-robin file store
roundRobinFileStore := storage.NewFileStore(newRoundRobinStore(stores...), storage.NewFileStoreParams())
//now we can actually upload a (random) file to the round-robin store
size := chunkCount * chunkSize
log.Debug("Storing data to file store")
fileHash, wait, err := roundRobinFileStore.Store(ctx, testutil.RandomReader(1, size), int64(size), false)
// wait until all chunks stored
if err != nil {
if yes, ok := disconnected.Load().(bool); ok && yes {
return err
}
err = wait(ctx)
if err != nil {
return err
}
log.Debug("Waiting for kademlia")
// TODO this does not seem to be correct usage of the function, as the simulation may have no kademlias
if _, err := sim.WaitTillHealthy(ctx); err != nil {
return err
}
//get the pivot node's filestore
item, ok := sim.NodeItem(pivot, bucketKeyFileStore)
if !ok {
return fmt.Errorf("No filestore")
}
pivotFileStore := item.(*storage.FileStore)
log.Debug("Starting retrieval routine")
retErrC := make(chan error)
go func() {
// start the retrieval on the pivot node - this will spawn retrieve requests for missing chunks
// we must wait for the peer connections to have started before requesting
n, err := readAll(pivotFileStore, fileHash)
log.Info(fmt.Sprintf("retrieved %v", fileHash), "read", n, "err", err)
retErrC <- err
}()
disconnected := watchDisconnections(ctx, sim)
defer func() {
if err != nil && disconnected.bool() {
err = errors.New("disconnect events received")
}
}()
//finally check that the pivot node gets all chunks via the root hash
log.Debug("Check retrieval")
success := true
var total int64
total, err = readAll(pivotFileStore, fileHash)
if err != nil {
return err
}
log.Info(fmt.Sprintf("check if %08x is available locally: number of bytes read %v/%v (error: %v)", fileHash, total, size, err))
if err != nil || total != int64(size) {
success = false
}
}()
//finally check that the pivot node gets all chunks via the root hash
log.Debug("Check retrieval")
success := true
var total int64
total, err = readAll(pivotFileStore, fileHash)
if err != nil {
return err
if !success {
return fmt.Errorf("Test failed, chunks not available on all nodes")
}
if err := <-retErrC; err != nil {
return fmt.Errorf("requesting chunks: %v", err)
}
log.Debug("Test terminated successfully")
return nil
})
if result.Error != nil {
t.Fatal(result.Error)
}
log.Info(fmt.Sprintf("check if %08x is available locally: number of bytes read %v/%v (error: %v)", fileHash, total, size, err))
if err != nil || total != int64(size) {
success = false
}
if !success {
return fmt.Errorf("Test failed, chunks not available on all nodes")
}
if err := <-retErrC; err != nil {
t.Fatalf("requesting chunks: %v", err)
}
log.Debug("Test terminated successfully")
return nil
})
if result.Error != nil {
t.Fatal(result.Error)
}
}
func BenchmarkDeliveryFromNodesWithoutCheck(b *testing.B) {
@ -657,21 +643,22 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, chunkCount int, skipCheck b
b.Fatal(err)
}
ctx := context.Background()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) (err error) {
nodeIDs := sim.UpNodeIDs()
node := nodeIDs[len(nodeIDs)-1]
item, ok := sim.NodeItem(node, bucketKeyFileStore)
if !ok {
b.Fatal("No filestore")
return errors.New("No filestore")
}
remoteFileStore := item.(*storage.FileStore)
pivotNode := nodeIDs[0]
item, ok = sim.NodeItem(pivotNode, bucketKeyNetStore)
if !ok {
b.Fatal("No filestore")
return errors.New("No filestore")
}
netStore := item.(*storage.NetStore)
@ -679,26 +666,10 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, chunkCount int, skipCheck b
return err
}
disconnections := sim.PeerEvents(
context.Background(),
sim.NodeIDs(),
simulation.NewPeerEventsFilter().Drop(),
)
var disconnected atomic.Value
go func() {
for d := range disconnections {
if d.Error != nil {
log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
disconnected.Store(true)
}
}
}()
disconnected := watchDisconnections(ctx, sim)
defer func() {
if err != nil {
if yes, ok := disconnected.Load().(bool); ok && yes {
err = errors.New("disconnect events received")
}
if err != nil && disconnected.bool() {
err = errors.New("disconnect events received")
}
}()
// benchmark loop
@ -713,12 +684,12 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, chunkCount int, skipCheck b
ctx := context.TODO()
hash, wait, err := remoteFileStore.Store(ctx, testutil.RandomReader(i, chunkSize), int64(chunkSize), false)
if err != nil {
b.Fatalf("expected no error. got %v", err)
return fmt.Errorf("store: %v", err)
}
// wait until all chunks stored
err = wait(ctx)
if err != nil {
b.Fatalf("expected no error. got %v", err)
return fmt.Errorf("wait store: %v", err)
}
// collect the hashes
hashes[i] = hash
@ -754,10 +725,7 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, chunkCount int, skipCheck b
break Loop
}
}
if err != nil {
b.Fatal(err)
}
return nil
return err
})
if result.Error != nil {
b.Fatal(result.Error)

30
swarm/network/stream/intervals_test.go
View file

@ -22,7 +22,6 @@ import (
"errors"
"fmt"
"sync"
"sync/atomic"
"testing"
"time"
@ -118,13 +117,11 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) {
_, wait, err := fileStore.Store(ctx, testutil.RandomReader(1, size), int64(size), false)
if err != nil {
log.Error("Store error: %v", "err", err)
t.Fatal(err)
return fmt.Errorf("store: %v", err)
}
err = wait(ctx)
if err != nil {
log.Error("Wait error: %v", "err", err)
t.Fatal(err)
return fmt.Errorf("wait store: %v", err)
}
item, ok = sim.NodeItem(checker, bucketKeyRegistry)
@ -136,32 +133,15 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) {
liveErrC := make(chan error)
historyErrC := make(chan error)
log.Debug("Watching for disconnections")
disconnections := sim.PeerEvents(
context.Background(),
sim.NodeIDs(),
simulation.NewPeerEventsFilter().Drop(),
)
err = registry.Subscribe(storer, NewStream(externalStreamName, "", live), history, Top)
if err != nil {
return err
}
var disconnected atomic.Value
go func() {
for d := range disconnections {
if d.Error != nil {
log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
disconnected.Store(true)
}
}
}()
disconnected := watchDisconnections(ctx, sim)
defer func() {
if err != nil {
if yes, ok := disconnected.Load().(bool); ok && yes {
err = errors.New("disconnect events received")
}
if err != nil && disconnected.bool() {
err = errors.New("disconnect events received")
}
}()

8
swarm/network/stream/lightnode_test.go
View file

@ -29,10 +29,10 @@ func TestLigthnodeRetrieveRequestWithRetrieve(t *testing.T) {
Syncing: SyncingDisabled,
}
tester, _, _, teardown, err := newStreamerTester(registryOptions)
defer teardown()
if err != nil {
t.Fatal(err)
}
defer teardown()
node := tester.Nodes[0]
@ -68,10 +68,10 @@ func TestLigthnodeRetrieveRequestWithoutRetrieve(t *testing.T) {
Syncing: SyncingDisabled,
}
tester, _, _, teardown, err := newStreamerTester(registryOptions)
defer teardown()
if err != nil {
t.Fatal(err)
}
defer teardown()
node := tester.Nodes[0]
@ -112,10 +112,10 @@ func TestLigthnodeRequestSubscriptionWithSync(t *testing.T) {
Syncing: SyncingRegisterOnly,
}
tester, _, _, teardown, err := newStreamerTester(registryOptions)
defer teardown()
if err != nil {
t.Fatal(err)
}
defer teardown()
node := tester.Nodes[0]
@ -157,10 +157,10 @@ func TestLigthnodeRequestSubscriptionWithoutSync(t *testing.T) {
Syncing: SyncingDisabled,
}
tester, _, _, teardown, err := newStreamerTester(registryOptions)
defer teardown()
if err != nil {
t.Fatal(err)
}
defer teardown()
node := tester.Nodes[0]

24
swarm/network/stream/norace_test.go Normal file
View file

@ -0,0 +1,24 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// +build !race
package stream
// Provide a flag to reduce the scope of tests when running them
// with race detector. Some of the tests are doing a lot of allocations
// on the heap, and race detector uses much more memory to track them.
const raceTest = false

23
swarm/network/stream/race_test.go Normal file
View file

@ -0,0 +1,23 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// +build race
package stream
// Reduce the scope of some tests when running with race detector,
// as it raises the memory consumption significantly.
const raceTest = true

15
swarm/network/stream/snapshot_retrieval_test.go
View file

@ -74,7 +74,7 @@ func TestRetrieval(t *testing.T) {
//if nodes/chunks have been provided via commandline,
//run the tests with these values
if *nodes != 0 && *chunks != 0 {
err := runRetrievalTest(*chunks, *nodes)
err := runRetrievalTest(t, *chunks, *nodes)
if err != nil {
t.Fatal(err)
}
@ -93,10 +93,12 @@ func TestRetrieval(t *testing.T) {
}
for _, n := range nodeCnt {
for _, c := range chnkCnt {
err := runRetrievalTest(c, n)
if err != nil {
t.Fatal(err)
}
t.Run(fmt.Sprintf("TestRetrieval_%d_%d", n, c), func(t *testing.T) {
err := runRetrievalTest(t, c, n)
if err != nil {
t.Fatal(err)
}
})
}
}
}
@ -225,7 +227,8 @@ simulation's `action` function.
The snapshot should have 'streamer' in its service list.
*/
func runRetrievalTest(chunkCount int, nodeCount int) error {
func runRetrievalTest(t *testing.T, chunkCount int, nodeCount int) error {
t.Helper()
sim := simulation.New(retrievalSimServiceMap)
defer sim.Close()

58
swarm/network/stream/snapshot_sync_test.go
View file

@ -17,11 +17,12 @@ package stream
import (
"context"
"errors"
"fmt"
"io/ioutil"
"os"
"runtime"
"sync"
"sync/atomic"
"testing"
"time"
@ -92,6 +93,15 @@ func TestSyncingViaGlobalSync(t *testing.T) {
if *longrunning {
chnkCnt = []int{1, 8, 32, 256, 1024}
nodeCnt = []int{16, 32, 64, 128, 256}
} else if raceTest {
// TestSyncingViaGlobalSync allocates a lot of memory
// with race detector. By reducing the number of chunks
// and nodes, memory consumption is lower and data races
// are still checked, while correctness of syncing is
// tested with more chunks and nodes in regular (!race)
// tests.
chnkCnt = []int{4}
nodeCnt = []int{16}
} else {
//default test
chnkCnt = []int{4, 32}
@ -113,7 +123,23 @@ var simServiceMap = map[string]simulation.ServiceFunc{
return nil, nil, err
}
r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{
var dir string
var store *state.DBStore
if raceTest {
// Use on-disk DBStore to reduce memory consumption in race tests.
dir, err = ioutil.TempDir("", "swarm-stream-")
if err != nil {
return nil, nil, err
}
store, err = state.NewDBStore(dir)
if err != nil {
return nil, nil, err
}
} else {
store = state.NewInmemoryStore()
}
r := NewRegistry(addr.ID(), delivery, netStore, store, &RegistryOptions{
Retrieval: RetrievalDisabled,
Syncing: SyncingAutoSubscribe,
SyncUpdateDelay: 3 * time.Second,
@ -156,36 +182,24 @@ func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) {
t.Fatal(err)
}
disconnections := sim.PeerEvents(
context.Background(),
sim.NodeIDs(),
simulation.NewPeerEventsFilter().Drop(),
)
var disconnected atomic.Value
go func() {
for d := range disconnections {
if d.Error != nil {
log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
disconnected.Store(true)
}
}
}()
result := runSim(conf, ctx, sim, chunkCount)
if result.Error != nil {
t.Fatal(result.Error)
}
if yes, ok := disconnected.Load().(bool); ok && yes {
t.Fatal("disconnect events received")
}
log.Info("Simulation ended")
}
func runSim(conf *synctestConfig, ctx context.Context, sim *simulation.Simulation, chunkCount int) simulation.Result {
return sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error {
return sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) (err error) {
disconnected := watchDisconnections(ctx, sim)
defer func() {
if err != nil && disconnected.bool() {
err = errors.New("disconnect events received")
}
}()
nodeIDs := sim.UpNodeIDs()
for _, n := range nodeIDs {
//get the kademlia overlay address from this ID

6
swarm/network/stream/stream.go
View file

@ -939,16 +939,22 @@ It returns a map of node IDs with an array of string representations of Stream o
func (api *API) GetPeerSubscriptions() map[string][]string {
//create the empty map
pstreams := make(map[string][]string)
//iterate all streamer peers
api.streamer.peersMu.RLock()
defer api.streamer.peersMu.RUnlock()
for id, p := range api.streamer.peers {
var streams []string
//every peer has a map of stream servers
//every stream server represents a subscription
p.serverMu.RLock()
for s := range p.servers {
//append the string representation of the stream
//to the list for this peer
streams = append(streams, s.String())
}
p.serverMu.RUnlock()
//set the array of stream servers to the map
pstreams[id.String()] = streams
}

70
swarm/network/stream/streamer_test.go
View file

@ -41,10 +41,10 @@ import (
func TestStreamerSubscribe(t *testing.T) {
tester, streamer, _, teardown, err := newStreamerTester(nil)
defer teardown()
if err != nil {
t.Fatal(err)
}
defer teardown()
stream := NewStream("foo", "", true)
err = streamer.Subscribe(tester.Nodes[0].ID(), stream, NewRange(0, 0), Top)
@ -55,10 +55,10 @@ func TestStreamerSubscribe(t *testing.T) {
func TestStreamerRequestSubscription(t *testing.T) {
tester, streamer, _, teardown, err := newStreamerTester(nil)
defer teardown()
if err != nil {
t.Fatal(err)
}
defer teardown()
stream := NewStream("foo", "", false)
err = streamer.RequestSubscription(tester.Nodes[0].ID(), stream, &Range{}, Top)
@ -146,10 +146,10 @@ func (self *testServer) Close() {
func TestStreamerDownstreamSubscribeUnsubscribeMsgExchange(t *testing.T) {
tester, streamer, _, teardown, err := newStreamerTester(nil)
defer teardown()
if err != nil {
t.Fatal(err)
}
defer teardown()
streamer.RegisterClientFunc("foo", func(p *Peer, t string, live bool) (Client, error) {
return newTestClient(t), nil
@ -239,10 +239,10 @@ func TestStreamerDownstreamSubscribeUnsubscribeMsgExchange(t *testing.T) {
func TestStreamerUpstreamSubscribeUnsubscribeMsgExchange(t *testing.T) {
tester, streamer, _, teardown, err := newStreamerTester(nil)
defer teardown()
if err != nil {
t.Fatal(err)
}
defer teardown()
stream := NewStream("foo", "", false)
@ -306,10 +306,10 @@ func TestStreamerUpstreamSubscribeUnsubscribeMsgExchange(t *testing.T) {
func TestStreamerUpstreamSubscribeUnsubscribeMsgExchangeLive(t *testing.T) {
tester, streamer, _, teardown, err := newStreamerTester(nil)
defer teardown()
if err != nil {
t.Fatal(err)
}
defer teardown()
stream := NewStream("foo", "", true)
@ -372,10 +372,10 @@ func TestStreamerUpstreamSubscribeUnsubscribeMsgExchangeLive(t *testing.T) {
func TestStreamerUpstreamSubscribeErrorMsgExchange(t *testing.T) {
tester, streamer, _, teardown, err := newStreamerTester(nil)
defer teardown()
if err != nil {
t.Fatal(err)
}
defer teardown()
streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) {
return newTestServer(t, 0), nil
@ -416,10 +416,10 @@ func TestStreamerUpstreamSubscribeErrorMsgExchange(t *testing.T) {
func TestStreamerUpstreamSubscribeLiveAndHistory(t *testing.T) {
tester, streamer, _, teardown, err := newStreamerTester(nil)
defer teardown()
if err != nil {
t.Fatal(err)
}
defer teardown()
stream := NewStream("foo", "", true)
@ -479,10 +479,10 @@ func TestStreamerUpstreamSubscribeLiveAndHistory(t *testing.T) {
func TestStreamerDownstreamCorruptHashesMsgExchange(t *testing.T) {
tester, streamer, _, teardown, err := newStreamerTester(nil)
defer teardown()
if err != nil {
t.Fatal(err)
}
defer teardown()
stream := NewStream("foo", "", true)
@ -544,10 +544,10 @@ func TestStreamerDownstreamCorruptHashesMsgExchange(t *testing.T) {
func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) {
tester, streamer, _, teardown, err := newStreamerTester(nil)
defer teardown()
if err != nil {
t.Fatal(err)
}
defer teardown()
stream := NewStream("foo", "", true)
@ -643,10 +643,10 @@ func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) {
func TestStreamerRequestSubscriptionQuitMsgExchange(t *testing.T) {
tester, streamer, _, teardown, err := newStreamerTester(nil)
defer teardown()
if err != nil {
t.Fatal(err)
}
defer teardown()
streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) {
return newTestServer(t, 10), nil
@ -780,10 +780,10 @@ func TestMaxPeerServersWithUnsubscribe(t *testing.T) {
Syncing: SyncingDisabled,
MaxPeerServers: maxPeerServers,
})
defer teardown()
if err != nil {
t.Fatal(err)
}
defer teardown()
streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) {
return newTestServer(t, 0), nil
@ -854,10 +854,10 @@ func TestMaxPeerServersWithoutUnsubscribe(t *testing.T) {
tester, streamer, _, teardown, err := newStreamerTester(&RegistryOptions{
MaxPeerServers: maxPeerServers,
})
defer teardown()
if err != nil {
t.Fatal(err)
}
defer teardown()
streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) {
return newTestServer(t, 0), nil
@ -940,10 +940,10 @@ func TestHasPriceImplementation(t *testing.T) {
Retrieval: RetrievalDisabled,
Syncing: SyncingDisabled,
})
defer teardown()
if err != nil {
t.Fatal(err)
}
defer teardown()
if r.prices == nil {
t.Fatal("No prices implementation available for the stream protocol")
@ -1177,6 +1177,7 @@ starts the simulation, waits for SyncUpdateDelay in order to kick off
stream registration, then tests that there are subscriptions.
*/
func TestGetSubscriptionsRPC(t *testing.T) {
// arbitrarily set to 4
nodeCount := 4
// run with more nodes if `longrunning` flag is set
@ -1188,19 +1189,16 @@ func TestGetSubscriptionsRPC(t *testing.T) {
// holds the msg code for SubscribeMsg
var subscribeMsgCode uint64
var ok bool
var expectedMsgCount = 0
var expectedMsgCount counter
// this channel signalizes that the expected amount of subscriptiosn is done
allSubscriptionsDone := make(chan struct{})
lock := sync.RWMutex{}
// after the test, we need to reset the subscriptionFunc to the default
defer func() { subscriptionFunc = doRequestSubscription }()
// we use this subscriptionFunc for this test: just increases count and calls the actual subscription
subscriptionFunc = func(r *Registry, p *network.Peer, bin uint8, subs map[enode.ID]map[Stream]struct{}) bool {
lock.Lock()
expectedMsgCount++
lock.Unlock()
expectedMsgCount.inc()
doRequestSubscription(r, p, bin, subs)
return true
}
@ -1290,24 +1288,24 @@ func TestGetSubscriptionsRPC(t *testing.T) {
select {
case <-allSubscriptionsDone:
case <-ctx.Done():
t.Fatal("Context timed out")
return errors.New("Context timed out")
}
log.Debug("Expected message count: ", "expectedMsgCount", expectedMsgCount)
log.Debug("Expected message count: ", "expectedMsgCount", expectedMsgCount.count())
//now iterate again, this time we call each node via RPC to get its subscriptions
realCount := 0
for _, node := range nodes {
//create rpc client
client, err := node.Client()
if err != nil {
t.Fatalf("create node 1 rpc client fail: %v", err)
return fmt.Errorf("create node 1 rpc client fail: %v", err)
}
//ask it for subscriptions
pstreams := make(map[string][]string)
err = client.Call(&pstreams, "stream_getPeerSubscriptions")
if err != nil {
t.Fatal(err)
return fmt.Errorf("client call stream_getPeerSubscriptions: %v", err)
}
//length of the subscriptions can not be smaller than number of peers
log.Debug("node subscriptions", "node", node.String())
@ -1324,8 +1322,9 @@ func TestGetSubscriptionsRPC(t *testing.T) {
}
}
// every node is mutually subscribed to each other, so the actual count is half of it
if realCount/2 != expectedMsgCount {
return fmt.Errorf("Real subscriptions and expected amount don't match; real: %d, expected: %d", realCount/2, expectedMsgCount)
emc := expectedMsgCount.count()
if realCount/2 != emc {
return fmt.Errorf("Real subscriptions and expected amount don't match; real: %d, expected: %d", realCount/2, emc)
}
return nil
})
@ -1333,3 +1332,26 @@ func TestGetSubscriptionsRPC(t *testing.T) {
t.Fatal(result.Error)
}
}
// counter is used to concurrently increment
// and read an integer value.
type counter struct {
v int
mu sync.RWMutex
}
// Increment the counter.
func (c *counter) inc() {
c.mu.Lock()
defer c.mu.Unlock()
c.v++
}
// Read the counter value.
func (c *counter) count() int {
c.mu.RLock()
defer c.mu.RUnlock()
return c.v
}

65
swarm/network/stream/syncer_test.go
View file

@ -22,8 +22,8 @@ import (
"fmt"
"io/ioutil"
"math"
"os"
"sync"
"sync/atomic"
"testing"
"time"
@ -44,9 +44,15 @@ const dataChunkCount = 200
func TestSyncerSimulation(t *testing.T) {
testSyncBetweenNodes(t, 2, dataChunkCount, true, 1)
testSyncBetweenNodes(t, 4, dataChunkCount, true, 1)
testSyncBetweenNodes(t, 8, dataChunkCount, true, 1)
testSyncBetweenNodes(t, 16, dataChunkCount, true, 1)
// This test uses much more memory when running with
// race detector. Allow it to finish successfully by
// reducing its scope, and still check for data races
// with the smallest number of nodes.
if !raceTest {
testSyncBetweenNodes(t, 4, dataChunkCount, true, 1)
testSyncBetweenNodes(t, 8, dataChunkCount, true, 1)
testSyncBetweenNodes(t, 16, dataChunkCount, true, 1)
}
}
func createMockStore(globalStore mock.GlobalStorer, id enode.ID, addr *network.BzzAddr) (lstore storage.ChunkStore, datadir string, err error) {
@ -80,7 +86,23 @@ func testSyncBetweenNodes(t *testing.T, nodes, chunkCount int, skipCheck bool, p
return nil, nil, err
}
r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{
var dir string
var store *state.DBStore
if raceTest {
// Use on-disk DBStore to reduce memory consumption in race tests.
dir, err = ioutil.TempDir("", "swarm-stream-")
if err != nil {
return nil, nil, err
}
store, err = state.NewDBStore(dir)
if err != nil {
return nil, nil, err
}
} else {
store = state.NewInmemoryStore()
}
r := NewRegistry(addr.ID(), delivery, netStore, store, &RegistryOptions{
Retrieval: RetrievalDisabled,
Syncing: SyncingAutoSubscribe,
SkipCheck: skipCheck,
@ -89,6 +111,9 @@ func testSyncBetweenNodes(t *testing.T, nodes, chunkCount int, skipCheck bool, p
cleanup = func() {
r.Close()
clean()
if dir != "" {
os.RemoveAll(dir)
}
}
return r, cleanup, nil
@ -114,26 +139,10 @@ func testSyncBetweenNodes(t *testing.T, nodes, chunkCount int, skipCheck bool, p
nodeIndex[id] = i
}
disconnections := sim.PeerEvents(
context.Background(),
sim.NodeIDs(),
simulation.NewPeerEventsFilter().Drop(),
)
var disconnected atomic.Value
go func() {
for d := range disconnections {
if d.Error != nil {
log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
disconnected.Store(true)
}
}
}()
disconnected := watchDisconnections(ctx, sim)
defer func() {
if err != nil {
if yes, ok := disconnected.Load().(bool); ok && yes {
err = errors.New("disconnect events received")
}
if err != nil && disconnected.bool() {
err = errors.New("disconnect events received")
}
}()
@ -142,7 +151,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, chunkCount int, skipCheck bool, p
id := nodeIDs[j]
client, err := sim.Net.GetNode(id).Client()
if err != nil {
t.Fatal(err)
return fmt.Errorf("node %s client: %v", id, err)
}
sid := nodeIDs[j+1]
client.CallContext(ctx, nil, "stream_subscribeStream", sid, NewStream("SYNC", FormatSyncBinKey(1), false), NewRange(0, 0), Top)
@ -158,7 +167,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, chunkCount int, skipCheck bool, p
size := chunkCount * chunkSize
_, wait, err := fileStore.Store(ctx, testutil.RandomReader(j, size), int64(size), false)
if err != nil {
t.Fatal(err.Error())
return fmt.Errorf("fileStore.Store: %v", err)
}
wait(ctx)
}
@ -273,7 +282,7 @@ func TestSameVersionID(t *testing.T) {
//the peers should connect, thus getting the peer should not return nil
if registry.getPeer(nodes[1]) == nil {
t.Fatal("Expected the peer to not be nil, but it is")
return errors.New("Expected the peer to not be nil, but it is")
}
return nil
})
@ -338,7 +347,7 @@ func TestDifferentVersionID(t *testing.T) {
//getting the other peer should fail due to the different version numbers
if registry.getPeer(nodes[1]) != nil {
t.Fatal("Expected the peer to be nil, but it is not")
return errors.New("Expected the peer to be nil, but it is not")
}
return nil
})

44
swarm/network/stream/visualized_snapshot_sync_sim_test.go
View file

@ -66,31 +66,6 @@ func setupSim(serviceMap map[string]simulation.ServiceFunc) (int, int, *simulati
return nodeCount, chunkCount, sim
}
//watch for disconnections and wait for healthy
func watchSim(sim *simulation.Simulation) (context.Context, context.CancelFunc) {
ctx, cancelSimRun := context.WithTimeout(context.Background(), 1*time.Minute)
if _, err := sim.WaitTillHealthy(ctx); err != nil {
panic(err)
}
disconnections := sim.PeerEvents(
context.Background(),
sim.NodeIDs(),
simulation.NewPeerEventsFilter().Drop(),
)
go func() {
for d := range disconnections {
log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
panic("unexpected disconnect")
cancelSimRun()
}
}()
return ctx, cancelSimRun
}
//This test requests bogus hashes into the network
func TestNonExistingHashesWithServer(t *testing.T) {
@ -102,19 +77,25 @@ func TestNonExistingHashesWithServer(t *testing.T) {
panic(err)
}
ctx, cancelSimRun := watchSim(sim)
defer cancelSimRun()
//in order to get some meaningful visualization, it is beneficial
//to define a minimum duration of this test
testDuration := 20 * time.Second
result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error {
result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) (err error) {
disconnected := watchDisconnections(ctx, sim)
defer func() {
if err != nil {
if yes, ok := disconnected.Load().(bool); ok && yes {
err = errors.New("disconnect events received")
}
}
}()
//check on the node's FileStore (netstore)
id := sim.Net.GetRandomUpNode().ID()
item, ok := sim.NodeItem(id, bucketKeyFileStore)
if !ok {
t.Fatalf("No filestore")
return errors.New("No filestore")
}
fileStore := item.(*storage.FileStore)
//create a bogus hash
@ -213,9 +194,6 @@ func TestSnapshotSyncWithServer(t *testing.T) {
panic(err)
}
ctx, cancelSimRun := watchSim(sim)
defer cancelSimRun()
//run the sim
result := runSim(conf, ctx, sim, chunkCount)

281
swarm/pss/keystore.go Normal file
View file

@ -0,0 +1,281 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package pss
import (
"crypto/ecdsa"
"errors"
"fmt"
"sync"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/swarm/log"
whisper "github.com/ethereum/go-ethereum/whisper/whisperv6"
)
type KeyStore struct {
w *whisper.Whisper // key and encryption backend
mx sync.RWMutex
pubKeyPool map[string]map[Topic]*pssPeer // mapping of hex public keys to peer address by topic.
symKeyPool map[string]map[Topic]*pssPeer // mapping of symkeyids to peer address by topic.
symKeyDecryptCache []*string // fast lookup of symkeys recently used for decryption; last used is on top of stack
symKeyDecryptCacheCursor int // modular cursor pointing to last used, wraps on symKeyDecryptCache array
}
func loadKeyStore() *KeyStore {
return &KeyStore{
w: whisper.New(&whisper.DefaultConfig),
pubKeyPool: make(map[string]map[Topic]*pssPeer),
symKeyPool: make(map[string]map[Topic]*pssPeer),
symKeyDecryptCache: make([]*string, defaultSymKeyCacheCapacity),
}
}
func (ks *KeyStore) isSymKeyStored(key string) bool {
ks.mx.RLock()
defer ks.mx.RUnlock()
var ok bool
_, ok = ks.symKeyPool[key]
return ok
}
func (ks *KeyStore) isPubKeyStored(key string) bool {
ks.mx.RLock()
defer ks.mx.RUnlock()
var ok bool
_, ok = ks.pubKeyPool[key]
return ok
}
func (ks *KeyStore) getPeerSym(symkeyid string, topic Topic) (*pssPeer, bool) {
ks.mx.RLock()
defer ks.mx.RUnlock()
psp, ok := ks.symKeyPool[symkeyid][topic]
return psp, ok
}
func (ks *KeyStore) getPeerPub(pubkeyid string, topic Topic) (*pssPeer, bool) {
ks.mx.RLock()
defer ks.mx.RUnlock()
psp, ok := ks.pubKeyPool[pubkeyid][topic]
return psp, ok
}
// Links a peer ECDSA public key to a topic.
// This is required for asymmetric message exchange on the given topic.
// The value in `address` will be used as a routing hint for the public key / topic association.
func (ks *KeyStore) SetPeerPublicKey(pubkey *ecdsa.PublicKey, topic Topic, address PssAddress) error {
if err := validateAddress(address); err != nil {
return err
}
pubkeybytes := crypto.FromECDSAPub(pubkey)
if len(pubkeybytes) == 0 {
return fmt.Errorf("invalid public key: %v", pubkey)
}
pubkeyid := common.ToHex(pubkeybytes)
psp := &pssPeer{
address: address,
}
ks.mx.Lock()
if _, ok := ks.pubKeyPool[pubkeyid]; !ok {
ks.pubKeyPool[pubkeyid] = make(map[Topic]*pssPeer)
}
ks.pubKeyPool[pubkeyid][topic] = psp
ks.mx.Unlock()
log.Trace("added pubkey", "pubkeyid", pubkeyid, "topic", topic, "address", address)
return nil
}
// adds a symmetric key to the pss key pool, and optionally adds the key to the
// collection of keys used to attempt symmetric decryption of incoming messages
func (ks *KeyStore) addSymmetricKeyToPool(keyid string, topic Topic, address PssAddress, addtocache bool, protected bool) {
psp := &pssPeer{
address: address,
protected: protected,
}
ks.mx.Lock()
if _, ok := ks.symKeyPool[keyid]; !ok {
ks.symKeyPool[keyid] = make(map[Topic]*pssPeer)
}
ks.symKeyPool[keyid][topic] = psp
ks.mx.Unlock()
if addtocache {
ks.symKeyDecryptCacheCursor++
ks.symKeyDecryptCache[ks.symKeyDecryptCacheCursor%cap(ks.symKeyDecryptCache)] = &keyid
}
}
// Returns all recorded topic and address combination for a specific public key
func (ks *KeyStore) GetPublickeyPeers(keyid string) (topic []Topic, address []PssAddress, err error) {
ks.mx.RLock()
defer ks.mx.RUnlock()
for t, peer := range ks.pubKeyPool[keyid] {
topic = append(topic, t)
address = append(address, peer.address)
}
return topic, address, nil
}
func (ks *KeyStore) getPeerAddress(keyid string, topic Topic) (PssAddress, error) {
ks.mx.RLock()
defer ks.mx.RUnlock()
if peers, ok := ks.pubKeyPool[keyid]; ok {
if t, ok := peers[topic]; ok {
return t.address, nil
}
}
return nil, fmt.Errorf("peer with pubkey %s, topic %x not found", keyid, topic)
}
// Attempt to decrypt, validate and unpack a symmetrically encrypted message.
// If successful, returns the unpacked whisper ReceivedMessage struct
// encapsulating the decrypted message, and the whisper backend id
// of the symmetric key used to decrypt the message.
// It fails if decryption of the message fails or if the message is corrupted.
func (ks *KeyStore) processSym(envelope *whisper.Envelope) (*whisper.ReceivedMessage, string, PssAddress, error) {
metrics.GetOrRegisterCounter("pss.process.sym", nil).Inc(1)
for i := ks.symKeyDecryptCacheCursor; i > ks.symKeyDecryptCacheCursor-cap(ks.symKeyDecryptCache) && i > 0; i-- {
symkeyid := ks.symKeyDecryptCache[i%cap(ks.symKeyDecryptCache)]
symkey, err := ks.w.GetSymKey(*symkeyid)
if err != nil {
continue
}
recvmsg, err := envelope.OpenSymmetric(symkey)
if err != nil {
continue
}
if !recvmsg.ValidateAndParse() {
return nil, "", nil, errors.New("symmetrically encrypted message has invalid signature or is corrupt")
}
var from PssAddress
ks.mx.RLock()
if ks.symKeyPool[*symkeyid][Topic(envelope.Topic)] != nil {
from = ks.symKeyPool[*symkeyid][Topic(envelope.Topic)].address
}
ks.mx.RUnlock()
ks.symKeyDecryptCacheCursor++
ks.symKeyDecryptCache[ks.symKeyDecryptCacheCursor%cap(ks.symKeyDecryptCache)] = symkeyid
return recvmsg, *symkeyid, from, nil
}
return nil, "", nil, errors.New("could not decrypt message")
}
// Attempt to decrypt, validate and unpack an asymmetrically encrypted message.
// If successful, returns the unpacked whisper ReceivedMessage struct
// encapsulating the decrypted message, and the byte representation of
// the public key used to decrypt the message.
// It fails if decryption of message fails, or if the message is corrupted.
func (ks *Pss) processAsym(envelope *whisper.Envelope) (*whisper.ReceivedMessage, string, PssAddress, error) {
metrics.GetOrRegisterCounter("pss.process.asym", nil).Inc(1)
recvmsg, err := envelope.OpenAsymmetric(ks.privateKey)
if err != nil {
return nil, "", nil, fmt.Errorf("could not decrypt message: %s", err)
}
// check signature (if signed), strip padding
if !recvmsg.ValidateAndParse() {
return nil, "", nil, errors.New("invalid message")
}
pubkeyid := common.ToHex(crypto.FromECDSAPub(recvmsg.Src))
var from PssAddress
ks.mx.RLock()
if ks.pubKeyPool[pubkeyid][Topic(envelope.Topic)] != nil {
from = ks.pubKeyPool[pubkeyid][Topic(envelope.Topic)].address
}
ks.mx.RUnlock()
return recvmsg, pubkeyid, from, nil
}
// Symkey garbage collection
// a key is removed if:
// - it is not marked as protected
// - it is not in the incoming decryption cache
func (ks *Pss) cleanKeys() (count int) {
for keyid, peertopics := range ks.symKeyPool {
var expiredtopics []Topic
for topic, psp := range peertopics {
if psp.protected {
continue
}
var match bool
for i := ks.symKeyDecryptCacheCursor; i > ks.symKeyDecryptCacheCursor-cap(ks.symKeyDecryptCache) && i > 0; i-- {
cacheid := ks.symKeyDecryptCache[i%cap(ks.symKeyDecryptCache)]
if *cacheid == keyid {
match = true
}
}
if !match {
expiredtopics = append(expiredtopics, topic)
}
}
for _, topic := range expiredtopics {
ks.mx.Lock()
delete(ks.symKeyPool[keyid], topic)
log.Trace("symkey cleanup deletion", "symkeyid", keyid, "topic", topic, "val", ks.symKeyPool[keyid])
ks.mx.Unlock()
count++
}
}
return count
}
// Automatically generate a new symkey for a topic and address hint
func (ks *KeyStore) GenerateSymmetricKey(topic Topic, address PssAddress, addToCache bool) (string, error) {
keyid, err := ks.w.GenerateSymKey()
if err == nil {
ks.addSymmetricKeyToPool(keyid, topic, address, addToCache, false)
}
return keyid, err
}
// Returns a symmetric key byte sequence stored in the whisper backend by its unique id.
// Passes on the error value from the whisper backend.
func (ks *KeyStore) GetSymmetricKey(symkeyid string) ([]byte, error) {
return ks.w.GetSymKey(symkeyid)
}
// Links a peer symmetric key (arbitrary byte sequence) to a topic.
//
// This is required for symmetrically encrypted message exchange on the given topic.
//
// The key is stored in the whisper backend.
//
// If addtocache is set to true, the key will be added to the cache of keys
// used to attempt symmetric decryption of incoming messages.
//
// Returns a string id that can be used to retrieve the key bytes
// from the whisper backend (see pss.GetSymmetricKey())
func (ks *KeyStore) SetSymmetricKey(key []byte, topic Topic, address PssAddress, addtocache bool) (string, error) {
if err := validateAddress(address); err != nil {
return "", err
}
return ks.setSymmetricKey(key, topic, address, addtocache, true)
}
func (ks *KeyStore) setSymmetricKey(key []byte, topic Topic, address PssAddress, addtocache bool, protected bool) (string, error) {
keyid, err := ks.w.AddSymKeyDirect(key)
if err == nil {
ks.addSymmetricKeyToPool(keyid, topic, address, addtocache, protected)
}
return keyid, err
}

8
swarm/pss/protocol.go
View file

@ -241,20 +241,16 @@ func (p *Protocol) AddPeer(peer *p2p.Peer, topic Topic, asymmetric bool, key str
rw.sendFunc = p.Pss.SendSym
}
if asymmetric {
p.Pss.pubKeyPoolMu.Lock()
if _, ok := p.Pss.pubKeyPool[key]; !ok {
if !p.Pss.isPubKeyStored(key) {
return nil, fmt.Errorf("asym key does not exist: %s", key)
}
p.Pss.pubKeyPoolMu.Unlock()
p.RWPoolMu.Lock()
p.pubKeyRWPool[key] = rw
p.RWPoolMu.Unlock()
} else {
p.Pss.symKeyPoolMu.Lock()
if _, ok := p.Pss.symKeyPool[key]; !ok {
if !p.Pss.isSymKeyStored(key) {
return nil, fmt.Errorf("symkey does not exist: %s", key)
}
p.Pss.symKeyPoolMu.Unlock()
p.RWPoolMu.Lock()
p.symKeyRWPool[key] = rw
p.RWPoolMu.Unlock()

257
swarm/pss/pss.go
View file

@ -112,10 +112,11 @@ func (params *PssParams) WithPrivateKey(privatekey *ecdsa.PrivateKey) *PssParams
//
// Implements node.Service
type Pss struct {
*network.Kademlia // we can get the Kademlia address from this
privateKey *ecdsa.PrivateKey // pss can have it's own independent key
w *whisper.Whisper // key and encryption backend
auxAPIs []rpc.API // builtins (handshake, test) can add APIs
*network.Kademlia // we can get the Kademlia address from this
*KeyStore
privateKey *ecdsa.PrivateKey // pss can have it's own independent key
auxAPIs []rpc.API // builtins (handshake, test) can add APIs
// sending and forwarding
fwdPool map[string]*protocols.Peer // keep track of all peers sitting on the pssmsg routing layer
@ -128,15 +129,6 @@ type Pss struct {
capstring string
outbox chan *PssMsg
// keys and peers
pubKeyPool map[string]map[Topic]*pssPeer // mapping of hex public keys to peer address by topic.
pubKeyPoolMu sync.RWMutex
symKeyPool map[string]map[Topic]*pssPeer // mapping of symkeyids to peer address by topic.
symKeyPoolMu sync.RWMutex
symKeyDecryptCache []*string // fast lookup of symkeys recently used for decryption; last used is on top of stack
symKeyDecryptCacheCursor int // modular cursor pointing to last used, wraps on symKeyDecryptCache array
symKeyDecryptCacheCapacity int // max amount of symkeys to keep.
// message handling
handlers map[Topic]map[*handler]bool // topic and version based pss payload handlers. See pss.Handle()
handlersMu sync.RWMutex
@ -165,9 +157,10 @@ func NewPss(k *network.Kademlia, params *PssParams) (*Pss, error) {
Version: pssVersion,
}
ps := &Pss{
Kademlia: k,
Kademlia: k,
KeyStore: loadKeyStore(),
privateKey: params.privateKey,
w: whisper.New(&whisper.DefaultConfig),
quitC: make(chan struct{}),
fwdPool: make(map[string]*protocols.Peer),
@ -178,11 +171,6 @@ func NewPss(k *network.Kademlia, params *PssParams) (*Pss, error) {
capstring: cap.String(),
outbox: make(chan *PssMsg, defaultOutboxCapacity),
pubKeyPool: make(map[string]map[Topic]*pssPeer),
symKeyPool: make(map[string]map[Topic]*pssPeer),
symKeyDecryptCache: make([]*string, params.SymKeyCacheCapacity),
symKeyDecryptCacheCapacity: params.SymKeyCacheCapacity,
handlers: make(map[Topic]map[*handler]bool),
topicHandlerCaps: make(map[Topic]*handlerCaps),
@ -539,227 +527,6 @@ func (p *Pss) isSelfPossibleRecipient(msg *PssMsg, prox bool) bool {
return depth <= po
}
/////////////////////////////////////////////////////////////////////
// SECTION: Encryption
/////////////////////////////////////////////////////////////////////
// Links a peer ECDSA public key to a topic
//
// This is required for asymmetric message exchange
// on the given topic
//
// The value in `address` will be used as a routing hint for the
// public key / topic association
func (p *Pss) SetPeerPublicKey(pubkey *ecdsa.PublicKey, topic Topic, address PssAddress) error {
if err := validateAddress(address); err != nil {
return err
}
pubkeybytes := crypto.FromECDSAPub(pubkey)
if len(pubkeybytes) == 0 {
return fmt.Errorf("invalid public key: %v", pubkey)
}
pubkeyid := common.ToHex(pubkeybytes)
psp := &pssPeer{
address: address,
}
p.pubKeyPoolMu.Lock()
if _, ok := p.pubKeyPool[pubkeyid]; !ok {
p.pubKeyPool[pubkeyid] = make(map[Topic]*pssPeer)
}
p.pubKeyPool[pubkeyid][topic] = psp
p.pubKeyPoolMu.Unlock()
log.Trace("added pubkey", "pubkeyid", pubkeyid, "topic", topic, "address", address)
return nil
}
// Automatically generate a new symkey for a topic and address hint
func (p *Pss) GenerateSymmetricKey(topic Topic, address PssAddress, addToCache bool) (string, error) {
keyid, err := p.w.GenerateSymKey()
if err != nil {
return "", err
}
p.addSymmetricKeyToPool(keyid, topic, address, addToCache, false)
return keyid, nil
}
// Links a peer symmetric key (arbitrary byte sequence) to a topic
//
// This is required for symmetrically encrypted message exchange
// on the given topic
//
// The key is stored in the whisper backend.
//
// If addtocache is set to true, the key will be added to the cache of keys
// used to attempt symmetric decryption of incoming messages.
//
// Returns a string id that can be used to retrieve the key bytes
// from the whisper backend (see pss.GetSymmetricKey())
func (p *Pss) SetSymmetricKey(key []byte, topic Topic, address PssAddress, addtocache bool) (string, error) {
if err := validateAddress(address); err != nil {
return "", err
}
return p.setSymmetricKey(key, topic, address, addtocache, true)
}
func (p *Pss) setSymmetricKey(key []byte, topic Topic, address PssAddress, addtocache bool, protected bool) (string, error) {
keyid, err := p.w.AddSymKeyDirect(key)
if err != nil {
return "", err
}
p.addSymmetricKeyToPool(keyid, topic, address, addtocache, protected)
return keyid, nil
}
// adds a symmetric key to the pss key pool, and optionally adds the key
// to the collection of keys used to attempt symmetric decryption of
// incoming messages
func (p *Pss) addSymmetricKeyToPool(keyid string, topic Topic, address PssAddress, addtocache bool, protected bool) {
psp := &pssPeer{
address: address,
protected: protected,
}
p.symKeyPoolMu.Lock()
if _, ok := p.symKeyPool[keyid]; !ok {
p.symKeyPool[keyid] = make(map[Topic]*pssPeer)
}
p.symKeyPool[keyid][topic] = psp
p.symKeyPoolMu.Unlock()
if addtocache {
p.symKeyDecryptCacheCursor++
p.symKeyDecryptCache[p.symKeyDecryptCacheCursor%cap(p.symKeyDecryptCache)] = &keyid
}
key, _ := p.GetSymmetricKey(keyid)
log.Trace("added symkey", "symkeyid", keyid, "symkey", common.ToHex(key), "topic", topic, "address", address, "cache", addtocache)
}
// Returns a symmetric key byte seqyence stored in the whisper backend
// by its unique id
//
// Passes on the error value from the whisper backend
func (p *Pss) GetSymmetricKey(symkeyid string) ([]byte, error) {
symkey, err := p.w.GetSymKey(symkeyid)
if err != nil {
return nil, err
}
return symkey, nil
}
// Returns all recorded topic and address combination for a specific public key
func (p *Pss) GetPublickeyPeers(keyid string) (topic []Topic, address []PssAddress, err error) {
p.pubKeyPoolMu.RLock()
defer p.pubKeyPoolMu.RUnlock()
for t, peer := range p.pubKeyPool[keyid] {
topic = append(topic, t)
address = append(address, peer.address)
}
return topic, address, nil
}
func (p *Pss) getPeerAddress(keyid string, topic Topic) (PssAddress, error) {
p.pubKeyPoolMu.RLock()
defer p.pubKeyPoolMu.RUnlock()
if peers, ok := p.pubKeyPool[keyid]; ok {
if t, ok := peers[topic]; ok {
return t.address, nil
}
}
return nil, fmt.Errorf("peer with pubkey %s, topic %x not found", keyid, topic)
}
// Attempt to decrypt, validate and unpack a
// symmetrically encrypted message
// If successful, returns the unpacked whisper ReceivedMessage struct
// encapsulating the decrypted message, and the whisper backend id
// of the symmetric key used to decrypt the message.
// It fails if decryption of the message fails or if the message is corrupted
func (p *Pss) processSym(envelope *whisper.Envelope) (*whisper.ReceivedMessage, string, PssAddress, error) {
metrics.GetOrRegisterCounter("pss.process.sym", nil).Inc(1)
for i := p.symKeyDecryptCacheCursor; i > p.symKeyDecryptCacheCursor-cap(p.symKeyDecryptCache) && i > 0; i-- {
symkeyid := p.symKeyDecryptCache[i%cap(p.symKeyDecryptCache)]
symkey, err := p.w.GetSymKey(*symkeyid)
if err != nil {
continue
}
recvmsg, err := envelope.OpenSymmetric(symkey)
if err != nil {
continue
}
if !recvmsg.ValidateAndParse() {
return nil, "", nil, fmt.Errorf("symmetrically encrypted message has invalid signature or is corrupt")
}
p.symKeyPoolMu.Lock()
from := p.symKeyPool[*symkeyid][Topic(envelope.Topic)].address
p.symKeyPoolMu.Unlock()
p.symKeyDecryptCacheCursor++
p.symKeyDecryptCache[p.symKeyDecryptCacheCursor%cap(p.symKeyDecryptCache)] = symkeyid
return recvmsg, *symkeyid, from, nil
}
return nil, "", nil, fmt.Errorf("could not decrypt message")
}
// Attempt to decrypt, validate and unpack an
// asymmetrically encrypted message
// If successful, returns the unpacked whisper ReceivedMessage struct
// encapsulating the decrypted message, and the byte representation of
// the public key used to decrypt the message.
// It fails if decryption of message fails, or if the message is corrupted
func (p *Pss) processAsym(envelope *whisper.Envelope) (*whisper.ReceivedMessage, string, PssAddress, error) {
metrics.GetOrRegisterCounter("pss.process.asym", nil).Inc(1)
recvmsg, err := envelope.OpenAsymmetric(p.privateKey)
if err != nil {
return nil, "", nil, fmt.Errorf("could not decrypt message: %s", err)
}
// check signature (if signed), strip padding
if !recvmsg.ValidateAndParse() {
return nil, "", nil, fmt.Errorf("invalid message")
}
pubkeyid := common.ToHex(crypto.FromECDSAPub(recvmsg.Src))
var from PssAddress
p.pubKeyPoolMu.Lock()
if p.pubKeyPool[pubkeyid][Topic(envelope.Topic)] != nil {
from = p.pubKeyPool[pubkeyid][Topic(envelope.Topic)].address
}
p.pubKeyPoolMu.Unlock()
return recvmsg, pubkeyid, from, nil
}
// Symkey garbage collection
// a key is removed if:
// - it is not marked as protected
// - it is not in the incoming decryption cache
func (p *Pss) cleanKeys() (count int) {
for keyid, peertopics := range p.symKeyPool {
var expiredtopics []Topic
for topic, psp := range peertopics {
if psp.protected {
continue
}
var match bool
for i := p.symKeyDecryptCacheCursor; i > p.symKeyDecryptCacheCursor-cap(p.symKeyDecryptCache) && i > 0; i-- {
cacheid := p.symKeyDecryptCache[i%cap(p.symKeyDecryptCache)]
if *cacheid == keyid {
match = true
}
}
if !match {
expiredtopics = append(expiredtopics, topic)
}
}
for _, topic := range expiredtopics {
p.symKeyPoolMu.Lock()
delete(p.symKeyPool[keyid], topic)
log.Trace("symkey cleanup deletion", "symkeyid", keyid, "topic", topic, "val", p.symKeyPool[keyid])
p.symKeyPoolMu.Unlock()
count++
}
}
return
}
/////////////////////////////////////////////////////////////////////
// SECTION: Message sending
/////////////////////////////////////////////////////////////////////
@ -817,9 +584,7 @@ func (p *Pss) SendSym(symkeyid string, topic Topic, msg []byte) error {
if err != nil {
return fmt.Errorf("missing valid send symkey %s: %v", symkeyid, err)
}
p.symKeyPoolMu.Lock()
psp, ok := p.symKeyPool[symkeyid][topic]
p.symKeyPoolMu.Unlock()
psp, ok := p.getPeerSym(symkeyid, topic)
if !ok {
return fmt.Errorf("invalid topic '%s' for symkey '%s'", topic.String(), symkeyid)
}
@ -833,9 +598,7 @@ func (p *Pss) SendAsym(pubkeyid string, topic Topic, msg []byte) error {
if _, err := crypto.UnmarshalPubkey(common.FromHex(pubkeyid)); err != nil {
return fmt.Errorf("Cannot unmarshal pubkey: %x", pubkeyid)
}
p.pubKeyPoolMu.Lock()
psp, ok := p.pubKeyPool[pubkeyid][topic]
p.pubKeyPoolMu.Unlock()
psp, ok := p.getPeerPub(pubkeyid, topic)
if !ok {
return fmt.Errorf("invalid topic '%s' for pubkey '%s'", topic.String(), pubkeyid)
}

13
swarm/state/dbstore.go
View file

@ -88,18 +88,15 @@ func (s *DBStore) Get(key string, i interface{}) (err error) {
// Put stores an object that implements Binary for a specific key.
func (s *DBStore) Put(key string, i interface{}) (err error) {
var bytes []byte
marshaler, ok := i.(encoding.BinaryMarshaler)
if !ok {
if bytes, err = json.Marshal(i); err != nil {
return err
}
} else {
if marshaler, ok := i.(encoding.BinaryMarshaler); ok {
if bytes, err = marshaler.MarshalBinary(); err != nil {
return err
}
} else {
if bytes, err = json.Marshal(i); err != nil {
return err
}
}
return s.db.Put([]byte(key), bytes, nil)
}

26
swarm/storage/filestore.go
View file

@ -20,6 +20,7 @@ import (
"context"
"io"
"sort"
"sync"
)
/*
@ -101,38 +102,45 @@ func (f *FileStore) HashSize() int {
// GetAllReferences is a public API. This endpoint returns all chunk hashes (only) for a given file
func (f *FileStore) GetAllReferences(ctx context.Context, data io.Reader, toEncrypt bool) (addrs AddressCollection, err error) {
// create a special kind of putter, which only will store the references
putter := &HashExplorer{
putter := &hashExplorer{
hasherStore: NewHasherStore(f.ChunkStore, f.hashFunc, toEncrypt),
References: make([]Reference, 0),
}
// do the actual splitting anyway, no way around it
_, _, err = PyramidSplit(ctx, data, putter, putter)
_, wait, err := PyramidSplit(ctx, data, putter, putter)
if err != nil {
return nil, err
}
// wait for splitting to be complete and all chunks processed
err = wait(ctx)
if err != nil {
return nil, err
}
// collect all references
addrs = NewAddressCollection(0)
for _, ref := range putter.References {
for _, ref := range putter.references {
addrs = append(addrs, Address(ref))
}
sort.Sort(addrs)
return addrs, nil
}
// HashExplorer is a special kind of putter which will only store chunk references
type HashExplorer struct {
// hashExplorer is a special kind of putter which will only store chunk references
type hashExplorer struct {
*hasherStore
References []Reference
references []Reference
lock sync.Mutex
}
// HashExplorer's Put will add just the chunk hashes to its `References`
func (he *HashExplorer) Put(ctx context.Context, chunkData ChunkData) (Reference, error) {
func (he *hashExplorer) Put(ctx context.Context, chunkData ChunkData) (Reference, error) {
// Need to do the actual Put, which returns the references
ref, err := he.hasherStore.Put(ctx, chunkData)
if err != nil {
return nil, err
}
// internally store the reference
he.References = append(he.References, ref)
he.lock.Lock()
he.references = append(he.references, ref)
he.lock.Unlock()
return ref, nil
}

9
swarm/storage/ldbstore.go
View file

@ -317,7 +317,6 @@ func decodeData(addr Address, data []byte) (*chunk, error) {
}
func (s *LDBStore) collectGarbage() error {
// prevent duplicate gc from starting when one is already running
select {
case <-s.gc.runC:
@ -335,7 +334,6 @@ func (s *LDBStore) collectGarbage() error {
s.startGC(int(entryCnt))
log.Debug("collectGarbage", "target", s.gc.target, "entryCnt", entryCnt)
var totalDeleted int
for s.gc.count < s.gc.target {
it := s.db.NewIterator()
ok := it.Seek([]byte{keyGCIdx})
@ -371,15 +369,15 @@ func (s *LDBStore) collectGarbage() error {
}
s.writeBatch(s.gc.batch, wEntryCnt)
log.Trace("garbage collect batch done", "batch", singleIterationCount, "total", s.gc.count)
s.lock.Unlock()
it.Release()
log.Trace("garbage collect batch done", "batch", singleIterationCount, "total", s.gc.count)
}
s.gc.runC <- struct{}{}
metrics.GetOrRegisterCounter("ldbstore.collectgarbage.delete", nil).Inc(int64(s.gc.count))
log.Debug("garbage collect done", "c", s.gc.count)
s.gc.runC <- struct{}{}
metrics.GetOrRegisterCounter("ldbstore.collectgarbage.delete", nil).Inc(int64(totalDeleted))
return nil
}
@ -1049,7 +1047,6 @@ func (s *LDBStore) Close() {
s.lock.Unlock()
// force writing out current batch
s.writeCurrentBatch()
close(s.batchesC)
s.db.Close()
}

20
swarm/storage/netstore.go
View file

@ -128,7 +128,25 @@ func (n *NetStore) FetchFunc(ctx context.Context, ref Address) func(context.Cont
func (n *NetStore) Close() {
close(n.closeC)
n.store.Close()
// TODO: loop through fetchers to cancel them
wg := sync.WaitGroup{}
for _, key := range n.fetchers.Keys() {
if f, ok := n.fetchers.Get(key); ok {
if fetch, ok := f.(*fetcher); ok {
wg.Add(1)
go func(fetch *fetcher) {
defer wg.Done()
fetch.cancel()
select {
case <-fetch.deliveredC:
case <-fetch.cancelledC:
}
}(fetch)
}
}
}
wg.Wait()
}
// get attempts at retrieving the chunk from LocalStore

154
swarm/swarm.go
View file

@ -79,7 +79,7 @@ type Swarm struct {
swap *swap.Swap
stateStore *state.DBStore
accountingMetrics *protocols.AccountingMetrics
startTime time.Time
cleanupFuncs []func() error
tracerClose io.Closer
}
@ -106,9 +106,10 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e
}
self = &Swarm{
config: config,
backend: backend,
privateKey: config.ShiftPrivateKey(),
config: config,
backend: backend,
privateKey: config.ShiftPrivateKey(),
cleanupFuncs: []func() error{},
}
log.Debug("Setting up Swarm service components")
@ -343,51 +344,51 @@ Start is called when the stack is started
* TODO: start subservices like sword, swear, swarmdns
*/
// implements the node.Service interface
func (self *Swarm) Start(srv *p2p.Server) error {
self.startTime = time.Now()
func (s *Swarm) Start(srv *p2p.Server) error {
startTime := time.Now()
self.tracerClose = tracing.Closer
s.tracerClose = tracing.Closer
// update uaddr to correct enode
newaddr := self.bzz.UpdateLocalAddr([]byte(srv.Self().String()))
newaddr := s.bzz.UpdateLocalAddr([]byte(srv.Self().String()))
log.Info("Updated bzz local addr", "oaddr", fmt.Sprintf("%x", newaddr.OAddr), "uaddr", fmt.Sprintf("%s", newaddr.UAddr))
// set chequebook
//TODO: Currently if swap is enabled and no chequebook (or inexistent) contract is provided, the node would crash.
//Once we integrate back the contracts, this check MUST be revisited
if self.config.SwapEnabled && self.config.SwapAPI != "" {
if s.config.SwapEnabled && s.config.SwapAPI != "" {
ctx := context.Background() // The initial setup has no deadline.
err := self.SetChequebook(ctx)
err := s.SetChequebook(ctx)
if err != nil {
return fmt.Errorf("Unable to set chequebook for SWAP: %v", err)
}
log.Debug(fmt.Sprintf("-> cheque book for SWAP: %v", self.config.Swap.Chequebook()))
log.Debug(fmt.Sprintf("-> cheque book for SWAP: %v", s.config.Swap.Chequebook()))
} else {
log.Debug(fmt.Sprintf("SWAP disabled: no cheque book set"))
}
log.Info("Starting bzz service")
err := self.bzz.Start(srv)
err := s.bzz.Start(srv)
if err != nil {
log.Error("bzz failed", "err", err)
return err
}
log.Info("Swarm network started", "bzzaddr", fmt.Sprintf("%x", self.bzz.Hive.BaseAddr()))
log.Info("Swarm network started", "bzzaddr", fmt.Sprintf("%x", s.bzz.Hive.BaseAddr()))
if self.ps != nil {
self.ps.Start(srv)
if s.ps != nil {
s.ps.Start(srv)
}
// start swarm http proxy server
if self.config.Port != "" {
addr := net.JoinHostPort(self.config.ListenAddr, self.config.Port)
server := httpapi.NewServer(self.api, self.config.Cors)
if s.config.Port != "" {
addr := net.JoinHostPort(s.config.ListenAddr, s.config.Port)
server := httpapi.NewServer(s.api, s.config.Cors)
if self.config.Cors != "" {
log.Debug("Swarm HTTP proxy CORS headers", "allowedOrigins", self.config.Cors)
if s.config.Cors != "" {
log.Debug("Swarm HTTP proxy CORS headers", "allowedOrigins", s.config.Cors)
}
log.Debug("Starting Swarm HTTP proxy", "port", self.config.Port)
log.Debug("Starting Swarm HTTP proxy", "port", s.config.Port)
go func() {
err := server.ListenAndServe(addr)
if err != nil {
@ -396,61 +397,71 @@ func (self *Swarm) Start(srv *p2p.Server) error {
}()
}
self.periodicallyUpdateGauges()
doneC := make(chan struct{})
s.cleanupFuncs = append(s.cleanupFuncs, func() error {
close(doneC)
return nil
})
go func(time.Time) {
for {
select {
case <-time.After(updateGaugesPeriod):
uptimeGauge.Update(time.Since(startTime).Nanoseconds())
requestsCacheGauge.Update(int64(s.netStore.RequestsCacheLen()))
case <-doneC:
return
}
}
}(startTime)
startCounter.Inc(1)
self.streamer.Start(srv)
s.streamer.Start(srv)
return nil
}
func (self *Swarm) periodicallyUpdateGauges() {
ticker := time.NewTicker(updateGaugesPeriod)
go func() {
for range ticker.C {
self.updateGauges()
}
}()
}
func (self *Swarm) updateGauges() {
uptimeGauge.Update(time.Since(self.startTime).Nanoseconds())
requestsCacheGauge.Update(int64(self.netStore.RequestsCacheLen()))
}
// implements the node.Service interface
// stops all component services.
func (self *Swarm) Stop() error {
if self.tracerClose != nil {
err := self.tracerClose.Close()
func (s *Swarm) Stop() error {
if s.tracerClose != nil {
err := s.tracerClose.Close()
if err != nil {
return err
}
}
if self.ps != nil {
self.ps.Stop()
if s.ps != nil {
s.ps.Stop()
}
if ch := self.config.Swap.Chequebook(); ch != nil {
if ch := s.config.Swap.Chequebook(); ch != nil {
ch.Stop()
ch.Save()
}
if self.swap != nil {
self.swap.Close()
if s.swap != nil {
s.swap.Close()
}
if self.accountingMetrics != nil {
self.accountingMetrics.Close()
if s.accountingMetrics != nil {
s.accountingMetrics.Close()
}
if self.netStore != nil {
self.netStore.Close()
if s.netStore != nil {
s.netStore.Close()
}
self.sfs.Stop()
s.sfs.Stop()
stopCounter.Inc(1)
self.streamer.Stop()
s.streamer.Stop()
err := self.bzz.Stop()
if self.stateStore != nil {
self.stateStore.Close()
err := s.bzz.Stop()
if s.stateStore != nil {
s.stateStore.Close()
}
for _, cleanF := range s.cleanupFuncs {
err = cleanF()
if err != nil {
log.Error("encountered an error while running cleanup function", "err", err)
break
}
}
return err
}
@ -471,68 +482,73 @@ func (s *Swarm) Protocols() (protos []p2p.Protocol) {
// implements node.Service
// APIs returns the RPC API descriptors the Swarm implementation offers
func (self *Swarm) APIs() []rpc.API {
func (s *Swarm) APIs() []rpc.API {
apis := []rpc.API{
// public APIs
{
Namespace: "bzz",
Version: "3.0",
Service: &Info{self.config, chequebook.ContractParams},
Service: &Info{s.config, chequebook.ContractParams},
Public: true,
},
// admin APIs
{
Namespace: "bzz",
Version: "3.0",
Service: api.NewInspector(self.api, self.bzz.Hive, self.netStore),
Service: api.NewInspector(s.api, s.bzz.Hive, s.netStore),
Public: false,
},
{
Namespace: "chequebook",
Version: chequebook.Version,
Service: chequebook.NewAPI(self.config.Swap.Chequebook),
Service: chequebook.NewAPI(s.config.Swap.Chequebook),
Public: false,
},
{
Namespace: "swarmfs",
Version: fuse.Swarmfs_Version,
Service: self.sfs,
Service: s.sfs,
Public: false,
},
{
Namespace: "accounting",
Version: protocols.AccountingVersion,
Service: protocols.NewAccountingApi(self.accountingMetrics),
Service: protocols.NewAccountingApi(s.accountingMetrics),
Public: false,
},
}
apis = append(apis, self.bzz.APIs()...)
apis = append(apis, s.bzz.APIs()...)
if self.ps != nil {
apis = append(apis, self.ps.APIs()...)
if s.ps != nil {
apis = append(apis, s.ps.APIs()...)
}
return apis
}
// SetChequebook ensures that the local checquebook is set up on chain.
func (self *Swarm) SetChequebook(ctx context.Context) error {
err := self.config.Swap.SetChequebook(ctx, self.backend, self.config.Path)
func (s *Swarm) SetChequebook(ctx context.Context) error {
err := s.config.Swap.SetChequebook(ctx, s.backend, s.config.Path)
if err != nil {
return err
}
log.Info(fmt.Sprintf("new chequebook set (%v): saving config file, resetting all connections in the hive", self.config.Swap.Contract.Hex()))
log.Info(fmt.Sprintf("new chequebook set (%v): saving config file, resetting all connections in the hive", s.config.Swap.Contract.Hex()))
return nil
}
// RegisterPssProtocol adds a devp2p protocol to the swarm node's Pss instance
func (s *Swarm) RegisterPssProtocol(topic *pss.Topic, spec *protocols.Spec, targetprotocol *p2p.Protocol, options *pss.ProtocolParams) (*pss.Protocol, error) {
return pss.RegisterProtocol(s.ps, topic, spec, targetprotocol, options)
}
// serialisable info about swarm
type Info struct {
*api.Config
*chequebook.Params
}
func (self *Info) Info() *Info {
return self
func (s *Info) Info() *Info {
return s
}

3
trie/encoding.go
View file

@ -52,6 +52,9 @@ func hexToCompact(hex []byte) []byte {
}
func compactToHex(compact []byte) []byte {
if len(compact) == 0 {
return compact
}
base := keybytesToHex(compact)
// delete terminator flag
if base[0] < 2 {

13
trie/trie_test.go
View file

@ -614,3 +614,16 @@ func updateString(trie *Trie, k, v string) {
func deleteString(trie *Trie, k string) {
trie.Delete([]byte(k))
}
func TestDecodeNode(t *testing.T) {
t.Parallel()
var (
hash = make([]byte, 20)
elems = make([]byte, 20)
)
for i := 0; i < 5000000; i++ {
rand.Read(hash)
rand.Read(elems)
decodeNode(hash, elems, 1)
}
}

2
vendor/github.com/allegro/bigcache/bigcache.go generated vendored
View file

@ -102,7 +102,7 @@ func (c *BigCache) Close() error {
}
// Get reads entry for the key.
// It returns an EntryNotFoundError when
// It returns an ErrEntryNotFound when
// no entry exists for the given key.
func (c *BigCache) Get(key string) ([]byte, error) {
hashedKey := c.hash.Sum64(key)

17
vendor/github.com/allegro/bigcache/entry_not_found_error.go generated vendored
View file

@ -1,17 +1,6 @@
package bigcache
import "fmt"
import "errors"
// EntryNotFoundError is an error type struct which is returned when entry was not found for provided key
type EntryNotFoundError struct {
key string
}
func notFound(key string) error {
return &EntryNotFoundError{key}
}
// Error returned when entry does not exist.
func (e EntryNotFoundError) Error() string {
return fmt.Sprintf("Entry %q not found", e.key)
}
// ErrEntryNotFound is an error type struct which is returned when entry was not found for provided key
var ErrEntryNotFound = errors.New("Entry not found")

36
vendor/github.com/allegro/bigcache/queue/bytes_queue.go generated vendored
View file

@ -16,6 +16,12 @@ const (
minimumEmptyBlobSize = 32 + headerEntrySize
)
var (
errEmptyQueue = &queueError{"Empty queue"}
errInvalidIndex = &queueError{"Index must be greater than zero. Invalid index."}
errIndexOutOfBounds = &queueError{"Index out of range"}
)
// BytesQueue is a non-thread safe queue type of fifo based on bytes array.
// For every push operation index of entry is returned. It can be used to read the entry later
type BytesQueue struct {
@ -162,6 +168,11 @@ func (q *BytesQueue) Get(index int) ([]byte, error) {
return data, err
}
// CheckGet checks if an entry can be read from index
func (q *BytesQueue) CheckGet(index int) error {
return q.peekCheckErr(index)
}
// Capacity returns number of allocated bytes for queue
func (q *BytesQueue) Capacity() int {
return q.capacity
@ -177,18 +188,35 @@ func (e *queueError) Error() string {
return e.message
}
func (q *BytesQueue) peek(index int) ([]byte, int, error) {
// peekCheckErr is identical to peek, but does not actually return any data
func (q *BytesQueue) peekCheckErr(index int) error {
if q.count == 0 {
return nil, 0, &queueError{"Empty queue"}
return errEmptyQueue
}
if index <= 0 {
return nil, 0, &queueError{"Index must be grater than zero. Invalid index."}
return errInvalidIndex
}
if index+headerEntrySize >= len(q.array) {
return nil, 0, &queueError{"Index out of range"}
return errIndexOutOfBounds
}
return nil
}
func (q *BytesQueue) peek(index int) ([]byte, int, error) {
if q.count == 0 {
return nil, 0, errEmptyQueue
}
if index <= 0 {
return nil, 0, errInvalidIndex
}
if index+headerEntrySize >= len(q.array) {
return nil, 0, errIndexOutOfBounds
}
blockSize := int(binary.LittleEndian.Uint32(q.array[index : index+headerEntrySize]))

Some files were not shown because too many files have changed in this diff Show more