From 18a7d313386194be39e733ac3043988690f42464 Mon Sep 17 00:00:00 2001 From: Jim McDonald Date: Mon, 15 Jan 2018 10:57:06 +0000 Subject: [PATCH 001/107] miner: avoid unnecessary work (#15883) --- core/gaspool.go | 5 +++++ miner/worker.go | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/core/gaspool.go b/core/gaspool.go index c3ee5c198f..e3795c1ee9 100644 --- a/core/gaspool.go +++ b/core/gaspool.go @@ -44,6 +44,11 @@ func (gp *GasPool) SubGas(amount uint64) error { return nil } +// Gas returns the amount of gas remaining in the pool. +func (gp *GasPool) Gas() uint64 { + return uint64(*gp) +} + func (gp *GasPool) String() string { return fmt.Sprintf("%d", *gp) } diff --git a/miner/worker.go b/miner/worker.go index 638f759bf5..1520277e17 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -512,6 +512,11 @@ func (env *Work) commitTransactions(mux *event.TypeMux, txs *types.TransactionsB var coalescedLogs []*types.Log for { + // If we don't have enough gas for any further transactions then we're done + if gp.Gas() < params.TxGas { + log.Trace("Not enough gas for further transactions", "gp", gp) + break + } // Retrieve the next transaction and abort if all done tx := txs.Peek() if tx == nil { From 216e584899ed522088419438c9c605a20b5dc9ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 15 Jan 2018 15:32:14 +0200 Subject: [PATCH 002/107] Revert "trie: make fullnode children hash calculation concurrently (#15131)" (#15889) This reverts commit 0f7fbb85d6e939510a3e3bb6493a9a332ddfd8e8. --- trie/hasher.go | 111 +++++++++++--------------------------------- trie/secure_trie.go | 8 ++-- trie/trie.go | 1 + 3 files changed, 32 insertions(+), 88 deletions(-) diff --git a/trie/hasher.go b/trie/hasher.go index 5186d76698..4719aabf62 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -26,46 +26,27 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) -// calculator is a utility used by the hasher to calculate the hash value of the tree node. -type calculator struct { - sha hash.Hash - buffer *bytes.Buffer +type hasher struct { + tmp *bytes.Buffer + sha hash.Hash + cachegen, cachelimit uint16 } -// calculatorPool is a set of temporary calculators that may be individually saved and retrieved. -var calculatorPool = sync.Pool{ +// hashers live in a global pool. +var hasherPool = sync.Pool{ New: func() interface{} { - return &calculator{buffer: new(bytes.Buffer), sha: sha3.NewKeccak256()} + return &hasher{tmp: new(bytes.Buffer), sha: sha3.NewKeccak256()} }, } -// hasher hasher is used to calculate the hash value of the whole tree. -type hasher struct { - cachegen uint16 - cachelimit uint16 - threaded bool - mu sync.Mutex -} - func newHasher(cachegen, cachelimit uint16) *hasher { - h := &hasher{ - cachegen: cachegen, - cachelimit: cachelimit, - } + h := hasherPool.Get().(*hasher) + h.cachegen, h.cachelimit = cachegen, cachelimit return h } -// newCalculator retrieves a cleaned calculator from calculator pool. -func (h *hasher) newCalculator() *calculator { - calculator := calculatorPool.Get().(*calculator) - calculator.buffer.Reset() - calculator.sha.Reset() - return calculator -} - -// returnCalculator returns a no longer used calculator to the pool. -func (h *hasher) returnCalculator(calculator *calculator) { - calculatorPool.Put(calculator) +func returnHasherToPool(h *hasher) { + hasherPool.Put(h) } // hash collapses a node down into a hash node, also returning a copy of the @@ -142,48 +123,15 @@ func (h *hasher) hashChildren(original node, db DatabaseWriter) (node, node, err // Hash the full node's children, caching the newly hashed subtrees collapsed, cached := n.copy(), n.copy() - // hashChild is a helper to hash a single child, which is called either on the - // same thread as the caller or in a goroutine for the toplevel branching. - hashChild := func(index int, wg *sync.WaitGroup) { - if wg != nil { - defer wg.Done() + for i := 0; i < 16; i++ { + if n.Children[i] != nil { + collapsed.Children[i], cached.Children[i], err = h.hash(n.Children[i], db, false) + if err != nil { + return original, original, err + } + } else { + collapsed.Children[i] = valueNode(nil) // Ensure that nil children are encoded as empty strings. } - // Ensure that nil children are encoded as empty strings. - if collapsed.Children[index] == nil { - collapsed.Children[index] = valueNode(nil) - return - } - // Hash all other children properly - var herr error - collapsed.Children[index], cached.Children[index], herr = h.hash(n.Children[index], db, false) - if herr != nil { - h.mu.Lock() // rarely if ever locked, no congenstion - err = herr - h.mu.Unlock() - } - } - // If we're not running in threaded mode yet, span a goroutine for each child - if !h.threaded { - // Disable further threading - h.threaded = true - - // Hash all the children concurrently - var wg sync.WaitGroup - for i := 0; i < 16; i++ { - wg.Add(1) - go hashChild(i, &wg) - } - wg.Wait() - - // Reenable threading for subsequent hash calls - h.threaded = false - } else { - for i := 0; i < 16; i++ { - hashChild(i, nil) - } - } - if err != nil { - return original, original, err } cached.Children[16] = n.Children[16] if collapsed.Children[16] == nil { @@ -202,29 +150,24 @@ func (h *hasher) store(n node, db DatabaseWriter, force bool) (node, error) { if _, isHash := n.(hashNode); n == nil || isHash { return n, nil } - calculator := h.newCalculator() - defer h.returnCalculator(calculator) - // Generate the RLP encoding of the node - if err := rlp.Encode(calculator.buffer, n); err != nil { + h.tmp.Reset() + if err := rlp.Encode(h.tmp, n); err != nil { panic("encode error: " + err.Error()) } - if calculator.buffer.Len() < 32 && !force { + + if h.tmp.Len() < 32 && !force { return n, nil // Nodes smaller than 32 bytes are stored inside their parent } // Larger nodes are replaced by their hash and stored in the database. hash, _ := n.cache() if hash == nil { - calculator.sha.Write(calculator.buffer.Bytes()) - hash = hashNode(calculator.sha.Sum(nil)) + h.sha.Reset() + h.sha.Write(h.tmp.Bytes()) + hash = hashNode(h.sha.Sum(nil)) } if db != nil { - // db might be a leveldb batch, which is not safe for concurrent writes - h.mu.Lock() - err := db.Put(hash, calculator.buffer.Bytes()) - h.mu.Unlock() - - return hash, err + return hash, db.Put(hash, h.tmp.Bytes()) } return hash, nil } diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 1fde45165e..20c303f31c 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -199,10 +199,10 @@ func (t *SecureTrie) secKey(key []byte) []byte { // invalid on the next call to hashKey or secKey. func (t *SecureTrie) hashKey(key []byte) []byte { h := newHasher(0, 0) - calculator := h.newCalculator() - calculator.sha.Write(key) - buf := calculator.sha.Sum(t.hashKeyBuf[:0]) - h.returnCalculator(calculator) + h.sha.Reset() + h.sha.Write(key) + buf := h.sha.Sum(t.hashKeyBuf[:0]) + returnHasherToPool(h) return buf } diff --git a/trie/trie.go b/trie/trie.go index 7c1b5e1b61..8fe98d8351 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -501,5 +501,6 @@ func (t *Trie) hashRoot(db DatabaseWriter) (node, node, error) { return hashNode(emptyRoot.Bytes()), nil, nil } h := newHasher(t.cachegen, t.cachelimit) + defer returnHasherToPool(h) return h.hash(t.root, db, true) } From 1b0a051d4c617547e3f735461c42ce962f4b9b03 Mon Sep 17 00:00:00 2001 From: Christopher Dro Date: Sat, 13 Jan 2018 17:59:02 -0800 Subject: [PATCH 003/107] [Store] Allow parameters to be set via cli and env vars --- cmd/swarm/config.go | 48 +++++++++++++++++++++++++++------------ cmd/swarm/main.go | 25 ++++++++++++++++++++ swarm/storage/netstore.go | 4 +++- 3 files changed, 62 insertions(+), 15 deletions(-) diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index e6a64cd85d..b23d452869 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -57,20 +57,24 @@ var ( //constants for environment variables const ( - SWARM_ENV_CHEQUEBOOK_ADDR = "SWARM_CHEQUEBOOK_ADDR" - SWARM_ENV_ACCOUNT = "SWARM_ACCOUNT" - SWARM_ENV_LISTEN_ADDR = "SWARM_LISTEN_ADDR" - SWARM_ENV_PORT = "SWARM_PORT" - SWARM_ENV_NETWORK_ID = "SWARM_NETWORK_ID" - SWARM_ENV_SWAP_ENABLE = "SWARM_SWAP_ENABLE" - SWARM_ENV_SWAP_API = "SWARM_SWAP_API" - SWARM_ENV_SYNC_ENABLE = "SWARM_SYNC_ENABLE" - SWARM_ENV_ENS_API = "SWARM_ENS_API" - SWARM_ENV_ENS_ADDR = "SWARM_ENS_ADDR" - SWARM_ENV_CORS = "SWARM_CORS" - SWARM_ENV_BOOTNODES = "SWARM_BOOTNODES" - SWARM_ENV_PSS_ENABLE = "SWARM_PSS_ENABLE" - GETH_ENV_DATADIR = "GETH_DATADIR" + SWARM_ENV_CHEQUEBOOK_ADDR = "SWARM_CHEQUEBOOK_ADDR" + SWARM_ENV_ACCOUNT = "SWARM_ACCOUNT" + SWARM_ENV_LISTEN_ADDR = "SWARM_LISTEN_ADDR" + SWARM_ENV_PORT = "SWARM_PORT" + SWARM_ENV_NETWORK_ID = "SWARM_NETWORK_ID" + SWARM_ENV_SWAP_ENABLE = "SWARM_SWAP_ENABLE" + SWARM_ENV_SWAP_API = "SWARM_SWAP_API" + SWARM_ENV_SYNC_ENABLE = "SWARM_SYNC_ENABLE" + SWARM_ENV_ENS_API = "SWARM_ENS_API" + SWARM_ENV_ENS_ADDR = "SWARM_ENS_ADDR" + SWARM_ENV_CORS = "SWARM_CORS" + SWARM_ENV_BOOTNODES = "SWARM_BOOTNODES" + SWARM_ENV_PSS_ENABLE = "SWARM_PSS_ENABLE" + SWARM_ENV_STORE_PATH = "SWARM_STORE_PATH" + SWARM_ENV_STORE_CAPACITY = "SWARM_STORE_CAPACITY" + SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY" + SWARM_ENV_STORE_RADIUS = "SWARM_STORE_RADIUS" + GETH_ENV_DATADIR = "GETH_DATADIR" ) // These settings ensure that TOML keys use the same names as Go struct fields. @@ -216,6 +220,22 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con currentConfig.PssEnabled = true } + if storePath := ctx.GlobalString(SwarmStorePath.Name); storePath != "" { + currentConfig.StoreParams.ChunkDbPath = storePath + } + + if storeCapacity := ctx.GlobalUint64(SwarmStoreCapacity.Name); storeCapacity != 0 { + currentConfig.StoreParams.DbCapacity = uint64(storeCapacity) + } + + if storeCacheCapacity := ctx.GlobalUint(SwarmStoreCacheCapacity.Name); storeCacheCapacity != 0 { + currentConfig.StoreParams.CacheCapacity = uint(storeCacheCapacity) + } + + if storeRadius := ctx.GlobalInt(SwarmStoreRadius.Name); storeRadius != 0 { + currentConfig.StoreParams.Radius = int(storeRadius) + } + return currentConfig } diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 296f4a609c..c0698d5dc2 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -154,6 +154,26 @@ var ( Usage: "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')", EnvVar: SWARM_ENV_CORS, } + SwarmStorePath = cli.StringFlag{ + Name: "store.path", + Usage: "Path to leveldb chunk DB (default <$GETH_ENV_DIR>/swarm/bzz-<$BZZ_KEY>/chunks)", + EnvVar: SWARM_ENV_STORE_PATH, + } + SwarmStoreCapacity = cli.Uint64Flag{ + Name: "store.size", + Usage: "Number of chunks (5M is roughly 20-25GB) (default 5000000)", + EnvVar: SWARM_ENV_STORE_CAPACITY, + } + SwarmStoreCacheCapacity = cli.UintFlag{ + Name: "store.cache.size", + Usage: "Number of recent chunks cached in memory (default 5000)", + EnvVar: SWARM_ENV_STORE_CACHE_CAPACITY, + } + SwarmStoreRadius = cli.IntFlag{ + Name: "store.radius", + Usage: "Minimum proximity order (number of identical prefix bits of address key) for chunks to warrant storage (default 0)", + EnvVar: SWARM_ENV_STORE_RADIUS, + } // the following flags are deprecated and should be removed in the future DeprecatedEthAPIFlag = cli.StringFlag{ @@ -367,6 +387,11 @@ DEPRECATED: use 'swarm db clean'. SwarmUploadMimeType, // pss flags SwarmPssEnabledFlag, + // storage flags + SwarmStorePath, + SwarmStoreCapacity, + SwarmStoreCacheCapacity, + SwarmStoreRadius, //deprecated flags DeprecatedEthAPIFlag, } diff --git a/swarm/storage/netstore.go b/swarm/storage/netstore.go index 5d4f17deb1..7661f122b5 100644 --- a/swarm/storage/netstore.go +++ b/swarm/storage/netstore.go @@ -69,7 +69,9 @@ func NewDefaultStoreParams() (self *StoreParams) { //this can only finally be set after all config options (file, cmd line, env vars) //have been evaluated func (self *StoreParams) Init(path string) { - self.ChunkDbPath = filepath.Join(path, "chunks") + if self.ChunkDbPath == "" { + self.ChunkDbPath = filepath.Join(path, "chunks") + } } // netstore contructor, takes path argument that is used to initialise dbStore, From f08cd94fb755471cb78091af99ef7026afb392f3 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 16 Jan 2018 15:42:41 +0100 Subject: [PATCH 004/107] cmd/ethkey: fix formatting, review nits (#15807) This commit: - Adds a --msgfile option to read the message to sign from a file instead of command line argument. - Adds a unit test for signing subcommands. - Removes some weird whitespace in the code. --- cmd/ethkey/generate.go | 63 +++++++++++------------ cmd/ethkey/inspect.go | 1 + cmd/ethkey/main.go | 41 +++++++-------- cmd/ethkey/message.go | 97 ++++++++++++++++++++---------------- cmd/ethkey/message_test.go | 70 ++++++++++++++++++++++++++ cmd/ethkey/run_test.go | 54 ++++++++++++++++++++ internal/cmdtest/test_cmd.go | 8 +-- 7 files changed, 235 insertions(+), 99 deletions(-) create mode 100644 cmd/ethkey/message_test.go create mode 100644 cmd/ethkey/run_test.go diff --git a/cmd/ethkey/generate.go b/cmd/ethkey/generate.go index dee0e9d70e..6d57d17fb4 100644 --- a/cmd/ethkey/generate.go +++ b/cmd/ethkey/generate.go @@ -1,8 +1,23 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + package main import ( "crypto/ecdsa" - "crypto/rand" "fmt" "io/ioutil" "os" @@ -26,16 +41,16 @@ var commandGenerate = cli.Command{ ArgsUsage: "[ ]", Description: ` Generate a new keyfile. -If you want to use an existing private key to use in the keyfile, it can be -specified by setting --privatekey with the location of the file containing the -private key.`, + +If you want to encrypt an existing private key, it can be specified by setting +--privatekey with the location of the file containing the private key. +`, Flags: []cli.Flag{ passphraseFlag, jsonFlag, cli.StringFlag{ - Name: "privatekey", - Usage: "the file from where to read the private key to " + - "generate a keyfile for", + Name: "privatekey", + Usage: "file containing a raw private key to encrypt", }, }, Action: func(ctx *cli.Context) error { @@ -51,32 +66,19 @@ private key.`, } var privateKey *ecdsa.PrivateKey - - // First check if a private key file is provided. - privateKeyFile := ctx.String("privatekey") - if privateKeyFile != "" { - privateKeyBytes, err := ioutil.ReadFile(privateKeyFile) + var err error + if file := ctx.String("privatekey"); file != "" { + // Load private key from file. + privateKey, err = crypto.LoadECDSA(file) if err != nil { - utils.Fatalf("Failed to read the private key file '%s': %v", - privateKeyFile, err) + utils.Fatalf("Can't load private key: %v", err) } - - pk, err := crypto.HexToECDSA(string(privateKeyBytes)) - if err != nil { - utils.Fatalf( - "Could not construct ECDSA private key from file content: %v", - err) - } - privateKey = pk - } - - // If not loaded, generate random. - if privateKey == nil { - pk, err := ecdsa.GenerateKey(crypto.S256(), rand.Reader) + } else { + // If not loaded, generate random. + privateKey, err = crypto.GenerateKey() if err != nil { utils.Fatalf("Failed to generate random private key: %v", err) } - privateKey = pk } // Create the keyfile object with a random UUID. @@ -89,8 +91,7 @@ private key.`, // Encrypt key with passphrase. passphrase := getPassPhrase(ctx, true) - keyjson, err := keystore.EncryptKey(key, passphrase, - keystore.StandardScryptN, keystore.StandardScryptP) + keyjson, err := keystore.EncryptKey(key, passphrase, keystore.StandardScryptN, keystore.StandardScryptP) if err != nil { utils.Fatalf("Error encrypting key: %v", err) } @@ -110,7 +111,7 @@ private key.`, if ctx.Bool(jsonFlag.Name) { mustPrintJSON(out) } else { - fmt.Println("Address: ", out.Address) + fmt.Println("Address:", out.Address) } return nil }, diff --git a/cmd/ethkey/inspect.go b/cmd/ethkey/inspect.go index 8a7aeef848..219a5460b8 100644 --- a/cmd/ethkey/inspect.go +++ b/cmd/ethkey/inspect.go @@ -23,6 +23,7 @@ var commandInspect = cli.Command{ ArgsUsage: "", Description: ` Print various information about the keyfile. + Private key information can be printed by using the --private flag; make sure to use this feature with great caution!`, Flags: []cli.Flag{ diff --git a/cmd/ethkey/main.go b/cmd/ethkey/main.go index b9b7a18e05..2a9e5ee483 100644 --- a/cmd/ethkey/main.go +++ b/cmd/ethkey/main.go @@ -28,30 +28,11 @@ const ( defaultKeyfileName = "keyfile.json" ) -var ( - gitCommit = "" // Git SHA1 commit hash of the release (set via linker flags) +// Git SHA1 commit hash of the release (set via linker flags) +var gitCommit = "" - app *cli.App // the main app instance -) +var app *cli.App -var ( // Commonly used command line flags. - passphraseFlag = cli.StringFlag{ - Name: "passwordfile", - Usage: "the file that contains the passphrase for the keyfile", - } - - jsonFlag = cli.BoolFlag{ - Name: "json", - Usage: "output JSON instead of human-readable format", - } - - messageFlag = cli.StringFlag{ - Name: "message", - Usage: "the file that contains the message to sign/verify", - } -) - -// Configure the app instance. func init() { app = utils.NewApp(gitCommit, "an Ethereum key manager") app.Commands = []cli.Command{ @@ -62,6 +43,22 @@ func init() { } } +// Commonly used command line flags. +var ( + passphraseFlag = cli.StringFlag{ + Name: "passwordfile", + Usage: "the file that contains the passphrase for the keyfile", + } + jsonFlag = cli.BoolFlag{ + Name: "json", + Usage: "output JSON instead of human-readable format", + } + messageFlag = cli.StringFlag{ + Name: "message", + Usage: "the file that contains the message to sign/verify", + } +) + func main() { if err := app.Run(os.Args); err != nil { fmt.Fprintln(os.Stderr, err) diff --git a/cmd/ethkey/message.go b/cmd/ethkey/message.go index ae6b6552d3..531a931c82 100644 --- a/cmd/ethkey/message.go +++ b/cmd/ethkey/message.go @@ -1,11 +1,25 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + package main import ( "encoding/hex" "fmt" "io/ioutil" - "os" - "strings" "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/cmd/utils" @@ -18,26 +32,33 @@ type outputSign struct { Signature string } +var msgfileFlag = cli.StringFlag{ + Name: "msgfile", + Usage: "file containing the message to sign/verify", +} + var commandSignMessage = cli.Command{ Name: "signmessage", Usage: "sign a message", - ArgsUsage: " ", + ArgsUsage: " ", Description: ` Sign the message with a keyfile. -It is possible to refer to a file containing the message.`, + +To sign a message contained in a file, use the --msgfile flag. +`, Flags: []cli.Flag{ passphraseFlag, jsonFlag, + msgfileFlag, }, Action: func(ctx *cli.Context) error { - keyfilepath := ctx.Args().First() - message := []byte(ctx.Args().Get(1)) + message := getMessage(ctx, 1) // Load the keyfile. + keyfilepath := ctx.Args().First() keyjson, err := ioutil.ReadFile(keyfilepath) if err != nil { - utils.Fatalf("Failed to read the keyfile at '%s': %v", - keyfilepath, err) + utils.Fatalf("Failed to read the keyfile at '%s': %v", keyfilepath, err) } // Decrypt key with passphrase. @@ -47,29 +68,15 @@ It is possible to refer to a file containing the message.`, utils.Fatalf("Error decrypting key: %v", err) } - if len(message) == 0 { - utils.Fatalf("A message must be provided") - } - // Read message if file. - if _, err := os.Stat(string(message)); err == nil { - message, err = ioutil.ReadFile(string(message)) - if err != nil { - utils.Fatalf("Failed to read the message file: %v", err) - } - } - signature, err := crypto.Sign(signHash(message), key.PrivateKey) if err != nil { utils.Fatalf("Failed to sign message: %v", err) } - - out := outputSign{ - Signature: hex.EncodeToString(signature), - } + out := outputSign{Signature: hex.EncodeToString(signature)} if ctx.Bool(jsonFlag.Name) { mustPrintJSON(out) } else { - fmt.Println("Signature: ", out.Signature) + fmt.Println("Signature:", out.Signature) } return nil }, @@ -84,53 +91,40 @@ type outputVerify struct { var commandVerifyMessage = cli.Command{ Name: "verifymessage", Usage: "verify the signature of a signed message", - ArgsUsage: "
", + ArgsUsage: "
", Description: ` Verify the signature of the message. It is possible to refer to a file containing the message.`, Flags: []cli.Flag{ jsonFlag, + msgfileFlag, }, Action: func(ctx *cli.Context) error { addressStr := ctx.Args().First() signatureHex := ctx.Args().Get(1) - message := []byte(ctx.Args().Get(2)) + message := getMessage(ctx, 2) - // Determine whether it is a keyfile, public key or address. if !common.IsHexAddress(addressStr) { utils.Fatalf("Invalid address: %s", addressStr) } address := common.HexToAddress(addressStr) - signature, err := hex.DecodeString(signatureHex) if err != nil { utils.Fatalf("Signature encoding is not hexadecimal: %v", err) } - if len(message) == 0 { - utils.Fatalf("A message must be provided") - } - // Read message if file. - if _, err := os.Stat(string(message)); err == nil { - message, err = ioutil.ReadFile(string(message)) - if err != nil { - utils.Fatalf("Failed to read the message file: %v", err) - } - } - recoveredPubkey, err := crypto.SigToPub(signHash(message), signature) if err != nil || recoveredPubkey == nil { utils.Fatalf("Signature verification failed: %v", err) } recoveredPubkeyBytes := crypto.FromECDSAPub(recoveredPubkey) recoveredAddress := crypto.PubkeyToAddress(*recoveredPubkey) - success := address == recoveredAddress out := outputVerify{ Success: success, RecoveredPublicKey: hex.EncodeToString(recoveredPubkeyBytes), - RecoveredAddress: strings.ToLower(recoveredAddress.Hex()), + RecoveredAddress: recoveredAddress.Hex(), } if ctx.Bool(jsonFlag.Name) { mustPrintJSON(out) @@ -140,9 +134,26 @@ It is possible to refer to a file containing the message.`, } else { fmt.Println("Signature verification failed!") } - fmt.Println("Recovered public key: ", out.RecoveredPublicKey) - fmt.Println("Recovered address: ", out.RecoveredAddress) + fmt.Println("Recovered public key:", out.RecoveredPublicKey) + fmt.Println("Recovered address:", out.RecoveredAddress) } return nil }, } + +func getMessage(ctx *cli.Context, msgarg int) []byte { + if file := ctx.String("msgfile"); file != "" { + if len(ctx.Args()) > msgarg { + utils.Fatalf("Can't use --msgfile and message argument at the same time.") + } + msg, err := ioutil.ReadFile(file) + if err != nil { + utils.Fatalf("Can't read message file: %v", err) + } + return msg + } else if len(ctx.Args()) == msgarg+1 { + return []byte(ctx.Args().Get(msgarg)) + } + utils.Fatalf("Invalid number of arguments: want %d, got %d", msgarg+1, len(ctx.Args())) + return nil +} diff --git a/cmd/ethkey/message_test.go b/cmd/ethkey/message_test.go new file mode 100644 index 0000000000..fb16f03d02 --- /dev/null +++ b/cmd/ethkey/message_test.go @@ -0,0 +1,70 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +func TestMessageSignVerify(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "ethkey-test") + if err != nil { + t.Fatal("Can't create temporary directory:", err) + } + defer os.RemoveAll(tmpdir) + + keyfile := filepath.Join(tmpdir, "the-keyfile") + message := "test message" + + // Create the key. + generate := runEthkey(t, "generate", keyfile) + generate.Expect(` +!! Unsupported terminal, password will be echoed. +Passphrase: {{.InputLine "foobar"}} +Repeat passphrase: {{.InputLine "foobar"}} +`) + _, matches := generate.ExpectRegexp(`Address: (0x[0-9a-fA-F]{40})\n`) + address := matches[1] + generate.ExpectExit() + + // Sign a message. + sign := runEthkey(t, "signmessage", keyfile, message) + sign.Expect(` +!! Unsupported terminal, password will be echoed. +Passphrase: {{.InputLine "foobar"}} +`) + _, matches = sign.ExpectRegexp(`Signature: ([0-9a-f]+)\n`) + signature := matches[1] + sign.ExpectExit() + + // Verify the message. + verify := runEthkey(t, "verifymessage", address, signature, message) + _, matches = verify.ExpectRegexp(` +Signature verification successful! +Recovered public key: [0-9a-f]+ +Recovered address: (0x[0-9a-fA-F]{40}) +`) + recovered := matches[1] + verify.ExpectExit() + + if recovered != address { + t.Error("recovered address doesn't match generated key") + } +} diff --git a/cmd/ethkey/run_test.go b/cmd/ethkey/run_test.go new file mode 100644 index 0000000000..8ce4fe5cde --- /dev/null +++ b/cmd/ethkey/run_test.go @@ -0,0 +1,54 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "fmt" + "os" + "testing" + + "github.com/docker/docker/pkg/reexec" + "github.com/ethereum/go-ethereum/internal/cmdtest" +) + +type testEthkey struct { + *cmdtest.TestCmd +} + +// spawns ethkey with the given command line args. +func runEthkey(t *testing.T, args ...string) *testEthkey { + tt := new(testEthkey) + tt.TestCmd = cmdtest.NewTestCmd(t, tt) + tt.Run("ethkey-test", args...) + return tt +} + +func TestMain(m *testing.M) { + // Run the app if we've been exec'd as "ethkey-test" in runEthkey. + reexec.Register("ethkey-test", func() { + if err := app.Run(os.Args); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + os.Exit(0) + }) + // check if we have been reexec'd + if reexec.Init() { + return + } + os.Exit(m.Run()) +} diff --git a/internal/cmdtest/test_cmd.go b/internal/cmdtest/test_cmd.go index 541e51c4c0..fae61cfe32 100644 --- a/internal/cmdtest/test_cmd.go +++ b/internal/cmdtest/test_cmd.go @@ -25,6 +25,7 @@ import ( "os" "os/exec" "regexp" + "strings" "sync" "testing" "text/template" @@ -141,9 +142,10 @@ func (tt *TestCmd) matchExactOutput(want []byte) error { // Note that an arbitrary amount of output may be consumed by the // regular expression. This usually means that expect cannot be used // after ExpectRegexp. -func (tt *TestCmd) ExpectRegexp(resource string) (*regexp.Regexp, []string) { +func (tt *TestCmd) ExpectRegexp(regex string) (*regexp.Regexp, []string) { + regex = strings.TrimPrefix(regex, "\n") var ( - re = regexp.MustCompile(resource) + re = regexp.MustCompile(regex) rtee = &runeTee{in: tt.stdout} matches []int ) @@ -151,7 +153,7 @@ func (tt *TestCmd) ExpectRegexp(resource string) (*regexp.Regexp, []string) { output := rtee.buf.Bytes() if matches == nil { tt.Fatalf("Output did not match:\n---------------- (stdout text)\n%s\n---------------- (regular expression)\n%s", - output, resource) + output, regex) return re, nil } tt.Logf("Matched stdout text:\n%s", output) From 370dca4491de92ee81e0755cee1cb5df12f0f665 Mon Sep 17 00:00:00 2001 From: George Ornbo Date: Tue, 16 Jan 2018 15:45:13 +0000 Subject: [PATCH 005/107] core/vm: Fix comment typo --- core/vm/evm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/vm/evm.go b/core/vm/evm.go index 8796a633ec..46e7baff4c 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -391,7 +391,7 @@ func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.I return ret, contractAddr, contract.Gas, err } -// ChainConfig returns the evmironment's chain configuration +// ChainConfig returns the environment's chain configuration func (evm *EVM) ChainConfig() *params.ChainConfig { return evm.chainConfig } // Interpreter returns the EVM interpreter From 5a671d424911064a54df8394da76e890a7bd1cf6 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 8 Jan 2018 13:15:57 +0100 Subject: [PATCH 006/107] all: update generated code (#15808) * core/types, core/vm, eth, tests: regenerate gencodec files * Makefile: update devtools target Install protoc-gen-go and print reminders about npm, solc and protoc. Also switch to github.com/kevinburke/go-bindata because it's more maintained. * contracts/ens: update contracts and regenerate with solidity v0.4.19 The newer upstream version of the FIFSRegistrar contract doesn't set the resolver anymore. The resolver is now deployed separately. * contracts/release: regenerate with solidity v0.4.19 * contracts/chequebook: fix fallback and regenerate with solidity v0.4.19 The contract didn't have a fallback function, payments would be rejected when compiled with newer solidity. References to 'mortal' and 'owned' use the local file system so we can compile without network access. * p2p/discv5: regenerate with recent stringer * cmd/faucet: regenerate * dashboard: regenerate * eth/tracers: regenerate * internal/jsre/deps: regenerate * dashboard: avoid sed -i because it's not portable * accounts/usbwallet/internal/trezor: fix go generate warnings --- dashboard/dashboard.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dashboard/dashboard.go b/dashboard/dashboard.go index 2ac2652ee9..5947918d23 100644 --- a/dashboard/dashboard.go +++ b/dashboard/dashboard.go @@ -21,6 +21,8 @@ package dashboard //go:generate go-bindata -nometadata -o assets.go -prefix assets -nocompress -pkg dashboard assets/dashboard.html assets/bundle.js //go:generate sh -c "sed 's#var _bundleJs#//nolint:misspell\\\n&#' assets.go > assets.go.tmp && mv assets.go.tmp assets.go" //go:generate sh -c "sed 's#var _dashboardHtml#//nolint:misspell\\\n&#' assets.go > assets.go.tmp && mv assets.go.tmp assets.go" +//go:generate go-bindata -nometadata -o assets.go -prefix assets -nocompress -pkg dashboard assets/public/... +//go:generate sh -c "sed 's#var _public#//nolint:misspell\\\n&#' assets.go > assets.go.tmp && mv assets.go.tmp assets.go" //go:generate gofmt -w -s assets.go import ( From 008a84ee9f17871d4bfc68817d008a030dd326ae Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 15 Dec 2017 18:43:02 +0100 Subject: [PATCH 007/107] cmd/utils, p2p, swarm, whisper: Make tests pass --- p2p/nat/natupnp_test.go | 1 + p2p/protocols/protocol_test.go | 5 +++++ swarm/api/config_test.go | 10 +++------- swarm/network/kademlia_test.go | 6 +++++- swarm/network/simulations/discovery/discovery.go | 1 + swarm/network/simulations/discovery/discovery_test.go | 2 ++ swarm/pss/pss_test.go | 1 + whisper/whisperv5/peer_test.go | 2 +- whisper/whisperv6/peer_test.go | 5 +++++ 9 files changed, 24 insertions(+), 9 deletions(-) create mode 100644 swarm/network/simulations/discovery/discovery.go diff --git a/p2p/nat/natupnp_test.go b/p2p/nat/natupnp_test.go index 79f6d25ae8..5695b822d6 100644 --- a/p2p/nat/natupnp_test.go +++ b/p2p/nat/natupnp_test.go @@ -29,6 +29,7 @@ import ( ) func TestUPNP_DDWRT(t *testing.T) { + t.Skip("broken") if runtime.GOOS == "windows" { t.Skipf("disabled to avoid firewall prompt") } diff --git a/p2p/protocols/protocol_test.go b/p2p/protocols/protocol_test.go index c79d34eee6..149e19353c 100644 --- a/p2p/protocols/protocol_test.go +++ b/p2p/protocols/protocol_test.go @@ -320,6 +320,11 @@ func runMultiplePeers(t *testing.T, peer int, errs ...error) { if !pp.Has(s.IDs[0]) { t.Fatalf("missing peer test-0: %v (%v)", pp, s.IDs) } + for !pp.Has(s.IDs[1]) { + time.Sleep(1) + log.Trace(fmt.Sprintf("missing peer test-1: %v (%v)", pp, s.IDs)) + } + if !pp.Has(s.IDs[1]) { t.Fatalf("missing peer test-1: %v (%v)", pp, s.IDs) } diff --git a/swarm/api/config_test.go b/swarm/api/config_test.go index 5636b6dafb..4851f19fc5 100644 --- a/swarm/api/config_test.go +++ b/swarm/api/config_test.go @@ -33,8 +33,8 @@ func TestConfig(t *testing.T) { t.Fatalf("failed to load private key: %v", err) } - one := NewDefaultConfig() - two := NewDefaultConfig() + one := NewConfig() + two := NewConfig() if equal := reflect.DeepEqual(one, two); !equal { t.Fatal("Two default configs are not equal") @@ -55,11 +55,7 @@ func TestConfig(t *testing.T) { t.Fatal("Failed to correctly initialize SwapParams") } - if one.SyncParams.RequestDbPath == one.Path { - t.Fatal("Failed to correctly initialize SyncParams") - } - - if one.HiveParams.KadDbPath == one.Path { + if one.HiveParams.MaxPeersPerRequest != 5 { t.Fatal("Failed to correctly initialize HiveParams") } diff --git a/swarm/network/kademlia_test.go b/swarm/network/kademlia_test.go index 5c09133f19..9597abbf35 100644 --- a/swarm/network/kademlia_test.go +++ b/swarm/network/kademlia_test.go @@ -400,7 +400,11 @@ func TestPruning(t *testing.T) { func TestKademliaHiveString(t *testing.T) { k := newTestKademlia("00000000").On("01000000", "00100000").Register("10000000", "10000001") h := k.String() - expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), MinProxBinSize: 2, MinBinSize: 1, MaxBinSize: 4\n000 0 | 2 8100 (0) 8000 (0)\n============ DEPTH: 1 ==========================================\n001 1 4000 | 1 4000 (0)\n002 1 2000 | 1 2000 (0)\n003 0 | 0\n004 0 | 0\n005 0 | 0\n006 0 | 0\n007 0 | 0\n=========================================================================" + expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), MinProxBinSize: 2, MinBinSize: 1, MaxBinSize: 4\n000 0 | 2 8100 (0) 8000 (0)\n============ DEPTH: 1 ==========================================\n001 1 4000 | 1 4000 (0)\n002 1 2000 | 1 2000 (0)\n" + for i := 3; i < 16; i++ { + expH += fmt.Sprintf("%03d 0 | 0\n", i) + } + expH += "=========================================================================" if expH[100:] != h[100:] { t.Fatalf("incorrect hive output. expected %v, got %v", expH, h) } diff --git a/swarm/network/simulations/discovery/discovery.go b/swarm/network/simulations/discovery/discovery.go new file mode 100644 index 0000000000..5844159aeb --- /dev/null +++ b/swarm/network/simulations/discovery/discovery.go @@ -0,0 +1 @@ +package discovery diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go index fc8d6f70ca..61acd84faa 100644 --- a/swarm/network/simulations/discovery/discovery_test.go +++ b/swarm/network/simulations/discovery/discovery_test.go @@ -71,6 +71,7 @@ func BenchmarkDiscovery_128_4(b *testing.B) { benchmarkDiscovery(b, 128, 4) } func BenchmarkDiscovery_256_4(b *testing.B) { benchmarkDiscovery(b, 256, 4) } func TestDiscoverySimulationDockerAdapter(t *testing.T) { + t.Skip("broken (cannot build image)") testDiscoverySimulationDockerAdapter(t, *nodeCount, *initCount) } @@ -83,6 +84,7 @@ func testDiscoverySimulationDockerAdapter(t *testing.T, nodes, conns int) { } func TestDiscoverySimulationExecAdapter(t *testing.T) { + t.Skip("broken (times out)") testDiscoverySimulationExecAdapter(t, *nodeCount, *initCount) } diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index 57d4a79170..ae03e3cca5 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -627,6 +627,7 @@ func worker(id int, jobs <-chan Job, rpcs map[discover.NodeID]*rpc.Client, pubke // nodes/msgs/addrbytes/adaptertype // if adaptertype is exec uses execadapter, simadapter otherwise func TestNetwork(t *testing.T) { + t.Skip("skip until proper local benchmark values for stress testing can be determined") t.Run("3/2000/4/sock", testNetwork) t.Run("4/2000/4/sock", testNetwork) t.Run("8/2000/4/sock", testNetwork) diff --git a/whisper/whisperv5/peer_test.go b/whisper/whisperv5/peer_test.go index bae2adb6f5..cc9b058624 100644 --- a/whisper/whisperv5/peer_test.go +++ b/whisper/whisperv5/peer_test.go @@ -156,7 +156,7 @@ func initialize(t *testing.T) { err = node.server.Start() if err != nil { - t.Fatalf("failed to start server %d.", i) + t.Skipf("failed to start server %d (port may be taken, skipping since there is no handler in test for this, should be ported to simulation framework): error is %v", i, err) } nodes[i] = &node diff --git a/whisper/whisperv6/peer_test.go b/whisper/whisperv6/peer_test.go index 8a65cb7143..86868b653a 100644 --- a/whisper/whisperv6/peer_test.go +++ b/whisper/whisperv6/peer_test.go @@ -220,6 +220,11 @@ func initialize(t *testing.T) { }, } + err = node.server.Start() + if err != nil { + t.Skipf("failed to start server %d (port may be taken, skipping since there is no handler in test for this, should be ported to simulation framework): error is %v", i, err) + } + nodes[i] = &node } From 90c8e05ee0512da0683d063c58e29cec90c5c09b Mon Sep 17 00:00:00 2001 From: lash Date: Sat, 16 Dec 2017 00:33:08 +0100 Subject: [PATCH 008/107] swarm/network, swarm/pss: Add context cancels, missing format vars --- swarm/network/protocol.go | 5 +-- .../simulations/discovery/discovery_test.go | 2 +- swarm/pss/client/client_test.go | 10 +++-- swarm/pss/handshake.go | 5 ++- swarm/pss/protocol.go | 2 +- swarm/pss/protocol_test.go | 6 ++- swarm/pss/pss.go | 2 +- swarm/pss/pss_test.go | 44 ++++++++++++------- 8 files changed, 46 insertions(+), 30 deletions(-) diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index 79471c8ec6..22c8334030 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -207,9 +207,8 @@ func (b *Bzz) RunProtocol(spec *protocols.Spec, run func(*bzzPeer) error) func(* // performHandshake implements the negotiation of the bzz handshake // shared among swarm subprotocols func performHandshake(p *protocols.Peer, handshake *HandshakeMsg) error { - ctx, _ := context.WithTimeout(context.Background(), bzzHandshakeTimeout) - // defer cancel() - // ctx, cancel := context.WithTimeout(context.Background(), bzzHandshakeTimeout) + ctx, cancel := context.WithTimeout(context.Background(), bzzHandshakeTimeout) + defer cancel() defer close(handshake.done) rsh, err := p.Handshake(ctx, handshake, checkHandshake) if err != nil { diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go index 61acd84faa..7dd59beacc 100644 --- a/swarm/network/simulations/discovery/discovery_test.go +++ b/swarm/network/simulations/discovery/discovery_test.go @@ -137,7 +137,7 @@ func benchmarkDiscovery(b *testing.B, nodes, conns int) { for i := 0; i < b.N; i++ { result, err := discoverySimulation(nodes, conns, adapters.NewSimAdapter(services)) if err != nil { - b.Fatalf("setting up simulation failed", result) + b.Fatalf("setting up simulation failed: %v", err) } if result.Error != nil { b.Logf("simulation failed: %s", result.Error) diff --git a/swarm/pss/client/client_test.go b/swarm/pss/client/client_test.go index e773018a16..a6a909b136 100644 --- a/swarm/pss/client/client_test.go +++ b/swarm/pss/client/client_test.go @@ -104,7 +104,8 @@ func TestClientHandshake(t *testing.T) { lproto := pss.NewPingProtocol(lpssping) rproto := pss.NewPingProtocol(rpssping) - ctx, _ := context.WithTimeout(context.Background(), time.Second*10) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() err = lpsc.RunProtocol(ctx, lproto) if err != nil { t.Fatal(err) @@ -231,13 +232,14 @@ func newServices() adapters.Services { "pss": func(ctx *adapters.ServiceContext) (node.Service, error) { cachedir, err := ioutil.TempDir("", "pss-cache") if err != nil { - return nil, fmt.Errorf("create pss cache tmpdir failed", "error", err) + return nil, fmt.Errorf("create pss cache tmpdir failed: %v", err) } dpa, err := storage.NewLocalDPA(cachedir) if err != nil { - return nil, fmt.Errorf("local dpa creation failed", "error", err) + return nil, fmt.Errorf("local dpa creation failed: %v", err) } - ctxlocal, _ := context.WithTimeout(context.Background(), time.Second) + ctxlocal, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() keys, err := wapi.NewKeyPair(ctxlocal) privkey, err := w.GetPrivateKey(keys) psparams := pss.NewPssParams(privkey) diff --git a/swarm/pss/handshake.go b/swarm/pss/handshake.go index 95bf79ef5a..15f2a32a00 100644 --- a/swarm/pss/handshake.go +++ b/swarm/pss/handshake.go @@ -268,7 +268,7 @@ func (self *HandshakeController) handler(msg []byte, p *p2p.Peer, asymmetric boo if !asymmetric { if self.symKeyIndex[symkeyid] != nil { if self.symKeyIndex[symkeyid].count >= self.symKeyIndex[symkeyid].limit { - return fmt.Errorf("discarding message using expired key", "symkeyid", symkeyid) + return fmt.Errorf("discarding message using expired key: %s", symkeyid) } self.symKeyIndex[symkeyid].count++ log.Trace("increment symkey recv use", "symsymkeyid", symkeyid, "count", self.symKeyIndex[symkeyid].count, "limit", self.symKeyIndex[symkeyid].limit, "receiver", common.ToHex(crypto.FromECDSAPub(self.pss.PublicKey()))) @@ -457,7 +457,8 @@ func (self *HandshakeAPI) Handshake(pubkeyid string, topic Topic, sync bool, flu return keys, err } if sync { - ctx, _ := context.WithTimeout(context.Background(), self.ctrl.symKeyRequestTimeout) + ctx, cancel := context.WithTimeout(context.Background(), self.ctrl.symKeyRequestTimeout) + defer cancel() select { case keys = <-hsc: log.Trace("sync handshake response receive", "key", keys) diff --git a/swarm/pss/protocol.go b/swarm/pss/protocol.go index 6c5c289559..9f7c0a6cfe 100644 --- a/swarm/pss/protocol.go +++ b/swarm/pss/protocol.go @@ -227,7 +227,7 @@ func (self *Protocol) AddPeer(p *p2p.Peer, run func(*p2p.Peer, p2p.MsgReadWriter } go func() { err := run(p, rw) - log.Warn(fmt.Sprintf("pss vprotocol quit on addr %v topic %v: %v", topic, err)) + log.Warn(fmt.Sprintf("pss vprotocol quit on %v topic %v: %v", p, topic, err)) }() return rw, nil } diff --git a/swarm/pss/protocol_test.go b/swarm/pss/protocol_test.go index 54cd4226d7..b30fc0430d 100644 --- a/swarm/pss/protocol_test.go +++ b/swarm/pss/protocol_test.go @@ -73,11 +73,13 @@ func testProtocol(t *testing.T) { time.Sleep(time.Millisecond * 1000) // replace with hive healthy code lmsgC := make(chan APIMsg) - lctx, _ := context.WithTimeout(context.Background(), time.Second*10) + lctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic) defer lsub.Unsubscribe() rmsgC := make(chan APIMsg) - rctx, _ := context.WithTimeout(context.Background(), time.Second*10) + rctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic) defer rsub.Unsubscribe() diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index 507b4ab655..0cd226ba08 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -501,7 +501,7 @@ func (self *Pss) processSym(envelope *whisper.Envelope) (*whisper.ReceivedMessag func (self *Pss) processAsym(envelope *whisper.Envelope) (*whisper.ReceivedMessage, string, *PssAddress, error) { recvmsg, err := envelope.OpenAsymmetric(self.privateKey) if err != nil { - return nil, "", nil, fmt.Errorf("could not decrypt message: %v", "err", err) + return nil, "", nil, fmt.Errorf("could not decrypt message: %v", err) } // check signature (if signed), strip padding if !recvmsg.Validate() { diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index ae03e3cca5..1b26bfb3fe 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -137,7 +137,8 @@ func TestTopic(t *testing.T) { func TestCache(t *testing.T) { var err error to, _ := hex.DecodeString("08090a0b0c0d0e0f1011121314150001020304050607161718191a1b1c1d1e1f") - ctx, _ := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() keys, err := wapi.NewKeyPair(ctx) privkey, err := w.GetPrivateKey(keys) if err != nil { @@ -211,7 +212,8 @@ func TestAddressMatch(t *testing.T) { remoteaddr := []byte("feedbeef") kadparams := network.NewKadParams() kad := network.NewKademlia(localaddr, kadparams) - ctx, _ := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() keys, err := wapi.NewKeyPair(ctx) if err != nil { t.Fatalf("Could not generate private key: %v", err) @@ -255,12 +257,14 @@ func TestAddressMatch(t *testing.T) { // set and generate pubkeys and symkeys func TestKeys(t *testing.T) { // make our key and init pss with it - ctx, _ := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() ourkeys, err := wapi.NewKeyPair(ctx) if err != nil { t.Fatalf("create 'our' key fail") } - ctx, _ = context.WithTimeout(context.Background(), time.Second) + ctx, cancel = context.WithTimeout(context.Background(), time.Second) + defer cancel() theirkeys, err := wapi.NewKeyPair(ctx) if err != nil { t.Fatalf("create 'their' key fail") @@ -449,12 +453,14 @@ func testSymSend(t *testing.T) { // at this point we've verified that symkeys are saved and match on each peer // now try sending symmetrically encrypted message, both directions lmsgC := make(chan APIMsg) - lctx, _ := context.WithTimeout(context.Background(), time.Second*10) + lctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic) log.Trace("lsub", "id", lsub) defer lsub.Unsubscribe() rmsgC := make(chan APIMsg) - rctx, _ := context.WithTimeout(context.Background(), time.Second*10) + rctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic) log.Trace("rsub", "id", rsub) defer rsub.Unsubscribe() @@ -562,12 +568,14 @@ func testAsymSend(t *testing.T) { time.Sleep(time.Millisecond * 500) // replace with hive healthy code lmsgC := make(chan APIMsg) - lctx, _ := context.WithTimeout(context.Background(), time.Second*10) + lctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic) log.Trace("lsub", "id", lsub) defer lsub.Unsubscribe() rmsgC := make(chan APIMsg) - rctx, _ := context.WithTimeout(context.Background(), time.Second*10) + rctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic) log.Trace("rsub", "id", rsub) defer rsub.Unsubscribe() @@ -627,7 +635,6 @@ func worker(id int, jobs <-chan Job, rpcs map[discover.NodeID]*rpc.Client, pubke // nodes/msgs/addrbytes/adaptertype // if adaptertype is exec uses execadapter, simadapter otherwise func TestNetwork(t *testing.T) { - t.Skip("skip until proper local benchmark values for stress testing can be determined") t.Run("3/2000/4/sock", testNetwork) t.Run("4/2000/4/sock", testNetwork) t.Run("8/2000/4/sock", testNetwork) @@ -835,7 +842,8 @@ func benchmarkSymKeySend(b *testing.B) { if err != nil { b.Fatalf("benchmark called with invalid msgsize param '%s': %v", msgsizestring[1], err) } - ctx, _ := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() keys, err := wapi.NewKeyPair(ctx) privkey, err := w.GetPrivateKey(keys) ps := newTestPss(privkey, nil, nil) @@ -878,7 +886,8 @@ func benchmarkAsymKeySend(b *testing.B) { if err != nil { b.Fatalf("benchmark called with invalid msgsize param '%s': %v", msgsizestring[1], err) } - ctx, _ := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() keys, err := wapi.NewKeyPair(ctx) privkey, err := w.GetPrivateKey(keys) ps := newTestPss(privkey, nil, nil) @@ -923,7 +932,8 @@ func benchmarkSymkeyBruteforceChangeaddr(b *testing.B) { } pssmsgs := make([]*PssMsg, 0, keycount) var keyid string - ctx, _ := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() keys, err := wapi.NewKeyPair(ctx) privkey, err := w.GetPrivateKey(keys) if cachesize > 0 { @@ -1005,7 +1015,8 @@ func benchmarkSymkeyBruteforceSameaddr(b *testing.B) { } } addr := make([]PssAddress, keycount) - ctx, _ := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() keys, err := wapi.NewKeyPair(ctx) privkey, err := w.GetPrivateKey(keys) if cachesize > 0 { @@ -1122,17 +1133,18 @@ func newServices() adapters.Services { pssProtocolName: func(ctx *adapters.ServiceContext) (node.Service, error) { cachedir, err := ioutil.TempDir("", "pss-cache") if err != nil { - return nil, fmt.Errorf("create pss cache tmpdir failed", "error", err) + return nil, fmt.Errorf("create pss cache tmpdir failed: %v", err) } dpa, err := storage.NewLocalDPA(cachedir) if err != nil { - return nil, fmt.Errorf("local dpa creation failed", "error", err) + return nil, fmt.Errorf("local dpa creation failed: %v", err) } // execadapter does not exec init() initTest() - ctxlocal, _ := context.WithTimeout(context.Background(), time.Second) + ctxlocal, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() keys, err := wapi.NewKeyPair(ctxlocal) privkey, err := w.GetPrivateKey(keys) pssp := NewPssParams(privkey) From aaa65e1ab75efdcdf3786b2085f73c80ca7a3a0c Mon Sep 17 00:00:00 2001 From: lash Date: Tue, 19 Dec 2017 17:59:56 +0100 Subject: [PATCH 009/107] swarm/pss: Skip network tests with many nodes --- swarm/pss/pss_test.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index 1b26bfb3fe..c8c1379370 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -634,13 +634,15 @@ func worker(id int, jobs <-chan Job, rpcs map[discover.NodeID]*rpc.Client, pubke // params in run name: // nodes/msgs/addrbytes/adaptertype // if adaptertype is exec uses execadapter, simadapter otherwise +// +// ( some tests are commented out because of resource limitations on Travis) func TestNetwork(t *testing.T) { t.Run("3/2000/4/sock", testNetwork) t.Run("4/2000/4/sock", testNetwork) - t.Run("8/2000/4/sock", testNetwork) - t.Run("16/2000/4/sock", testNetwork) - t.Run("32/2000/4/sock", testNetwork) - t.Run("64/2000/4/sim", testNetwork) + // t.Run("8/2000/4/sock", testNetwork) + // t.Run("16/2000/4/sock", testNetwork) + // t.Run("32/2000/4/sock", testNetwork) + // t.Run("64/2000/4/sim", testNetwork) } func testNetwork(t *testing.T) { From bd46c3906ce447450f4df18613505c6be4a520ab Mon Sep 17 00:00:00 2001 From: lash Date: Wed, 20 Dec 2017 15:39:07 +0100 Subject: [PATCH 010/107] swarm/pss: Increase timeout on pss network test --- swarm/pss/pss_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index c8c1379370..a5d7e97947 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -799,7 +799,7 @@ func testNetwork(t *testing.T) { } finalmsgcount := 0 - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) defer cancel() outer: for i := 0; i < int(msgcount); i++ { From d6d5b6285a3f3cc29496a6154768432bb3b3144c Mon Sep 17 00:00:00 2001 From: lash Date: Sat, 23 Dec 2017 02:55:15 +0100 Subject: [PATCH 011/107] swarm/pss: Another attempt at timeout allocation TestNetwork; 3 mins --- swarm/pss/pss_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index a5d7e97947..1cd5587e89 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -799,7 +799,7 @@ func testNetwork(t *testing.T) { } finalmsgcount := 0 - ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) defer cancel() outer: for i := 0; i < int(msgcount); i++ { From 4177ea74a72234246a442d2137d6ca2e63c522f5 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 28 Dec 2017 14:41:23 +0100 Subject: [PATCH 012/107] swarm/pss: Skip tests for 3 and 4 node networks. --- swarm/pss/pss_test.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index 1cd5587e89..492e764248 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -637,11 +637,19 @@ func worker(id int, jobs <-chan Job, rpcs map[discover.NodeID]*rpc.Client, pubke // // ( some tests are commented out because of resource limitations on Travis) func TestNetwork(t *testing.T) { - t.Run("3/2000/4/sock", testNetwork) - t.Run("4/2000/4/sock", testNetwork) - // t.Run("8/2000/4/sock", testNetwork) - // t.Run("16/2000/4/sock", testNetwork) - // t.Run("32/2000/4/sock", testNetwork) + //t.Run("3/2000/4/sock", testNetwork) + //t.Run("4/2000/4/sock", testNetwork) + t.Run("8/2000/4/sock", testNetwork) + t.Run("16/2000/4/sock", testNetwork) + t.Run("8/3000/4/sock", testNetwork) + t.Run("16/3000/4/sock", testNetwork) + //t.Run("32/2000/4/sock", testNetwork) + + t.Run("8/2000/4/sim", testNetwork) + t.Run("16/2000/4/sim", testNetwork) + t.Run("8/3000/4/sim", testNetwork) + t.Run("16/3000/4/sim", testNetwork) + //t.Run("32/2000/4/sim", testNetwork) // t.Run("64/2000/4/sim", testNetwork) } From 19baf31ef53acce0ab5a12e6444ba7d55f50b2d9 Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 29 Dec 2017 00:43:08 +0100 Subject: [PATCH 013/107] swarm/pss: Deactivate failing network test --- swarm/pss/pss_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index 492e764248..35f85e93f8 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -637,6 +637,7 @@ func worker(id int, jobs <-chan Job, rpcs map[discover.NodeID]*rpc.Client, pubke // // ( some tests are commented out because of resource limitations on Travis) func TestNetwork(t *testing.T) { + t.Skip("Temporarily deactivated because not all messages can be delivered") //t.Run("3/2000/4/sock", testNetwork) //t.Run("4/2000/4/sock", testNetwork) t.Run("8/2000/4/sock", testNetwork) From 945656ccfd9db555a053e1f8898d97f93fe9180b Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 5 Jan 2018 19:08:01 +0100 Subject: [PATCH 014/107] p2p/protocols, pot, swarm: Typos --- p2p/protocols/protocol.go | 2 +- p2p/protocols/protocol_test.go | 44 ++++++++++++++++----------------- pot/doc.go | 4 +-- pot/pot.go | 6 ++--- swarm/network/discovery_test.go | 2 +- swarm/network/hive_test.go | 2 +- swarm/network/kademlia.go | 2 +- swarm/network/protocol.go | 2 +- swarm/network/protocol_test.go | 8 +++--- swarm/pss/handshake.go | 4 +-- swarm/pss/pss.go | 2 +- swarm/pss/pss_test.go | 6 ++--- swarm/storage/chunker_test.go | 4 ++- 13 files changed, 45 insertions(+), 43 deletions(-) diff --git a/p2p/protocols/protocol.go b/p2p/protocols/protocol.go index 7b04069edf..bb934ca45a 100644 --- a/p2p/protocols/protocol.go +++ b/p2p/protocols/protocol.go @@ -183,7 +183,7 @@ type Peer struct { // NewPeer constructs a new peer // this constructor is called by the p2p.Protocol#Run function -// the first two arguments are comming the arguments passed to p2p.Protocol.Run function +// the first two arguments are coming the arguments passed to p2p.Protocol.Run function // the third argument is the CodeMap describing the protocol messages and options func NewPeer(p *p2p.Peer, rw p2p.MsgReadWriter, spec *Spec) *Peer { return &Peer{ diff --git a/p2p/protocols/protocol_test.go b/p2p/protocols/protocol_test.go index 149e19353c..a4641ed8bd 100644 --- a/p2p/protocols/protocol_test.go +++ b/p2p/protocols/protocol_test.go @@ -154,18 +154,18 @@ func protocolTester(t *testing.T, pp *p2ptest.TestPeerPool) *p2ptest.ProtocolTes func protoHandshakeExchange(id discover.NodeID, proto *protoHandshake) []p2ptest.Exchange { return []p2ptest.Exchange{ - p2ptest.Exchange{ + { Expects: []p2ptest.Expect{ - p2ptest.Expect{ + { Code: 0, Msg: &protoHandshake{42, "420"}, Peer: id, }, }, }, - p2ptest.Exchange{ + { Triggers: []p2ptest.Trigger{ - p2ptest.Trigger{ + { Code: 0, Msg: proto, Peer: id, @@ -207,18 +207,18 @@ func TestProtoHandshakeSuccess(t *testing.T) { func moduleHandshakeExchange(id discover.NodeID, resp uint) []p2ptest.Exchange { return []p2ptest.Exchange{ - p2ptest.Exchange{ + { Expects: []p2ptest.Expect{ - p2ptest.Expect{ + { Code: 1, Msg: &hs0{42}, Peer: id, }, }, }, - p2ptest.Exchange{ + { Triggers: []p2ptest.Trigger{ - p2ptest.Trigger{ + { Code: 1, Msg: &hs0{resp}, Peer: id, @@ -255,42 +255,42 @@ func TestModuleHandshakeSuccess(t *testing.T) { func testMultiPeerSetup(a, b discover.NodeID) []p2ptest.Exchange { return []p2ptest.Exchange{ - p2ptest.Exchange{ + { Label: "primary handshake", Expects: []p2ptest.Expect{ - p2ptest.Expect{ + { Code: 0, Msg: &protoHandshake{42, "420"}, Peer: a, }, - p2ptest.Expect{ + { Code: 0, Msg: &protoHandshake{42, "420"}, Peer: b, }, }, }, - p2ptest.Exchange{ + { Label: "module handshake", Triggers: []p2ptest.Trigger{ - p2ptest.Trigger{ + { Code: 0, Msg: &protoHandshake{42, "420"}, Peer: a, }, - p2ptest.Trigger{ + { Code: 0, Msg: &protoHandshake{42, "420"}, Peer: b, }, }, Expects: []p2ptest.Expect{ - p2ptest.Expect{ + { Code: 1, Msg: &hs0{42}, Peer: a, }, - p2ptest.Expect{ + { Code: 1, Msg: &hs0{42}, Peer: b, @@ -298,10 +298,10 @@ func testMultiPeerSetup(a, b discover.NodeID) []p2ptest.Exchange { }, }, - p2ptest.Exchange{Label: "alternative module handshake", Triggers: []p2ptest.Trigger{p2ptest.Trigger{Code: 1, Msg: &hs0{41}, Peer: a}, - p2ptest.Trigger{Code: 1, Msg: &hs0{41}, Peer: b}}}, - p2ptest.Exchange{Label: "repeated module handshake", Triggers: []p2ptest.Trigger{p2ptest.Trigger{Code: 1, Msg: &hs0{1}, Peer: a}}}, - p2ptest.Exchange{Label: "receiving repeated module handshake", Expects: []p2ptest.Expect{p2ptest.Expect{Code: 1, Msg: &hs0{43}, Peer: a}}}} + {Label: "alternative module handshake", Triggers: []p2ptest.Trigger{{Code: 1, Msg: &hs0{41}, Peer: a}, + {Code: 1, Msg: &hs0{41}, Peer: b}}}, + {Label: "repeated module handshake", Triggers: []p2ptest.Trigger{{Code: 1, Msg: &hs0{1}, Peer: a}}}, + {Label: "receiving repeated module handshake", Expects: []p2ptest.Expect{{Code: 1, Msg: &hs0{43}, Peer: a}}}} } func runMultiplePeers(t *testing.T, peer int, errs ...error) { @@ -332,7 +332,7 @@ func runMultiplePeers(t *testing.T, peer int, errs ...error) { // peer 0 sends kill request for peer with index s.TestExchanges(p2ptest.Exchange{ Triggers: []p2ptest.Trigger{ - p2ptest.Trigger{ + { Code: 2, Msg: &kill{s.IDs[peer]}, Peer: s.IDs[0], @@ -343,7 +343,7 @@ func runMultiplePeers(t *testing.T, peer int, errs ...error) { // the peer not killed sends a drop request s.TestExchanges(p2ptest.Exchange{ Triggers: []p2ptest.Trigger{ - p2ptest.Trigger{ + { Code: 3, Msg: &drop{}, Peer: s.IDs[(peer+1)%2], diff --git a/pot/doc.go b/pot/doc.go index 47d0357d93..4c0a03065d 100644 --- a/pot/doc.go +++ b/pot/doc.go @@ -48,8 +48,8 @@ concurrent routines, Pot * retrieval, insertion and deletion by key involves log(n) pointer lookups * for any item retrieval (defined as common prefix on the binary key) -* provide syncronous iterators respecting proximity ordering wrt any item -* provide asyncronous iterator (for parallel execution of operations) over n items +* provide synchronous iterators respecting proximity ordering wrt any item +* provide asynchronous iterator (for parallel execution of operations) over n items * allows cheap iteration over ranges * asymmetric concurrent merge (union) diff --git a/pot/pot.go b/pot/pot.go index 87f51af49c..dfda84804d 100644 --- a/pot/pot.go +++ b/pot/pot.go @@ -559,7 +559,7 @@ func (t *Pot) eachBin(val Val, pof Pof, po int, f func(int, int, func(func(val V } -// EachNeighbour is a syncronous iterator over neighbours of any target val +// EachNeighbour is a synchronous iterator over neighbours of any target val // the order of elements retrieved reflect proximity order to the target // TODO: add maximum proxbin to start range of iteration func (t *Pot) EachNeighbour(val Val, pof Pof, f func(Val, int) bool) bool { @@ -615,7 +615,7 @@ func (t *Pot) eachNeighbour(val Val, pof Pof, f func(Val, int) bool) bool { return true } -// EachNeighbourAsync called on (val, max, maxPos, f, wait) is an asyncronous iterator +// EachNeighbourAsync called on (val, max, maxPos, f, wait) is an asynchronous iterator // over elements not closer than maxPos wrt val. // val does not need to be match an element of the Pot, but if it does, and // maxPos is keylength than it is included in the iteration @@ -762,7 +762,7 @@ func (t *Pot) eachNeighbourAsync(val Val, pof Pof, max int, maxPos int, f func(V // getPos called on (n) returns the forking node at PO n and its index if it exists // otherwise nil -// caller is suppoed to hold the lock +// caller is supposed to hold the lock func (t *Pot) getPos(po int) (n *Pot, i int) { for i, n = range t.bins { if po > n.po { diff --git a/swarm/network/discovery_test.go b/swarm/network/discovery_test.go index ee90683a73..020a9b80dc 100644 --- a/swarm/network/discovery_test.go +++ b/swarm/network/discovery_test.go @@ -47,7 +47,7 @@ func TestDiscovery(t *testing.T) { s.TestExchanges(p2ptest.Exchange{ Label: "outgoing SubPeersMsg", Expects: []p2ptest.Expect{ - p2ptest.Expect{ + { Code: 3, Msg: &subPeersMsg{Depth: 0}, Peer: s.ProtocolTester.IDs[0], diff --git a/swarm/network/hive_test.go b/swarm/network/hive_test.go index 8e49e9029d..f55f14d5a4 100644 --- a/swarm/network/hive_test.go +++ b/swarm/network/hive_test.go @@ -46,7 +46,7 @@ func TestRegisterAndConnect(t *testing.T) { s.TestExchanges(p2ptest.Exchange{ Label: "getPeersMsg message", Expects: []p2ptest.Expect{ - p2ptest.Expect{ + { Code: 2, Msg: &subPeersMsg{0}, Peer: id, diff --git a/swarm/network/kademlia.go b/swarm/network/kademlia.go index 376ba9ad5a..d7bb7be6d7 100644 --- a/swarm/network/kademlia.go +++ b/swarm/network/kademlia.go @@ -424,7 +424,7 @@ func (k *Kademlia) callable(val pot.Val) OverlayAddr { return e.addr() } -// BaseAddr return the kademlia base addres +// BaseAddr return the kademlia base address func (k *Kademlia) BaseAddr() []byte { return k.base } diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index 22c8334030..d755f4a3fa 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -252,7 +252,7 @@ type bzzPeer struct { lastActive time.Time // time is updated whenever mutexes are releasing } -// Off returns the overlay peer record for offline persistance +// Off returns the overlay peer record for offline persistence func (p *bzzPeer) Off() OverlayAddr { return p.BzzAddr } diff --git a/swarm/network/protocol_test.go b/swarm/network/protocol_test.go index 1d7e165f02..e8ec4ebc37 100644 --- a/swarm/network/protocol_test.go +++ b/swarm/network/protocol_test.go @@ -57,18 +57,18 @@ func (t *testStore) Save(key string, v []byte) error { func HandshakeMsgExchange(lhs, rhs *HandshakeMsg, id discover.NodeID) []p2ptest.Exchange { return []p2ptest.Exchange{ - p2ptest.Exchange{ + { Expects: []p2ptest.Expect{ - p2ptest.Expect{ + { Code: 0, Msg: lhs, Peer: id, }, }, }, - p2ptest.Exchange{ + { Triggers: []p2ptest.Trigger{ - p2ptest.Trigger{ + { Code: 0, Msg: rhs, Peer: id, diff --git a/swarm/pss/handshake.go b/swarm/pss/handshake.go index 15f2a32a00..80aa729111 100644 --- a/swarm/pss/handshake.go +++ b/swarm/pss/handshake.go @@ -254,7 +254,7 @@ func (self *HandshakeController) cleanHandshake(pubkeyid string, topic *Topic, i func (self *HandshakeController) clean() { peerpubkeys := self.handshakes for pubkeyid, peertopics := range peerpubkeys { - for topic, _ := range peertopics { + for topic := range peertopics { self.cleanHandshake(pubkeyid, &topic, true, true) } } @@ -475,7 +475,7 @@ func (self *HandshakeAPI) AddHandshake(topic Topic) error { return nil } -// Deactivate handshake functionalty on a topic +// Deactivate handshake functionality on a topic func (self *HandshakeAPI) RemoveHandshake(topic *Topic) error { if _, ok := self.ctrl.deregisterFuncs[*topic]; ok { self.ctrl.deregisterFuncs[*topic]() diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index 0cd226ba08..5a35aba8c5 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -418,7 +418,7 @@ func (self *Pss) generateSymmetricKey(topic Topic, address *PssAddress, addToCac // If addtocache is set to true, the key will be added to the cache of keys // used to attempt symmetric decryption of incoming messages. // -// Returns a string id that can be used to retreive the key bytes +// Returns a string id that can be used to retrieve the key bytes // from the whisper backend (see pss.GetSymmetricKey()) func (self *Pss) SetSymmetricKey(key []byte, topic Topic, address *PssAddress, addtocache bool) (string, error) { keyid, err := self.w.AddSymKeyDirect(key) diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index 35f85e93f8..90e7e3ecb0 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -869,7 +869,7 @@ func benchmarkSymKeySend(b *testing.B) { } symkey, err := ps.w.GetSymKey(symkeyid) if err != nil { - b.Fatalf("could not retreive symkey: %v", err) + b.Fatalf("could not retrieve symkey: %v", err) } ps.SetSymmetricKey(symkey, topic, &to, false) @@ -962,7 +962,7 @@ func benchmarkSymkeyBruteforceChangeaddr(b *testing.B) { } symkey, err := ps.w.GetSymKey(keyid) if err != nil { - b.Fatalf("could not retreive symkey %s: %v", keyid, err) + b.Fatalf("could not retrieve symkey %s: %v", keyid, err) } wparams := &whisper.MessageParams{ TTL: defaultWhisperTTL, @@ -1046,7 +1046,7 @@ func benchmarkSymkeyBruteforceSameaddr(b *testing.B) { } symkey, err := ps.w.GetSymKey(keyid) if err != nil { - b.Fatalf("could not retreive symkey %s: %v", keyid, err) + b.Fatalf("could not retrieve symkey %s: %v", keyid, err) } wparams := &whisper.MessageParams{ TTL: defaultWhisperTTL, diff --git a/swarm/storage/chunker_test.go b/swarm/storage/chunker_test.go index 6b828970b6..19712a4709 100644 --- a/swarm/storage/chunker_test.go +++ b/swarm/storage/chunker_test.go @@ -113,7 +113,9 @@ func (self *chunkerTester) Append(chunker Splitter, rootKey Key, data io.Reader, // getting data chunk.SData = stored.SData chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) - close(chunk.C) + if chunk.C != nil { + close(chunk.C) + } } } } From fe51ebd31b71d032fddf526c2efae9876348ee6b Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Mon, 8 Jan 2018 12:22:11 +0100 Subject: [PATCH 015/107] p2p/simulations: fail test if pipe setup fails --- p2p/simulations/adapters/inproc_test.go | 30 ++++++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/p2p/simulations/adapters/inproc_test.go b/p2p/simulations/adapters/inproc_test.go index 76be7228d1..8ef65e1c87 100644 --- a/p2p/simulations/adapters/inproc_test.go +++ b/p2p/simulations/adapters/inproc_test.go @@ -25,7 +25,10 @@ import ( ) func TestSocketPipe(t *testing.T) { - c1, c2, _ := socketPipe() + c1, c2, err := socketPipe() + if err != nil { + t.Fatal(err) + } done := make(chan struct{}) @@ -67,7 +70,10 @@ func TestSocketPipe(t *testing.T) { } func TestSocketPipeBidirections(t *testing.T) { - c1, c2, _ := socketPipe() + c1, c2, err := socketPipe() + if err != nil { + t.Fatal(err) + } done := make(chan struct{}) @@ -124,7 +130,10 @@ func TestSocketPipeBidirections(t *testing.T) { } func TestTcpPipe(t *testing.T) { - c1, c2, _ := tcpPipe() + c1, c2, err := tcpPipe() + if err != nil { + t.Fatal(err) + } done := make(chan struct{}) @@ -166,7 +175,10 @@ func TestTcpPipe(t *testing.T) { } func TestTcpPipeBidirections(t *testing.T) { - c1, c2, _ := tcpPipe() + c1, c2, err := tcpPipe() + if err != nil { + t.Fatal(err) + } done := make(chan struct{}) @@ -226,7 +238,10 @@ func TestTcpPipeBidirections(t *testing.T) { } func TestNetPipe(t *testing.T) { - c1, c2, _ := netPipe() + c1, c2, err := netPipe() + if err != nil { + t.Fatal(err) + } done := make(chan struct{}) @@ -272,7 +287,10 @@ func TestNetPipe(t *testing.T) { } func TestNetPipeBidirections(t *testing.T) { - c1, c2, _ := netPipe() + c1, c2, err := netPipe() + if err != nil { + t.Fatal(err) + } done := make(chan struct{}) From d50fb250f0609c1aed9ed236b17f60f21e4eac43 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Mon, 8 Jan 2018 13:23:58 +0100 Subject: [PATCH 016/107] swarm/network: output substr of hive output --- swarm/network/kademlia_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/swarm/network/kademlia_test.go b/swarm/network/kademlia_test.go index 9597abbf35..5c3b847c05 100644 --- a/swarm/network/kademlia_test.go +++ b/swarm/network/kademlia_test.go @@ -406,6 +406,7 @@ func TestKademliaHiveString(t *testing.T) { } expH += "=========================================================================" if expH[100:] != h[100:] { - t.Fatalf("incorrect hive output. expected %v, got %v", expH, h) + t.Errorf("incorrect hive output. full - expected %v, got %v", expH, h) + t.Fatalf("incorrect hive output. substr - expected %v, got %v", expH[100:], h[100:]) } } From ca3bec16873436cda0da03f5c8055191616df06a Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Mon, 8 Jan 2018 17:50:56 +0100 Subject: [PATCH 017/107] swarm/pss: Run gofmt on pss.go --- swarm/pss/pss.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index 5a35aba8c5..ed0ea69221 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -190,7 +190,7 @@ var pssSpec = &protocols.Spec{ func (self *Pss) Protocols() []p2p.Protocol { return []p2p.Protocol{ - p2p.Protocol{ + { Name: pssSpec.Name, Version: pssSpec.Version, Length: pssSpec.Length(), @@ -209,7 +209,7 @@ func (self *Pss) Run(p *p2p.Peer, rw p2p.MsgReadWriter) error { func (self *Pss) APIs() []rpc.API { apis := []rpc.API{ - rpc.API{ + { Namespace: "pss", Version: "1.0", Service: NewAPI(self), From 9ac70ae69a06d2194bc7ff94633a0ed0560c2f6d Mon Sep 17 00:00:00 2001 From: lash Date: Tue, 9 Jan 2018 22:54:57 +0100 Subject: [PATCH 018/107] swarm/network: Omit date from hive string output --- swarm/network/kademlia_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/network/kademlia_test.go b/swarm/network/kademlia_test.go index 5c3b847c05..7e3c752dc2 100644 --- a/swarm/network/kademlia_test.go +++ b/swarm/network/kademlia_test.go @@ -405,7 +405,7 @@ func TestKademliaHiveString(t *testing.T) { expH += fmt.Sprintf("%03d 0 | 0\n", i) } expH += "=========================================================================" - if expH[100:] != h[100:] { + if expH[106:] != h[106:] { t.Errorf("incorrect hive output. full - expected %v, got %v", expH, h) t.Fatalf("incorrect hive output. substr - expected %v, got %v", expH[100:], h[100:]) } From 5e2a42960d964edb32cf70ba9b8df2f8fb592f26 Mon Sep 17 00:00:00 2001 From: lash Date: Wed, 10 Jan 2018 00:49:53 +0100 Subject: [PATCH 019/107] swarm/pss: Omit build of tests using t.Name for go 1.7 --- swarm/pss/handshake_test.go | 2 + swarm/pss/protocol_test.go | 2 + swarm/pss/pss_go18plus_test.go | 465 +++++++++++++++++++++++++++++++++ swarm/pss/pss_test.go | 443 ------------------------------- 4 files changed, 469 insertions(+), 443 deletions(-) create mode 100644 swarm/pss/pss_go18plus_test.go diff --git a/swarm/pss/handshake_test.go b/swarm/pss/handshake_test.go index 25620fb235..6cebdbbd5a 100644 --- a/swarm/pss/handshake_test.go +++ b/swarm/pss/handshake_test.go @@ -1,3 +1,5 @@ +//-build go1.7 + package pss import ( diff --git a/swarm/pss/protocol_test.go b/swarm/pss/protocol_test.go index b30fc0430d..793faa1397 100644 --- a/swarm/pss/protocol_test.go +++ b/swarm/pss/protocol_test.go @@ -1,3 +1,5 @@ +//-build go1.7 + package pss import ( diff --git a/swarm/pss/pss_go18plus_test.go b/swarm/pss/pss_go18plus_test.go new file mode 100644 index 0000000000..54e5b5f386 --- /dev/null +++ b/swarm/pss/pss_go18plus_test.go @@ -0,0 +1,465 @@ +//-build go17 +package pss + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/json" + "fmt" + "io/ioutil" + "math/rand" + "os" + "strconv" + "strings" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/simulations" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/swarm/network" +) + +// send symmetrically encrypted message between two directly connected peers +func TestSymSend(t *testing.T) { + t.Run("32", testSymSend) + t.Run("8", testSymSend) + t.Run("0", testSymSend) +} + +func testSymSend(t *testing.T) { + + // address hint size + var addrsize int64 + var err error + paramstring := strings.Split(t.Name(), "/") + addrsize, _ = strconv.ParseInt(paramstring[1], 10, 0) + log.Info("sym send test", "addrsize", addrsize) + + clients, err := setupNetwork(2) + if err != nil { + t.Fatal(err) + } + + var topic string + err = clients[0].Call(&topic, "pss_stringToTopic", "foo:42") + if err != nil { + t.Fatal(err) + } + + var loaddrhex string + err = clients[0].Call(&loaddrhex, "pss_baseAddr") + if err != nil { + t.Fatalf("rpc get node 1 baseaddr fail: %v", err) + } + loaddrhex = loaddrhex[:2+(addrsize*2)] + var roaddrhex string + err = clients[1].Call(&roaddrhex, "pss_baseAddr") + if err != nil { + t.Fatalf("rpc get node 2 baseaddr fail: %v", err) + } + roaddrhex = roaddrhex[:2+(addrsize*2)] + + // retrieve public key from pss instance + // set this public key reciprocally + var lpubkeyhex string + err = clients[0].Call(&lpubkeyhex, "pss_getPublicKey") + if err != nil { + t.Fatalf("rpc get node 1 pubkey fail: %v", err) + } + var rpubkeyhex string + err = clients[1].Call(&rpubkeyhex, "pss_getPublicKey") + if err != nil { + t.Fatalf("rpc get node 2 pubkey fail: %v", err) + } + + time.Sleep(time.Millisecond * 500) + + // at this point we've verified that symkeys are saved and match on each peer + // now try sending symmetrically encrypted message, both directions + lmsgC := make(chan APIMsg) + lctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic) + log.Trace("lsub", "id", lsub) + defer lsub.Unsubscribe() + rmsgC := make(chan APIMsg) + rctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic) + log.Trace("rsub", "id", rsub) + defer rsub.Unsubscribe() + + lrecvkey := network.RandomAddr().Over() + rrecvkey := network.RandomAddr().Over() + + var lkeyids [2]string + var rkeyids [2]string + + // manually set reciprocal symkeys + err = clients[0].Call(&lkeyids, "psstest_setSymKeys", rpubkeyhex, lrecvkey, rrecvkey, defaultSymKeySendLimit, topic, roaddrhex) + if err != nil { + t.Fatal(err) + } + err = clients[1].Call(&rkeyids, "psstest_setSymKeys", rpubkeyhex, rrecvkey, lrecvkey, defaultSymKeySendLimit, topic, loaddrhex) + if err != nil { + t.Fatal(err) + } + + // send and verify delivery + lmsg := []byte("plugh") + err = clients[1].Call(nil, "pss_sendSym", rkeyids[1], topic, hexutil.Encode(lmsg)) + if err != nil { + t.Fatal(err) + } + select { + case recvmsg := <-lmsgC: + if !bytes.Equal(recvmsg.Msg, lmsg) { + t.Fatalf("node 1 received payload mismatch: expected %v, got %v", lmsg, recvmsg) + } + case cerr := <-lctx.Done(): + t.Fatalf("test message timed out: %v", cerr) + } + rmsg := []byte("xyzzy") + err = clients[0].Call(nil, "pss_sendSym", lkeyids[1], topic, hexutil.Encode(rmsg)) + if err != nil { + t.Fatal(err) + } + select { + case recvmsg := <-rmsgC: + if !bytes.Equal(recvmsg.Msg, rmsg) { + t.Fatalf("node 2 received payload mismatch: expected %v, got %v", rmsg, recvmsg.Msg) + } + case cerr := <-rctx.Done(): + t.Fatalf("test message timed out: %v", cerr) + } +} + +// send asymmetrically encrypted message between two directly connected peers +func TestAsymSend(t *testing.T) { + t.Run("32", testAsymSend) + t.Run("8", testAsymSend) + t.Run("0", testAsymSend) +} + +func testAsymSend(t *testing.T) { + + // address hint size + var addrsize int64 + var err error + paramstring := strings.Split(t.Name(), "/") + addrsize, _ = strconv.ParseInt(paramstring[1], 10, 0) + log.Info("asym send test", "addrsize", addrsize) + + clients, err := setupNetwork(2) + if err != nil { + t.Fatal(err) + } + + var topic string + err = clients[0].Call(&topic, "pss_stringToTopic", "foo:42") + if err != nil { + t.Fatal(err) + } + + time.Sleep(time.Millisecond * 250) + + var loaddrhex string + err = clients[0].Call(&loaddrhex, "pss_baseAddr") + if err != nil { + t.Fatalf("rpc get node 1 baseaddr fail: %v", err) + } + loaddrhex = loaddrhex[:2+(addrsize*2)] + var roaddrhex string + err = clients[1].Call(&roaddrhex, "pss_baseAddr") + if err != nil { + t.Fatalf("rpc get node 2 baseaddr fail: %v", err) + } + roaddrhex = roaddrhex[:2+(addrsize*2)] + + // retrieve public key from pss instance + // set this public key reciprocally + var lpubkey string + err = clients[0].Call(&lpubkey, "pss_getPublicKey") + if err != nil { + t.Fatalf("rpc get node 1 pubkey fail: %v", err) + } + var rpubkey string + err = clients[1].Call(&rpubkey, "pss_getPublicKey") + if err != nil { + t.Fatalf("rpc get node 2 pubkey fail: %v", err) + } + + time.Sleep(time.Millisecond * 500) // replace with hive healthy code + + lmsgC := make(chan APIMsg) + lctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic) + log.Trace("lsub", "id", lsub) + defer lsub.Unsubscribe() + rmsgC := make(chan APIMsg) + rctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic) + log.Trace("rsub", "id", rsub) + defer rsub.Unsubscribe() + + // store reciprocal public keys + err = clients[0].Call(nil, "pss_setPeerPublicKey", rpubkey, topic, roaddrhex) + if err != nil { + t.Fatal(err) + } + err = clients[1].Call(nil, "pss_setPeerPublicKey", lpubkey, topic, loaddrhex) + if err != nil { + t.Fatal(err) + } + + // send and verify delivery + rmsg := []byte("xyzzy") + err = clients[0].Call(nil, "pss_sendAsym", rpubkey, topic, hexutil.Encode(rmsg)) + if err != nil { + t.Fatal(err) + } + select { + case recvmsg := <-rmsgC: + if !bytes.Equal(recvmsg.Msg, rmsg) { + t.Fatalf("node 2 received payload mismatch: expected %v, got %v", rmsg, recvmsg.Msg) + } + case cerr := <-rctx.Done(): + t.Fatalf("test message timed out: %v", cerr) + } + lmsg := []byte("plugh") + err = clients[1].Call(nil, "pss_sendAsym", lpubkey, topic, hexutil.Encode(lmsg)) + if err != nil { + t.Fatal(err) + } + select { + case recvmsg := <-lmsgC: + if !bytes.Equal(recvmsg.Msg, lmsg) { + t.Fatalf("node 1 received payload mismatch: expected %v, got %v", lmsg, recvmsg.Msg) + } + case cerr := <-lctx.Done(): + t.Fatalf("test message timed out: %v", cerr) + } +} + +type Job struct { + Msg []byte + SendNode discover.NodeID + RecvNode discover.NodeID +} + +func worker(id int, jobs <-chan Job, rpcs map[discover.NodeID]*rpc.Client, pubkeys map[discover.NodeID]string, topic string) { + for j := range jobs { + rpcs[j.SendNode].Call(nil, "pss_sendAsym", pubkeys[j.RecvNode], topic, hexutil.Encode(j.Msg)) + } +} + +// params in run name: +// nodes/msgs/addrbytes/adaptertype +// if adaptertype is exec uses execadapter, simadapter otherwise +// +// ( some tests are commented out because of resource limitations on Travis) +func TestNetwork(t *testing.T) { + t.Skip("Temporarily deactivated because not all messages can be delivered") + //t.Run("3/2000/4/sock", testNetwork) + //t.Run("4/2000/4/sock", testNetwork) + t.Run("8/2000/4/sock", testNetwork) + t.Run("16/2000/4/sock", testNetwork) + t.Run("8/3000/4/sock", testNetwork) + t.Run("16/3000/4/sock", testNetwork) + //t.Run("32/2000/4/sock", testNetwork) + + t.Run("8/2000/4/sim", testNetwork) + t.Run("16/2000/4/sim", testNetwork) + t.Run("8/3000/4/sim", testNetwork) + t.Run("16/3000/4/sim", testNetwork) + //t.Run("32/2000/4/sim", testNetwork) + // t.Run("64/2000/4/sim", testNetwork) +} + +func testNetwork(t *testing.T) { + type msgnotifyC struct { + id discover.NodeID + msgIdx int + } + + paramstring := strings.Split(t.Name(), "/") + nodecount, _ := strconv.ParseInt(paramstring[1], 10, 0) + msgcount, _ := strconv.ParseInt(paramstring[2], 10, 0) + addrsize, _ := strconv.ParseInt(paramstring[3], 10, 0) + adapter := paramstring[4] + + log.Info("network test", "nodecount", nodecount, "msgcount", msgcount, "addrhintsize", addrsize) + + nodes := make([]discover.NodeID, nodecount) + bzzaddrs := make(map[discover.NodeID]string, nodecount) + rpcs := make(map[discover.NodeID]*rpc.Client, nodecount) + pubkeys := make(map[discover.NodeID]string, nodecount) + + sentmsgs := make([][]byte, msgcount) + recvmsgs := make([]bool, msgcount) + nodemsgcount := make(map[discover.NodeID]int, nodecount) + + trigger := make(chan discover.NodeID) + + var a adapters.NodeAdapter + if adapter == "exec" { + dirname, err := ioutil.TempDir(".", "") + if err != nil { + t.Fatal(err) + } + a = adapters.NewExecAdapter(dirname) + } else if adapter == "sock" { + a = adapters.NewSocketAdapter(services) + } else if adapter == "tcp" { + a = adapters.NewTCPAdapter(services) + } else if adapter == "sim" { + a = adapters.NewSimAdapter(services) + } + net := simulations.NewNetwork(a, &simulations.NetworkConfig{ + ID: "0", + }) + defer net.Shutdown() + + f, err := os.Open(fmt.Sprintf("testdata/snapshot_%d.json", nodecount)) + if err != nil { + t.Fatal(err) + } + jsonbyte, err := ioutil.ReadAll(f) + if err != nil { + t.Fatal(err) + } + var snap simulations.Snapshot + err = json.Unmarshal(jsonbyte, &snap) + if err != nil { + t.Fatal(err) + } + err = net.Load(&snap) + if err != nil { + t.Fatal(err) + } + + triggerChecks := func(trigger chan discover.NodeID, id discover.NodeID, rpcclient *rpc.Client, topic string) error { + msgC := make(chan APIMsg) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + sub, err := rpcclient.Subscribe(ctx, "pss", msgC, "receive", topic) + if err != nil { + t.Fatal(err) + } + go func() { + defer sub.Unsubscribe() + for { + select { + case recvmsg := <-msgC: + idx, _ := binary.Uvarint(recvmsg.Msg) + if recvmsgs[idx] == false { + log.Debug("msg recv", "idx", idx, "id", id) + recvmsgs[idx] = true + trigger <- id + } + case <-sub.Err(): + return + } + } + }() + return nil + } + + var topic string + for i, nod := range net.GetNodes() { + nodes[i] = nod.ID() + rpcs[nodes[i]], err = nod.Client() + if err != nil { + t.Fatal(err) + } + if topic == "" { + err = rpcs[nodes[i]].Call(&topic, "pss_stringToTopic", "foo:42") + if err != nil { + t.Fatal(err) + } + } + var pubkey string + err = rpcs[nodes[i]].Call(&pubkey, "pss_getPublicKey") + if err != nil { + t.Fatal(err) + } + pubkeys[nod.ID()] = pubkey + var addrhex string + err = rpcs[nodes[i]].Call(&addrhex, "pss_baseAddr") + if err != nil { + t.Fatal(err) + } + bzzaddrs[nodes[i]] = addrhex + err = triggerChecks(trigger, nodes[i], rpcs[nodes[i]], topic) + if err != nil { + t.Fatal(err) + } + } + + // setup workers + jobs := make(chan Job, 10) + for w := 1; w <= 10; w++ { + go worker(w, jobs, rpcs, pubkeys, topic) + } + + for i := 0; i < int(msgcount); i++ { + sendnodeidx := rand.Intn(int(nodecount)) + recvnodeidx := rand.Intn(int(nodecount - 1)) + if recvnodeidx >= sendnodeidx { + recvnodeidx++ + } + nodemsgcount[nodes[recvnodeidx]]++ + sentmsgs[i] = make([]byte, 8) + c := binary.PutUvarint(sentmsgs[i], uint64(i)) + if c == 0 { + t.Fatal("0 byte message") + } + if err != nil { + t.Fatal(err) + } + err = rpcs[nodes[sendnodeidx]].Call(nil, "pss_setPeerPublicKey", pubkeys[nodes[recvnodeidx]], topic, bzzaddrs[nodes[recvnodeidx]]) + if err != nil { + t.Fatal(err) + } + + jobs <- Job{ + Msg: sentmsgs[i], + SendNode: nodes[sendnodeidx], + RecvNode: nodes[recvnodeidx], + } + } + + finalmsgcount := 0 + ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) + defer cancel() +outer: + for i := 0; i < int(msgcount); i++ { + select { + case id := <-trigger: + nodemsgcount[id]-- + finalmsgcount++ + case <-ctx.Done(): + log.Warn("timeout") + break outer + } + } + + for i, msg := range recvmsgs { + if !msg { + log.Debug("missing message", "idx", i) + } + } + t.Logf("%d of %d messages received", finalmsgcount, msgcount) + + if finalmsgcount != int(msgcount) { + t.Fatalf("%d messages were not received", int(msgcount)-finalmsgcount) + } + +} diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index 90e7e3ecb0..d4c11ae74d 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -4,9 +4,7 @@ import ( "bytes" "context" "crypto/ecdsa" - "encoding/binary" "encoding/hex" - "encoding/json" "flag" "fmt" "io/ioutil" @@ -19,7 +17,6 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -395,446 +392,6 @@ func TestMismatch(t *testing.T) { } -// send symmetrically encrypted message between two directly connected peers -func TestSymSend(t *testing.T) { - t.Run("32", testSymSend) - t.Run("8", testSymSend) - t.Run("0", testSymSend) -} - -func testSymSend(t *testing.T) { - - // address hint size - var addrsize int64 - var err error - paramstring := strings.Split(t.Name(), "/") - addrsize, _ = strconv.ParseInt(paramstring[1], 10, 0) - log.Info("sym send test", "addrsize", addrsize) - - clients, err := setupNetwork(2) - if err != nil { - t.Fatal(err) - } - - var topic string - err = clients[0].Call(&topic, "pss_stringToTopic", "foo:42") - if err != nil { - t.Fatal(err) - } - - var loaddrhex string - err = clients[0].Call(&loaddrhex, "pss_baseAddr") - if err != nil { - t.Fatalf("rpc get node 1 baseaddr fail: %v", err) - } - loaddrhex = loaddrhex[:2+(addrsize*2)] - var roaddrhex string - err = clients[1].Call(&roaddrhex, "pss_baseAddr") - if err != nil { - t.Fatalf("rpc get node 2 baseaddr fail: %v", err) - } - roaddrhex = roaddrhex[:2+(addrsize*2)] - - // retrieve public key from pss instance - // set this public key reciprocally - var lpubkeyhex string - err = clients[0].Call(&lpubkeyhex, "pss_getPublicKey") - if err != nil { - t.Fatalf("rpc get node 1 pubkey fail: %v", err) - } - var rpubkeyhex string - err = clients[1].Call(&rpubkeyhex, "pss_getPublicKey") - if err != nil { - t.Fatalf("rpc get node 2 pubkey fail: %v", err) - } - - time.Sleep(time.Millisecond * 500) - - // at this point we've verified that symkeys are saved and match on each peer - // now try sending symmetrically encrypted message, both directions - lmsgC := make(chan APIMsg) - lctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic) - log.Trace("lsub", "id", lsub) - defer lsub.Unsubscribe() - rmsgC := make(chan APIMsg) - rctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic) - log.Trace("rsub", "id", rsub) - defer rsub.Unsubscribe() - - lrecvkey := network.RandomAddr().Over() - rrecvkey := network.RandomAddr().Over() - - var lkeyids [2]string - var rkeyids [2]string - - // manually set reciprocal symkeys - err = clients[0].Call(&lkeyids, "psstest_setSymKeys", rpubkeyhex, lrecvkey, rrecvkey, defaultSymKeySendLimit, topic, roaddrhex) - if err != nil { - t.Fatal(err) - } - err = clients[1].Call(&rkeyids, "psstest_setSymKeys", rpubkeyhex, rrecvkey, lrecvkey, defaultSymKeySendLimit, topic, loaddrhex) - if err != nil { - t.Fatal(err) - } - - // send and verify delivery - lmsg := []byte("plugh") - err = clients[1].Call(nil, "pss_sendSym", rkeyids[1], topic, hexutil.Encode(lmsg)) - if err != nil { - t.Fatal(err) - } - select { - case recvmsg := <-lmsgC: - if !bytes.Equal(recvmsg.Msg, lmsg) { - t.Fatalf("node 1 received payload mismatch: expected %v, got %v", lmsg, recvmsg) - } - case cerr := <-lctx.Done(): - t.Fatalf("test message timed out: %v", cerr) - } - rmsg := []byte("xyzzy") - err = clients[0].Call(nil, "pss_sendSym", lkeyids[1], topic, hexutil.Encode(rmsg)) - if err != nil { - t.Fatal(err) - } - select { - case recvmsg := <-rmsgC: - if !bytes.Equal(recvmsg.Msg, rmsg) { - t.Fatalf("node 2 received payload mismatch: expected %v, got %v", rmsg, recvmsg.Msg) - } - case cerr := <-rctx.Done(): - t.Fatalf("test message timed out: %v", cerr) - } -} - -// send asymmetrically encrypted message between two directly connected peers -func TestAsymSend(t *testing.T) { - t.Run("32", testAsymSend) - t.Run("8", testAsymSend) - t.Run("0", testAsymSend) -} - -func testAsymSend(t *testing.T) { - - // address hint size - var addrsize int64 - var err error - paramstring := strings.Split(t.Name(), "/") - addrsize, _ = strconv.ParseInt(paramstring[1], 10, 0) - log.Info("asym send test", "addrsize", addrsize) - - clients, err := setupNetwork(2) - if err != nil { - t.Fatal(err) - } - - var topic string - err = clients[0].Call(&topic, "pss_stringToTopic", "foo:42") - if err != nil { - t.Fatal(err) - } - - time.Sleep(time.Millisecond * 250) - - var loaddrhex string - err = clients[0].Call(&loaddrhex, "pss_baseAddr") - if err != nil { - t.Fatalf("rpc get node 1 baseaddr fail: %v", err) - } - loaddrhex = loaddrhex[:2+(addrsize*2)] - var roaddrhex string - err = clients[1].Call(&roaddrhex, "pss_baseAddr") - if err != nil { - t.Fatalf("rpc get node 2 baseaddr fail: %v", err) - } - roaddrhex = roaddrhex[:2+(addrsize*2)] - - // retrieve public key from pss instance - // set this public key reciprocally - var lpubkey string - err = clients[0].Call(&lpubkey, "pss_getPublicKey") - if err != nil { - t.Fatalf("rpc get node 1 pubkey fail: %v", err) - } - var rpubkey string - err = clients[1].Call(&rpubkey, "pss_getPublicKey") - if err != nil { - t.Fatalf("rpc get node 2 pubkey fail: %v", err) - } - - time.Sleep(time.Millisecond * 500) // replace with hive healthy code - - lmsgC := make(chan APIMsg) - lctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic) - log.Trace("lsub", "id", lsub) - defer lsub.Unsubscribe() - rmsgC := make(chan APIMsg) - rctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic) - log.Trace("rsub", "id", rsub) - defer rsub.Unsubscribe() - - // store reciprocal public keys - err = clients[0].Call(nil, "pss_setPeerPublicKey", rpubkey, topic, roaddrhex) - if err != nil { - t.Fatal(err) - } - err = clients[1].Call(nil, "pss_setPeerPublicKey", lpubkey, topic, loaddrhex) - if err != nil { - t.Fatal(err) - } - - // send and verify delivery - rmsg := []byte("xyzzy") - err = clients[0].Call(nil, "pss_sendAsym", rpubkey, topic, hexutil.Encode(rmsg)) - if err != nil { - t.Fatal(err) - } - select { - case recvmsg := <-rmsgC: - if !bytes.Equal(recvmsg.Msg, rmsg) { - t.Fatalf("node 2 received payload mismatch: expected %v, got %v", rmsg, recvmsg.Msg) - } - case cerr := <-rctx.Done(): - t.Fatalf("test message timed out: %v", cerr) - } - lmsg := []byte("plugh") - err = clients[1].Call(nil, "pss_sendAsym", lpubkey, topic, hexutil.Encode(lmsg)) - if err != nil { - t.Fatal(err) - } - select { - case recvmsg := <-lmsgC: - if !bytes.Equal(recvmsg.Msg, lmsg) { - t.Fatalf("node 1 received payload mismatch: expected %v, got %v", lmsg, recvmsg.Msg) - } - case cerr := <-lctx.Done(): - t.Fatalf("test message timed out: %v", cerr) - } -} - -type Job struct { - Msg []byte - SendNode discover.NodeID - RecvNode discover.NodeID -} - -func worker(id int, jobs <-chan Job, rpcs map[discover.NodeID]*rpc.Client, pubkeys map[discover.NodeID]string, topic string) { - for j := range jobs { - rpcs[j.SendNode].Call(nil, "pss_sendAsym", pubkeys[j.RecvNode], topic, hexutil.Encode(j.Msg)) - } -} - -// params in run name: -// nodes/msgs/addrbytes/adaptertype -// if adaptertype is exec uses execadapter, simadapter otherwise -// -// ( some tests are commented out because of resource limitations on Travis) -func TestNetwork(t *testing.T) { - t.Skip("Temporarily deactivated because not all messages can be delivered") - //t.Run("3/2000/4/sock", testNetwork) - //t.Run("4/2000/4/sock", testNetwork) - t.Run("8/2000/4/sock", testNetwork) - t.Run("16/2000/4/sock", testNetwork) - t.Run("8/3000/4/sock", testNetwork) - t.Run("16/3000/4/sock", testNetwork) - //t.Run("32/2000/4/sock", testNetwork) - - t.Run("8/2000/4/sim", testNetwork) - t.Run("16/2000/4/sim", testNetwork) - t.Run("8/3000/4/sim", testNetwork) - t.Run("16/3000/4/sim", testNetwork) - //t.Run("32/2000/4/sim", testNetwork) - // t.Run("64/2000/4/sim", testNetwork) -} - -func testNetwork(t *testing.T) { - type msgnotifyC struct { - id discover.NodeID - msgIdx int - } - - paramstring := strings.Split(t.Name(), "/") - nodecount, _ := strconv.ParseInt(paramstring[1], 10, 0) - msgcount, _ := strconv.ParseInt(paramstring[2], 10, 0) - addrsize, _ := strconv.ParseInt(paramstring[3], 10, 0) - adapter := paramstring[4] - - log.Info("network test", "nodecount", nodecount, "msgcount", msgcount, "addrhintsize", addrsize) - - nodes := make([]discover.NodeID, nodecount) - bzzaddrs := make(map[discover.NodeID]string, nodecount) - rpcs := make(map[discover.NodeID]*rpc.Client, nodecount) - pubkeys := make(map[discover.NodeID]string, nodecount) - - sentmsgs := make([][]byte, msgcount) - recvmsgs := make([]bool, msgcount) - nodemsgcount := make(map[discover.NodeID]int, nodecount) - - trigger := make(chan discover.NodeID) - - var a adapters.NodeAdapter - if adapter == "exec" { - dirname, err := ioutil.TempDir(".", "") - if err != nil { - t.Fatal(err) - } - a = adapters.NewExecAdapter(dirname) - } else if adapter == "sock" { - a = adapters.NewSocketAdapter(services) - } else if adapter == "tcp" { - a = adapters.NewTCPAdapter(services) - } else if adapter == "sim" { - a = adapters.NewSimAdapter(services) - } - net := simulations.NewNetwork(a, &simulations.NetworkConfig{ - ID: "0", - }) - defer net.Shutdown() - - f, err := os.Open(fmt.Sprintf("testdata/snapshot_%d.json", nodecount)) - if err != nil { - t.Fatal(err) - } - jsonbyte, err := ioutil.ReadAll(f) - if err != nil { - t.Fatal(err) - } - var snap simulations.Snapshot - err = json.Unmarshal(jsonbyte, &snap) - if err != nil { - t.Fatal(err) - } - err = net.Load(&snap) - if err != nil { - t.Fatal(err) - } - - triggerChecks := func(trigger chan discover.NodeID, id discover.NodeID, rpcclient *rpc.Client, topic string) error { - msgC := make(chan APIMsg) - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - sub, err := rpcclient.Subscribe(ctx, "pss", msgC, "receive", topic) - if err != nil { - t.Fatal(err) - } - go func() { - defer sub.Unsubscribe() - for { - select { - case recvmsg := <-msgC: - idx, _ := binary.Uvarint(recvmsg.Msg) - if recvmsgs[idx] == false { - log.Debug("msg recv", "idx", idx, "id", id) - recvmsgs[idx] = true - trigger <- id - } - case <-sub.Err(): - return - } - } - }() - return nil - } - - var topic string - for i, nod := range net.GetNodes() { - nodes[i] = nod.ID() - rpcs[nodes[i]], err = nod.Client() - if err != nil { - t.Fatal(err) - } - if topic == "" { - err = rpcs[nodes[i]].Call(&topic, "pss_stringToTopic", "foo:42") - if err != nil { - t.Fatal(err) - } - } - var pubkey string - err = rpcs[nodes[i]].Call(&pubkey, "pss_getPublicKey") - if err != nil { - t.Fatal(err) - } - pubkeys[nod.ID()] = pubkey - var addrhex string - err = rpcs[nodes[i]].Call(&addrhex, "pss_baseAddr") - if err != nil { - t.Fatal(err) - } - bzzaddrs[nodes[i]] = addrhex - err = triggerChecks(trigger, nodes[i], rpcs[nodes[i]], topic) - if err != nil { - t.Fatal(err) - } - } - - // setup workers - jobs := make(chan Job, 10) - for w := 1; w <= 10; w++ { - go worker(w, jobs, rpcs, pubkeys, topic) - } - - for i := 0; i < int(msgcount); i++ { - sendnodeidx := rand.Intn(int(nodecount)) - recvnodeidx := rand.Intn(int(nodecount - 1)) - if recvnodeidx >= sendnodeidx { - recvnodeidx++ - } - nodemsgcount[nodes[recvnodeidx]]++ - sentmsgs[i] = make([]byte, 8) - c := binary.PutUvarint(sentmsgs[i], uint64(i)) - if c == 0 { - t.Fatal("0 byte message") - } - if err != nil { - t.Fatal(err) - } - err = rpcs[nodes[sendnodeidx]].Call(nil, "pss_setPeerPublicKey", pubkeys[nodes[recvnodeidx]], topic, bzzaddrs[nodes[recvnodeidx]]) - if err != nil { - t.Fatal(err) - } - - jobs <- Job{ - Msg: sentmsgs[i], - SendNode: nodes[sendnodeidx], - RecvNode: nodes[recvnodeidx], - } - } - - finalmsgcount := 0 - ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) - defer cancel() -outer: - for i := 0; i < int(msgcount); i++ { - select { - case id := <-trigger: - nodemsgcount[id]-- - finalmsgcount++ - case <-ctx.Done(): - log.Warn("timeout") - break outer - } - } - - for i, msg := range recvmsgs { - if !msg { - log.Debug("missing message", "idx", i) - } - } - t.Logf("%d of %d messages received", finalmsgcount, msgcount) - - if finalmsgcount != int(msgcount) { - t.Fatalf("%d messages were not received", int(msgcount)-finalmsgcount) - } - -} - // symmetric send performance with varying message sizes func BenchmarkSymkeySend(b *testing.B) { b.Run(fmt.Sprintf("%d", 256), benchmarkSymKeySend) From 1ae7e60d8dd91bd908618cb02f761d1be3b72833 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 10 Jan 2018 11:58:33 +0100 Subject: [PATCH 020/107] whisper: display err when failing to start server in tests --- whisper/whisperv6/peer_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/whisper/whisperv6/peer_test.go b/whisper/whisperv6/peer_test.go index 86868b653a..3275cc3f62 100644 --- a/whisper/whisperv6/peer_test.go +++ b/whisper/whisperv6/peer_test.go @@ -235,7 +235,7 @@ func initialize(t *testing.T) { // we need to wait until the first node actually starts err = nodes[0].server.Start() if err != nil { - t.Fatalf("failed to start the fisrt server.") + t.Fatal("failed to start the first server: ", err) } } From 0eb02fee4cb4afb2adc2657fc93860979449d31e Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 10 Jan 2018 12:02:33 +0100 Subject: [PATCH 021/107] p2p/sim: skip socketPipe tests if we cannot increase unix socket buffer size --- p2p/simulations/adapters/inproc_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/simulations/adapters/inproc_test.go b/p2p/simulations/adapters/inproc_test.go index 8ef65e1c87..5cece74e0c 100644 --- a/p2p/simulations/adapters/inproc_test.go +++ b/p2p/simulations/adapters/inproc_test.go @@ -27,7 +27,7 @@ import ( func TestSocketPipe(t *testing.T) { c1, c2, err := socketPipe() if err != nil { - t.Fatal(err) + t.Skip("system limit is less than desired. no buffer space available for socket. skipping test... err: ", err) } done := make(chan struct{}) @@ -72,7 +72,7 @@ func TestSocketPipe(t *testing.T) { func TestSocketPipeBidirections(t *testing.T) { c1, c2, err := socketPipe() if err != nil { - t.Fatal(err) + t.Skip("system limit is less than desired. no buffer space available for socket. skipping test... err: ", err) } done := make(chan struct{}) From 26187eada0bc6c8fa35518b6adabc9456f7e7d92 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 10 Jan 2018 12:15:49 +0100 Subject: [PATCH 022/107] swarm/network/sim: use sim adapter for network simulations --- swarm/network/simulations/discovery/discovery_test.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go index 7dd59beacc..17411cc8cf 100644 --- a/swarm/network/simulations/discovery/discovery_test.go +++ b/swarm/network/simulations/discovery/discovery_test.go @@ -98,12 +98,7 @@ func testDiscoverySimulationExecAdapter(t *testing.T, nodes, conns int) { } func TestDiscoverySimulationSimAdapter(t *testing.T) { - testDiscoverySimulationSimAdapter(t, *nodeCount, *initCount) -} - -func testDiscoverySimulationSimAdapter(t *testing.T, nodes, conns int) { - testDiscoverySimulation(t, nodes, conns, adapters.NewSocketAdapter(services)) - // testDiscoverySimulation(t, nodes, conns, adapters.NewSimAdapter(services)) + testDiscoverySimulation(t, *nodeCount, *initCount, adapters.NewSimAdapter(services)) } func testDiscoverySimulation(t *testing.T, nodes, conns int, adapter adapters.NodeAdapter) { From 020577f27dccf15ff3f47d9fcb64c6bafc531bbb Mon Sep 17 00:00:00 2001 From: lash Date: Sat, 13 Jan 2018 07:58:23 +0100 Subject: [PATCH 023/107] whisper: Remove server start inside goroutine --- whisper/whisperv6/peer_test.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/whisper/whisperv6/peer_test.go b/whisper/whisperv6/peer_test.go index 3275cc3f62..c892ade0b3 100644 --- a/whisper/whisperv6/peer_test.go +++ b/whisper/whisperv6/peer_test.go @@ -220,11 +220,6 @@ func initialize(t *testing.T) { }, } - err = node.server.Start() - if err != nil { - t.Skipf("failed to start server %d (port may be taken, skipping since there is no handler in test for this, should be ported to simulation framework): error is %v", i, err) - } - nodes[i] = &node } From d0c3029cdbdb063a1ec6b6b934739108ae7ce67b Mon Sep 17 00:00:00 2001 From: lash Date: Sun, 14 Jan 2018 09:40:47 +0100 Subject: [PATCH 024/107] swarm/pss: Remove comment to trigger travis --- swarm/pss/pss.go | 1 - 1 file changed, 1 deletion(-) diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index ed0ea69221..f49e488e59 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -22,7 +22,6 @@ import ( whisper "github.com/ethereum/go-ethereum/whisper/whisperv5" ) -// TODO: proper padding generation for messages const ( defaultPaddingByteSize = 16 defaultMsgTTL = time.Second * 8 From 63a44a80d67c9ae4bd319ffb6b4630541ce8b1a8 Mon Sep 17 00:00:00 2001 From: lash Date: Sun, 14 Jan 2018 20:58:54 +0100 Subject: [PATCH 025/107] pot, swarm/pss: Remove redundant conversions --- pot/address.go | 10 +++++----- swarm/pss/api.go | 2 +- swarm/pss/handshake.go | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pot/address.go b/pot/address.go index 350f15819a..039f8421f4 100644 --- a/pot/address.go +++ b/pot/address.go @@ -105,13 +105,13 @@ func posProximity(one, other Address, pos int) (ret int, eq bool) { if one[i] == other[i] { continue } - oxo := one[i] ^ other[i] + oxo := uint8(one[i] ^ other[i]) start := 0 if i == pos/8 { start = pos % 8 } for j := start; j < 8; j++ { - if (uint8(oxo)>>uint8(7-j))&0x01 != 0 { + if (oxo>>uint8(7-j))&0x01 != 0 { return i*8 + j, false } } @@ -173,13 +173,13 @@ func RandomAddress() Address { func NewAddressFromString(s string) []byte { ha := [32]byte{} - t := s + string(zerosBin)[:len(zerosBin)-len(s)] + t := s + zerosBin[:len(zerosBin)-len(s)] for i := 0; i < 4; i++ { n, err := strconv.ParseUint(t[i*64:(i+1)*64], 2, 64) if err != nil { panic("wrong format: " + err.Error()) } - binary.BigEndian.PutUint64(ha[i*8:(i+1)*8], uint64(n)) + binary.BigEndian.PutUint64(ha[i*8:(i+1)*8], n) } return ha[:] } @@ -229,7 +229,7 @@ func proximityOrder(one, other []byte, pos int) (int, bool) { start = pos % 8 } for j := start; j < 8; j++ { - if (uint8(oxo)>>uint8(7-j))&0x01 != 0 { + if (oxo>>uint8(7-j))&0x01 != 0 { return i*8 + j, false } } diff --git a/swarm/pss/api.go b/swarm/pss/api.go index 997800624b..6505d33023 100644 --- a/swarm/pss/api.go +++ b/swarm/pss/api.go @@ -96,7 +96,7 @@ func (pssapi *API) BaseAddr() (PssAddress, error) { func (pssapi *API) GetPublicKey() (keybytes hexutil.Bytes) { key := pssapi.Pss.PublicKey() keybytes = crypto.FromECDSAPub(key) - return hexutil.Bytes(keybytes) + return keybytes } // Set Public key to associate with a particular Pss peer diff --git a/swarm/pss/handshake.go b/swarm/pss/handshake.go index 80aa729111..17e7004798 100644 --- a/swarm/pss/handshake.go +++ b/swarm/pss/handshake.go @@ -444,7 +444,7 @@ func (self *HandshakeAPI) Handshake(pubkeyid string, topic Topic, sync bool, flu keycount = self.ctrl.symKeyCapacity } else { validkeys := self.ctrl.validKeys(pubkeyid, &topic, false) - keycount = uint8(self.ctrl.symKeyCapacity - uint8(len(validkeys))) + keycount = self.ctrl.symKeyCapacity - uint8(len(validkeys)) } if keycount == 0 { return keys, errors.New("Incoming symmetric key store is already full") From 068b0b6e6c02de26ac9ce62f4030bd3ad71cf687 Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 15 Jan 2018 00:04:01 +0100 Subject: [PATCH 026/107] swarm/pss, pot: Remove redundant conversion, go1.7 compat --- pot/address.go | 2 +- swarm/pss/pss_go18plus_test.go | 79 ++++++++++++++++++++++++++++++++++ swarm/pss/pss_test.go | 78 --------------------------------- 3 files changed, 80 insertions(+), 79 deletions(-) diff --git a/pot/address.go b/pot/address.go index 039f8421f4..3974ebcaac 100644 --- a/pot/address.go +++ b/pot/address.go @@ -105,7 +105,7 @@ func posProximity(one, other Address, pos int) (ret int, eq bool) { if one[i] == other[i] { continue } - oxo := uint8(one[i] ^ other[i]) + oxo := one[i] ^ other[i] start := 0 if i == pos/8 { start = pos % 8 diff --git a/swarm/pss/pss_go18plus_test.go b/swarm/pss/pss_go18plus_test.go index 54e5b5f386..f4fbd51566 100644 --- a/swarm/pss/pss_go18plus_test.go +++ b/swarm/pss/pss_go18plus_test.go @@ -463,3 +463,82 @@ outer: } } + +// symmetric send performance with varying message sizes +func BenchmarkSymkeySend(b *testing.B) { + b.Run(fmt.Sprintf("%d", 256), benchmarkSymKeySend) + b.Run(fmt.Sprintf("%d", 1024), benchmarkSymKeySend) + b.Run(fmt.Sprintf("%d", 1024*1024), benchmarkSymKeySend) + b.Run(fmt.Sprintf("%d", 1024*1024*10), benchmarkSymKeySend) + b.Run(fmt.Sprintf("%d", 1024*1024*100), benchmarkSymKeySend) +} + +func benchmarkSymKeySend(b *testing.B) { + msgsizestring := strings.Split(b.Name(), "/") + if len(msgsizestring) != 2 { + b.Fatalf("benchmark called without msgsize param") + } + msgsize, err := strconv.ParseInt(msgsizestring[1], 10, 0) + if err != nil { + b.Fatalf("benchmark called with invalid msgsize param '%s': %v", msgsizestring[1], err) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + keys, err := wapi.NewKeyPair(ctx) + privkey, err := w.GetPrivateKey(keys) + ps := newTestPss(privkey, nil, nil) + msg := make([]byte, msgsize) + rand.Read(msg) + topic := BytesToTopic([]byte("foo")) + to := make(PssAddress, 32) + copy(to[:], network.RandomAddr().Over()) + symkeyid, err := ps.generateSymmetricKey(topic, &to, true) + if err != nil { + b.Fatalf("could not generate symkey: %v", err) + } + symkey, err := ps.w.GetSymKey(symkeyid) + if err != nil { + b.Fatalf("could not retrieve symkey: %v", err) + } + ps.SetSymmetricKey(symkey, topic, &to, false) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + ps.SendSym(symkeyid, topic, msg) + } +} + +// asymmetric send performance with varying message sizes +func BenchmarkAsymkeySend(b *testing.B) { + b.Run(fmt.Sprintf("%d", 256), benchmarkAsymKeySend) + b.Run(fmt.Sprintf("%d", 1024), benchmarkAsymKeySend) + b.Run(fmt.Sprintf("%d", 1024*1024), benchmarkAsymKeySend) + b.Run(fmt.Sprintf("%d", 1024*1024*10), benchmarkAsymKeySend) + b.Run(fmt.Sprintf("%d", 1024*1024*100), benchmarkAsymKeySend) +} + +func benchmarkAsymKeySend(b *testing.B) { + msgsizestring := strings.Split(b.Name(), "/") + if len(msgsizestring) != 2 { + b.Fatalf("benchmark called without msgsize param") + } + msgsize, err := strconv.ParseInt(msgsizestring[1], 10, 0) + if err != nil { + b.Fatalf("benchmark called with invalid msgsize param '%s': %v", msgsizestring[1], err) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + keys, err := wapi.NewKeyPair(ctx) + privkey, err := w.GetPrivateKey(keys) + ps := newTestPss(privkey, nil, nil) + msg := make([]byte, msgsize) + rand.Read(msg) + topic := BytesToTopic([]byte("foo")) + to := make(PssAddress, 32) + copy(to[:], network.RandomAddr().Over()) + ps.SetPeerPublicKey(&privkey.PublicKey, topic, &to) + b.ResetTimer() + for i := 0; i < b.N; i++ { + ps.SendAsym(common.ToHex(crypto.FromECDSAPub(&privkey.PublicKey)), topic, msg) + } +} diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index d4c11ae74d..af05a84673 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -392,84 +392,6 @@ func TestMismatch(t *testing.T) { } -// symmetric send performance with varying message sizes -func BenchmarkSymkeySend(b *testing.B) { - b.Run(fmt.Sprintf("%d", 256), benchmarkSymKeySend) - b.Run(fmt.Sprintf("%d", 1024), benchmarkSymKeySend) - b.Run(fmt.Sprintf("%d", 1024*1024), benchmarkSymKeySend) - b.Run(fmt.Sprintf("%d", 1024*1024*10), benchmarkSymKeySend) - b.Run(fmt.Sprintf("%d", 1024*1024*100), benchmarkSymKeySend) -} - -func benchmarkSymKeySend(b *testing.B) { - msgsizestring := strings.Split(b.Name(), "/") - if len(msgsizestring) != 2 { - b.Fatalf("benchmark called without msgsize param") - } - msgsize, err := strconv.ParseInt(msgsizestring[1], 10, 0) - if err != nil { - b.Fatalf("benchmark called with invalid msgsize param '%s': %v", msgsizestring[1], err) - } - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - keys, err := wapi.NewKeyPair(ctx) - privkey, err := w.GetPrivateKey(keys) - ps := newTestPss(privkey, nil, nil) - msg := make([]byte, msgsize) - rand.Read(msg) - topic := BytesToTopic([]byte("foo")) - to := make(PssAddress, 32) - copy(to[:], network.RandomAddr().Over()) - symkeyid, err := ps.generateSymmetricKey(topic, &to, true) - if err != nil { - b.Fatalf("could not generate symkey: %v", err) - } - symkey, err := ps.w.GetSymKey(symkeyid) - if err != nil { - b.Fatalf("could not retrieve symkey: %v", err) - } - ps.SetSymmetricKey(symkey, topic, &to, false) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - ps.SendSym(symkeyid, topic, msg) - } -} - -// asymmetric send performance with varying message sizes -func BenchmarkAsymkeySend(b *testing.B) { - b.Run(fmt.Sprintf("%d", 256), benchmarkAsymKeySend) - b.Run(fmt.Sprintf("%d", 1024), benchmarkAsymKeySend) - b.Run(fmt.Sprintf("%d", 1024*1024), benchmarkAsymKeySend) - b.Run(fmt.Sprintf("%d", 1024*1024*10), benchmarkAsymKeySend) - b.Run(fmt.Sprintf("%d", 1024*1024*100), benchmarkAsymKeySend) -} - -func benchmarkAsymKeySend(b *testing.B) { - msgsizestring := strings.Split(b.Name(), "/") - if len(msgsizestring) != 2 { - b.Fatalf("benchmark called without msgsize param") - } - msgsize, err := strconv.ParseInt(msgsizestring[1], 10, 0) - if err != nil { - b.Fatalf("benchmark called with invalid msgsize param '%s': %v", msgsizestring[1], err) - } - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - keys, err := wapi.NewKeyPair(ctx) - privkey, err := w.GetPrivateKey(keys) - ps := newTestPss(privkey, nil, nil) - msg := make([]byte, msgsize) - rand.Read(msg) - topic := BytesToTopic([]byte("foo")) - to := make(PssAddress, 32) - copy(to[:], network.RandomAddr().Over()) - ps.SetPeerPublicKey(&privkey.PublicKey, topic, &to) - b.ResetTimer() - for i := 0; i < b.N; i++ { - ps.SendAsym(common.ToHex(crypto.FromECDSAPub(&privkey.PublicKey)), topic, msg) - } -} func BenchmarkSymkeyBruteforceChangeaddr(b *testing.B) { for i := 100; i < 100000; i = i * 10 { for j := 32; j < 10000; j = j * 8 { From 3791497eeb1102e303c36af801b73709f382ac5a Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 15 Jan 2018 05:14:25 +0100 Subject: [PATCH 027/107] swarm/pss: Add missing imports in pss 1.8+ test --- swarm/pss/pss_go18plus_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/swarm/pss/pss_go18plus_test.go b/swarm/pss/pss_go18plus_test.go index f4fbd51566..8b8f6669c3 100644 --- a/swarm/pss/pss_go18plus_test.go +++ b/swarm/pss/pss_go18plus_test.go @@ -15,7 +15,9 @@ import ( "testing" "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/simulations" From 833f2f9323600161ccead9c45a7bbf93724ac1fa Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 15 Jan 2018 19:01:26 +0100 Subject: [PATCH 028/107] p2p, pot, swarm/pss, swarm/storage, swarm: De-linting, versionfilter --- p2p/simulations/adapters/inproc_test.go | 18 +-- pot/pot_test.go | 10 +- swarm/pss/handshake_test.go | 2 +- swarm/pss/protocol_test.go | 2 +- swarm/pss/pss.go | 8 +- swarm/pss/pss_go18plus_test.go | 172 +++++++++++++++++++++++- swarm/pss/pss_test.go | 166 ----------------------- swarm/storage/resource.go | 5 +- swarm/swarm.go | 16 +-- 9 files changed, 191 insertions(+), 208 deletions(-) diff --git a/p2p/simulations/adapters/inproc_test.go b/p2p/simulations/adapters/inproc_test.go index 5cece74e0c..3939706aec 100644 --- a/p2p/simulations/adapters/inproc_test.go +++ b/p2p/simulations/adapters/inproc_test.go @@ -55,7 +55,7 @@ func TestSocketPipe(t *testing.T) { t.Fatal(err) } - if bytes.Compare(msg, out) != 0 { + if !bytes.Equal(msg, out) { t.Fatalf("expected %#v, got %#v", msg, out) } } @@ -96,7 +96,7 @@ func TestSocketPipeBidirections(t *testing.T) { t.Fatal(err) } - if bytes.Compare(out, []byte(`ping`)) == 0 { + if bytes.Equal(out, []byte(`ping`)) { msg := []byte(`pong`) _, err := c2.Write(msg) if err != nil { @@ -114,7 +114,7 @@ func TestSocketPipeBidirections(t *testing.T) { t.Fatal(err) } - if bytes.Compare(out, expected) != 0 { + if !bytes.Equal(out, expected) { t.Fatalf("expected %#v, got %#v", expected, out) } } @@ -160,7 +160,7 @@ func TestTcpPipe(t *testing.T) { t.Fatal(err) } - if bytes.Compare(msg, out) != 0 { + if !bytes.Equal(msg, out) { t.Fatalf("expected %#v, got %#v", msg, out) } } @@ -203,7 +203,7 @@ func TestTcpPipeBidirections(t *testing.T) { t.Fatal(err) } - if bytes.Compare(expected, out) != 0 { + if !bytes.Equal(expected, out) { t.Fatalf("expected %#v, got %#v", out, expected) } else { msg := []byte(fmt.Sprintf("pong %02d", i)) @@ -223,7 +223,7 @@ func TestTcpPipeBidirections(t *testing.T) { t.Fatal(err) } - if bytes.Compare(expected, out) != 0 { + if !bytes.Equal(expected, out) { t.Fatalf("expected %#v, got %#v", out, expected) } } @@ -271,7 +271,7 @@ func TestNetPipe(t *testing.T) { t.Fatal(err) } - if bytes.Compare(msg, out) != 0 { + if !bytes.Equal(msg, out) { t.Fatalf("expected %#v, got %#v", msg, out) } } @@ -323,7 +323,7 @@ func TestNetPipeBidirections(t *testing.T) { t.Fatal(err) } - if bytes.Compare(expected, out) != 0 { + if !bytes.Equal(expected, out) { t.Fatalf("expected %#v, got %#v", expected, out) } } @@ -341,7 +341,7 @@ func TestNetPipeBidirections(t *testing.T) { t.Fatal(err) } - if bytes.Compare(expected, out) != 0 { + if !bytes.Equal(expected, out) { t.Fatalf("expected %#v, got %#v", expected, out) } else { msg := []byte(fmt.Sprintf(pongTemplate, i)) diff --git a/pot/pot_test.go b/pot/pot_test.go index 7befdf71ba..72971ff18d 100644 --- a/pot/pot_test.go +++ b/pot/pot_test.go @@ -271,10 +271,7 @@ func testPotEachNeighbour(n *Pot, pof Pof, val Val, expCount int, fs ...func(Val } } count++ - if count == expCount { - return false - } - return true + return count == expCount }) if err == nil && count < expCount { return fmt.Errorf("not enough neighbours returned, expected %v, got %v", expCount, count) @@ -558,10 +555,7 @@ func benchmarkEachNeighbourSync(t *testing.B, max, count int, d time.Duration) { n.EachNeighbour(val, pof, func(v Val, po int) bool { time.Sleep(d) m++ - if m == count { - return false - } - return true + return m == count }) } t.StopTimer() diff --git a/swarm/pss/handshake_test.go b/swarm/pss/handshake_test.go index 6cebdbbd5a..a76741ec04 100644 --- a/swarm/pss/handshake_test.go +++ b/swarm/pss/handshake_test.go @@ -1,4 +1,4 @@ -//-build go1.7 +// +build go1.8 package pss diff --git a/swarm/pss/protocol_test.go b/swarm/pss/protocol_test.go index 793faa1397..e7016f96f1 100644 --- a/swarm/pss/protocol_test.go +++ b/swarm/pss/protocol_test.go @@ -1,4 +1,4 @@ -//-build go1.7 +// +build go1.8 package pss diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index f49e488e59..b5474ded82 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -215,9 +215,7 @@ func (self *Pss) APIs() []rpc.API { Public: true, }, } - for _, auxapi := range self.auxAPIs { - apis = append(apis, auxapi) - } + apis = append(apis, self.auxAPIs...) return apis } @@ -388,7 +386,7 @@ func (self *Pss) SetPeerPublicKey(pubkey *ecdsa.PublicKey, topic Topic, address address: address, } self.pubKeyPoolMu.Lock() - if _, ok := self.pubKeyPool[pubkeyid]; ok == false { + if _, ok := self.pubKeyPool[pubkeyid]; !ok { self.pubKeyPool[pubkeyid] = make(map[Topic]*pssPeer) } self.pubKeyPool[pubkeyid][topic] = psp @@ -537,7 +535,7 @@ func (self *Pss) cleanKeys() (count int) { match = true } } - if match == false { + if !match { expiredtopics = append(expiredtopics, topic) } } diff --git a/swarm/pss/pss_go18plus_test.go b/swarm/pss/pss_go18plus_test.go index 8b8f6669c3..d4bef49edf 100644 --- a/swarm/pss/pss_go18plus_test.go +++ b/swarm/pss/pss_go18plus_test.go @@ -1,4 +1,4 @@ -//-build go17 +// +build go1.8 package pss import ( @@ -19,11 +19,13 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/simulations" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm/network" + whisper "github.com/ethereum/go-ethereum/whisper/whisperv5" ) // send symmetrically encrypted message between two directly connected peers @@ -361,7 +363,7 @@ func testNetwork(t *testing.T) { select { case recvmsg := <-msgC: idx, _ := binary.Uvarint(recvmsg.Msg) - if recvmsgs[idx] == false { + if !recvmsgs[idx] { log.Debug("msg recv", "idx", idx, "id", id) recvmsgs[idx] = true trigger <- id @@ -544,3 +546,169 @@ func benchmarkAsymKeySend(b *testing.B) { ps.SendAsym(common.ToHex(crypto.FromECDSAPub(&privkey.PublicKey)), topic, msg) } } + +func BenchmarkSymkeyBruteforceChangeaddr(b *testing.B) { + for i := 100; i < 100000; i = i * 10 { + for j := 32; j < 10000; j = j * 8 { + b.Run(fmt.Sprintf("%d/%d", i, j), benchmarkSymkeyBruteforceChangeaddr) + } + //b.Run(fmt.Sprintf("%d", i), benchmarkSymkeyBruteforceChangeaddr) + } +} + +// decrypt performance using symkey cache, worst case +// (decrypt key always last in cache) +func benchmarkSymkeyBruteforceChangeaddr(b *testing.B) { + keycountstring := strings.Split(b.Name(), "/") + cachesize := int64(0) + var ps *Pss + if len(keycountstring) < 2 { + b.Fatalf("benchmark called without count param") + } + keycount, err := strconv.ParseInt(keycountstring[1], 10, 0) + if err != nil { + b.Fatalf("benchmark called with invalid count param '%s': %v", keycountstring[1], err) + } + if len(keycountstring) == 3 { + cachesize, err = strconv.ParseInt(keycountstring[2], 10, 0) + if err != nil { + b.Fatalf("benchmark called with invalid cachesize '%s': %v", keycountstring[2], err) + } + } + pssmsgs := make([]*PssMsg, 0, keycount) + var keyid string + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + keys, err := wapi.NewKeyPair(ctx) + privkey, err := w.GetPrivateKey(keys) + if cachesize > 0 { + ps = newTestPss(privkey, nil, &PssParams{SymKeyCacheCapacity: int(cachesize)}) + } else { + ps = newTestPss(privkey, nil, nil) + } + topic := BytesToTopic([]byte("foo")) + for i := 0; i < int(keycount); i++ { + to := make(PssAddress, 32) + copy(to[:], network.RandomAddr().Over()) + keyid, err = ps.generateSymmetricKey(topic, &to, true) + if err != nil { + b.Fatalf("cant generate symkey #%d: %v", i, err) + } + symkey, err := ps.w.GetSymKey(keyid) + if err != nil { + b.Fatalf("could not retrieve symkey %s: %v", keyid, err) + } + wparams := &whisper.MessageParams{ + TTL: defaultWhisperTTL, + KeySym: symkey, + Topic: whisper.TopicType(topic), + WorkTime: defaultWhisperWorkTime, + PoW: defaultWhisperPoW, + Payload: []byte("xyzzy"), + Padding: []byte("1234567890abcdef"), + } + woutmsg, err := whisper.NewSentMessage(wparams) + if err != nil { + b.Fatalf("could not create whisper message: %v", err) + } + env, err := woutmsg.Wrap(wparams) + if err != nil { + b.Fatalf("could not generate whisper envelope: %v", err) + } + ps.Register(&topic, func(msg []byte, p *p2p.Peer, asymmetric bool, keyid string) error { + return nil + }) + pssmsgs = append(pssmsgs, &PssMsg{ + To: to, + Payload: env, + }) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + if !ps.process(pssmsgs[len(pssmsgs)-(i%len(pssmsgs))-1]) { + b.Fatalf("pss processing failed: %v", err) + } + } +} + +func BenchmarkSymkeyBruteforceSameaddr(b *testing.B) { + for i := 100; i < 100000; i = i * 10 { + for j := 32; j < 10000; j = j * 8 { + b.Run(fmt.Sprintf("%d/%d", i, j), benchmarkSymkeyBruteforceSameaddr) + } + } +} + +// decrypt performance using symkey cache, best case +// (decrypt key always first in cache) +func benchmarkSymkeyBruteforceSameaddr(b *testing.B) { + var keyid string + var ps *Pss + cachesize := int64(0) + keycountstring := strings.Split(b.Name(), "/") + if len(keycountstring) < 2 { + b.Fatalf("benchmark called without count param") + } + keycount, err := strconv.ParseInt(keycountstring[1], 10, 0) + if err != nil { + b.Fatalf("benchmark called with invalid count param '%s': %v", keycountstring[1], err) + } + if len(keycountstring) == 3 { + cachesize, err = strconv.ParseInt(keycountstring[2], 10, 0) + if err != nil { + b.Fatalf("benchmark called with invalid cachesize '%s': %v", keycountstring[2], err) + } + } + addr := make([]PssAddress, keycount) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + keys, err := wapi.NewKeyPair(ctx) + privkey, err := w.GetPrivateKey(keys) + if cachesize > 0 { + ps = newTestPss(privkey, nil, &PssParams{SymKeyCacheCapacity: int(cachesize)}) + } else { + ps = newTestPss(privkey, nil, nil) + } + topic := BytesToTopic([]byte("foo")) + for i := 0; i < int(keycount); i++ { + copy(addr[i], network.RandomAddr().Over()) + keyid, err = ps.generateSymmetricKey(topic, &addr[i], true) + if err != nil { + b.Fatalf("cant generate symkey #%d: %v", i, err) + } + + } + symkey, err := ps.w.GetSymKey(keyid) + if err != nil { + b.Fatalf("could not retrieve symkey %s: %v", keyid, err) + } + wparams := &whisper.MessageParams{ + TTL: defaultWhisperTTL, + KeySym: symkey, + Topic: whisper.TopicType(topic), + WorkTime: defaultWhisperWorkTime, + PoW: defaultWhisperPoW, + Payload: []byte("xyzzy"), + Padding: []byte("1234567890abcdef"), + } + woutmsg, err := whisper.NewSentMessage(wparams) + if err != nil { + b.Fatalf("could not create whisper message: %v", err) + } + env, err := woutmsg.Wrap(wparams) + if err != nil { + b.Fatalf("could not generate whisper envelope: %v", err) + } + ps.Register(&topic, func(msg []byte, p *p2p.Peer, asymmetric bool, keyid string) error { + return nil + }) + pssmsg := &PssMsg{ + To: addr[len(addr)-1][:], + Payload: env, + } + for i := 0; i < b.N; i++ { + if !ps.process(pssmsg) { + b.Fatalf("pss processing failed: %v", err) + } + } +} diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index af05a84673..b9275f84c0 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -392,172 +392,6 @@ func TestMismatch(t *testing.T) { } -func BenchmarkSymkeyBruteforceChangeaddr(b *testing.B) { - for i := 100; i < 100000; i = i * 10 { - for j := 32; j < 10000; j = j * 8 { - b.Run(fmt.Sprintf("%d/%d", i, j), benchmarkSymkeyBruteforceChangeaddr) - } - //b.Run(fmt.Sprintf("%d", i), benchmarkSymkeyBruteforceChangeaddr) - } -} - -// decrypt performance using symkey cache, worst case -// (decrypt key always last in cache) -func benchmarkSymkeyBruteforceChangeaddr(b *testing.B) { - keycountstring := strings.Split(b.Name(), "/") - cachesize := int64(0) - var ps *Pss - if len(keycountstring) < 2 { - b.Fatalf("benchmark called without count param") - } - keycount, err := strconv.ParseInt(keycountstring[1], 10, 0) - if err != nil { - b.Fatalf("benchmark called with invalid count param '%s': %v", keycountstring[1], err) - } - if len(keycountstring) == 3 { - cachesize, err = strconv.ParseInt(keycountstring[2], 10, 0) - if err != nil { - b.Fatalf("benchmark called with invalid cachesize '%s': %v", keycountstring[2], err) - } - } - pssmsgs := make([]*PssMsg, 0, keycount) - var keyid string - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - keys, err := wapi.NewKeyPair(ctx) - privkey, err := w.GetPrivateKey(keys) - if cachesize > 0 { - ps = newTestPss(privkey, nil, &PssParams{SymKeyCacheCapacity: int(cachesize)}) - } else { - ps = newTestPss(privkey, nil, nil) - } - topic := BytesToTopic([]byte("foo")) - for i := 0; i < int(keycount); i++ { - to := make(PssAddress, 32) - copy(to[:], network.RandomAddr().Over()) - keyid, err = ps.generateSymmetricKey(topic, &to, true) - if err != nil { - b.Fatalf("cant generate symkey #%d: %v", i, err) - } - symkey, err := ps.w.GetSymKey(keyid) - if err != nil { - b.Fatalf("could not retrieve symkey %s: %v", keyid, err) - } - wparams := &whisper.MessageParams{ - TTL: defaultWhisperTTL, - KeySym: symkey, - Topic: whisper.TopicType(topic), - WorkTime: defaultWhisperWorkTime, - PoW: defaultWhisperPoW, - Payload: []byte("xyzzy"), - Padding: []byte("1234567890abcdef"), - } - woutmsg, err := whisper.NewSentMessage(wparams) - if err != nil { - b.Fatalf("could not create whisper message: %v", err) - } - env, err := woutmsg.Wrap(wparams) - if err != nil { - b.Fatalf("could not generate whisper envelope: %v", err) - } - ps.Register(&topic, func(msg []byte, p *p2p.Peer, asymmetric bool, keyid string) error { - return nil - }) - pssmsgs = append(pssmsgs, &PssMsg{ - To: to, - Payload: env, - }) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - if !ps.process(pssmsgs[len(pssmsgs)-(i%len(pssmsgs))-1]) { - b.Fatalf("pss processing failed: %v", err) - } - } -} - -func BenchmarkSymkeyBruteforceSameaddr(b *testing.B) { - for i := 100; i < 100000; i = i * 10 { - for j := 32; j < 10000; j = j * 8 { - b.Run(fmt.Sprintf("%d/%d", i, j), benchmarkSymkeyBruteforceSameaddr) - } - } -} - -// decrypt performance using symkey cache, best case -// (decrypt key always first in cache) -func benchmarkSymkeyBruteforceSameaddr(b *testing.B) { - var keyid string - var ps *Pss - cachesize := int64(0) - keycountstring := strings.Split(b.Name(), "/") - if len(keycountstring) < 2 { - b.Fatalf("benchmark called without count param") - } - keycount, err := strconv.ParseInt(keycountstring[1], 10, 0) - if err != nil { - b.Fatalf("benchmark called with invalid count param '%s': %v", keycountstring[1], err) - } - if len(keycountstring) == 3 { - cachesize, err = strconv.ParseInt(keycountstring[2], 10, 0) - if err != nil { - b.Fatalf("benchmark called with invalid cachesize '%s': %v", keycountstring[2], err) - } - } - addr := make([]PssAddress, keycount) - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - keys, err := wapi.NewKeyPair(ctx) - privkey, err := w.GetPrivateKey(keys) - if cachesize > 0 { - ps = newTestPss(privkey, nil, &PssParams{SymKeyCacheCapacity: int(cachesize)}) - } else { - ps = newTestPss(privkey, nil, nil) - } - topic := BytesToTopic([]byte("foo")) - for i := 0; i < int(keycount); i++ { - copy(addr[i], network.RandomAddr().Over()) - keyid, err = ps.generateSymmetricKey(topic, &addr[i], true) - if err != nil { - b.Fatalf("cant generate symkey #%d: %v", i, err) - } - - } - symkey, err := ps.w.GetSymKey(keyid) - if err != nil { - b.Fatalf("could not retrieve symkey %s: %v", keyid, err) - } - wparams := &whisper.MessageParams{ - TTL: defaultWhisperTTL, - KeySym: symkey, - Topic: whisper.TopicType(topic), - WorkTime: defaultWhisperWorkTime, - PoW: defaultWhisperPoW, - Payload: []byte("xyzzy"), - Padding: []byte("1234567890abcdef"), - } - woutmsg, err := whisper.NewSentMessage(wparams) - if err != nil { - b.Fatalf("could not create whisper message: %v", err) - } - env, err := woutmsg.Wrap(wparams) - if err != nil { - b.Fatalf("could not generate whisper envelope: %v", err) - } - ps.Register(&topic, func(msg []byte, p *p2p.Peer, asymmetric bool, keyid string) error { - return nil - }) - pssmsg := &PssMsg{ - To: addr[len(addr)-1][:], - Payload: env, - } - for i := 0; i < b.N; i++ { - if !ps.process(pssmsg) { - b.Fatalf("pss processing failed: %v", err) - } - } -} - // setup simulated network and connect nodes in circle func setupNetwork(numnodes int) (clients []*rpc.Client, err error) { nodes := make([]*simulations.Node, numnodes) diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 7745d75fbc..34444f92c2 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -637,10 +637,7 @@ func (self *ResourceHandler) verifyContent(chunkdata []byte) error { } func (self *ResourceHandler) hasUpdate(name string, period uint32) bool { - if self.resources[name].lastPeriod == period { - return true - } - return false + return self.resources[name].lastPeriod == period } type resourceChunkStore struct { diff --git a/swarm/swarm.go b/swarm/swarm.go index 48e736ce72..c47857093b 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -254,14 +254,10 @@ func (self *Swarm) Stop() error { // implements the node.Service interface func (self *Swarm) Protocols() (protos []p2p.Protocol) { - for _, p := range self.bzz.Protocols() { - protos = append(protos, p) - } + protos = append(protos, self.bzz.Protocols()...) if self.ps != nil { - for _, p := range self.ps.Protocols() { - protos = append(protos, p) - } + protos = append(protos, self.ps.Protocols()) } return } @@ -322,14 +318,10 @@ func (self *Swarm) APIs() []rpc.API { // {Namespace, Version, api.NewAdmin(self), false}, } - for _, api := range self.bzz.APIs() { - apis = append(apis, api) - } + apis = append(apis, self.bzz.APIs()) if self.ps != nil { - for _, api := range self.ps.APIs() { - apis = append(apis, api) - } + apis = append(apis, self.ps.APIs()) } return apis From 54336890e432c1d5bb33f9cfdbba9ca0d16a9ef3 Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 15 Jan 2018 20:54:11 +0100 Subject: [PATCH 029/107] swarm/pss: Fix wrong build tag --- swarm/pss/pss_go18plus_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/swarm/pss/pss_go18plus_test.go b/swarm/pss/pss_go18plus_test.go index d4bef49edf..30d94ee714 100644 --- a/swarm/pss/pss_go18plus_test.go +++ b/swarm/pss/pss_go18plus_test.go @@ -1,4 +1,5 @@ // +build go1.8 + package pss import ( From 01931c4e8dcec0865d34fc7aa145ca98b2084089 Mon Sep 17 00:00:00 2001 From: lash Date: Tue, 16 Jan 2018 03:00:00 +0100 Subject: [PATCH 030/107] swarm: Fix typo in API and Protocol append --- swarm/swarm.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/swarm/swarm.go b/swarm/swarm.go index c47857093b..14826bb021 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -257,7 +257,7 @@ func (self *Swarm) Protocols() (protos []p2p.Protocol) { protos = append(protos, self.bzz.Protocols()...) if self.ps != nil { - protos = append(protos, self.ps.Protocols()) + protos = append(protos, self.ps.Protocols()...) } return } @@ -318,10 +318,10 @@ func (self *Swarm) APIs() []rpc.API { // {Namespace, Version, api.NewAdmin(self), false}, } - apis = append(apis, self.bzz.APIs()) + apis = append(apis, self.bzz.APIs()...) if self.ps != nil { - apis = append(apis, self.ps.APIs()) + apis = append(apis, self.ps.APIs()...) } return apis From 079be17d1fc87ddac8f7f5046128a60f3f375702 Mon Sep 17 00:00:00 2001 From: lash Date: Tue, 16 Jan 2018 14:28:02 +0100 Subject: [PATCH 031/107] pot: Correct to reverse test results --- pot/pot_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pot/pot_test.go b/pot/pot_test.go index 72971ff18d..1175abd80c 100644 --- a/pot/pot_test.go +++ b/pot/pot_test.go @@ -271,7 +271,7 @@ func testPotEachNeighbour(n *Pot, pof Pof, val Val, expCount int, fs ...func(Val } } count++ - return count == expCount + return count != expCount }) if err == nil && count < expCount { return fmt.Errorf("not enough neighbours returned, expected %v, got %v", expCount, count) @@ -555,7 +555,7 @@ func benchmarkEachNeighbourSync(t *testing.B, max, count int, d time.Duration) { n.EachNeighbour(val, pof, func(v Val, po int) bool { time.Sleep(d) m++ - return m == count + return m != count }) } t.StopTimer() From d9652fdf7b0220e7717a3f3bf09fc97ae0584412 Mon Sep 17 00:00:00 2001 From: lash Date: Tue, 16 Jan 2018 19:35:23 +0100 Subject: [PATCH 032/107] swarm/pss: Fixed missing protoCtrl on go build tag < 1.8 --- swarm/pss/handshake_test.go | 2 +- swarm/pss/protocol_go18plus_test.go | 130 ++++++++++++++++++++++++++++ swarm/pss/protocol_test.go | 125 -------------------------- swarm/pss/pss_go18plus_test.go | 2 +- 4 files changed, 132 insertions(+), 127 deletions(-) create mode 100644 swarm/pss/protocol_go18plus_test.go diff --git a/swarm/pss/handshake_test.go b/swarm/pss/handshake_test.go index a76741ec04..ac70364aa3 100644 --- a/swarm/pss/handshake_test.go +++ b/swarm/pss/handshake_test.go @@ -1,4 +1,4 @@ -// +build go1.8 +// +build foo package pss diff --git a/swarm/pss/protocol_go18plus_test.go b/swarm/pss/protocol_go18plus_test.go new file mode 100644 index 0000000000..f835e39849 --- /dev/null +++ b/swarm/pss/protocol_go18plus_test.go @@ -0,0 +1,130 @@ +// +build go1.8 + +package pss + +import ( + "context" + "fmt" + "strconv" + "strings" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/discover" +) + +// simple ping pong protocol test for the pss devp2p emulation +func TestProtocol(t *testing.T) { + t.Run("32", testProtocol) + t.Run("8", testProtocol) + t.Run("0", testProtocol) +} + +func testProtocol(t *testing.T) { + + // address hint size + var addrsize int64 + paramstring := strings.Split(t.Name(), "/") + addrsize, _ = strconv.ParseInt(paramstring[1], 10, 0) + log.Info("protocol test", "addrsize", addrsize) + + topic := PingTopic.String() + + clients, err := setupNetwork(2) + if err != nil { + t.Fatal(err) + } + var loaddrhex string + err = clients[0].Call(&loaddrhex, "pss_baseAddr") + if err != nil { + t.Fatalf("rpc get node 1 baseaddr fail: %v", err) + } + loaddrhex = loaddrhex[:2+(addrsize*2)] + var roaddrhex string + err = clients[1].Call(&roaddrhex, "pss_baseAddr") + if err != nil { + t.Fatalf("rpc get node 2 baseaddr fail: %v", err) + } + roaddrhex = roaddrhex[:2+(addrsize*2)] + lnodeinfo := &p2p.NodeInfo{} + err = clients[0].Call(&lnodeinfo, "admin_nodeInfo") + if err != nil { + t.Fatalf("rpc nodeinfo node 11 fail: %v", err) + } + + var lpubkey string + err = clients[0].Call(&lpubkey, "pss_getPublicKey") + if err != nil { + t.Fatalf("rpc get node 1 pubkey fail: %v", err) + } + var rpubkey string + err = clients[1].Call(&rpubkey, "pss_getPublicKey") + if err != nil { + t.Fatalf("rpc get node 2 pubkey fail: %v", err) + } + + time.Sleep(time.Millisecond * 1000) // replace with hive healthy code + + lmsgC := make(chan APIMsg) + lctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic) + defer lsub.Unsubscribe() + rmsgC := make(chan APIMsg) + rctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic) + defer rsub.Unsubscribe() + + // set reciprocal public keys + err = clients[0].Call(nil, "pss_setPeerPublicKey", rpubkey, topic, roaddrhex) + if err != nil { + t.Fatal(err) + } + err = clients[1].Call(nil, "pss_setPeerPublicKey", lpubkey, topic, loaddrhex) + if err != nil { + t.Fatal(err) + } + + // add right peer's public key as protocol peer on left + nid, _ := discover.HexID("0x00") // this hack is needed to satisfy the p2p method + p := p2p.NewPeer(nid, fmt.Sprintf("%x", common.FromHex(loaddrhex)), []p2p.Cap{}) + _, err = pssprotocols[lnodeinfo.ID].protocol.AddPeer(p, pssprotocols[lnodeinfo.ID].run, PingTopic, true, rpubkey) + if err != nil { + t.Fatal(err) + } + + // sends ping asym, checks delivery + pssprotocols[lnodeinfo.ID].C <- false + select { + case <-lmsgC: + log.Debug("lnode ok") + case cerr := <-lctx.Done(): + t.Fatalf("test message timed out: %v", cerr) + } + select { + case <-rmsgC: + log.Debug("rnode ok") + case cerr := <-lctx.Done(): + t.Fatalf("test message timed out: %v", cerr) + } + + // sends ping asym, checks delivery + pssprotocols[lnodeinfo.ID].C <- false + select { + case <-lmsgC: + log.Debug("lnode ok") + case cerr := <-lctx.Done(): + t.Fatalf("test message timed out: %v", cerr) + } + select { + case <-rmsgC: + log.Debug("rnode ok") + case cerr := <-lctx.Done(): + t.Fatalf("test message timed out: %v", cerr) + } + +} diff --git a/swarm/pss/protocol_test.go b/swarm/pss/protocol_test.go index e7016f96f1..043618a802 100644 --- a/swarm/pss/protocol_test.go +++ b/swarm/pss/protocol_test.go @@ -1,19 +1,7 @@ -// +build go1.8 - package pss import ( - "context" - "fmt" - "strconv" - "strings" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/discover" ) type protoCtrl struct { @@ -21,116 +9,3 @@ type protoCtrl struct { protocol *Protocol run func(*p2p.Peer, p2p.MsgReadWriter) error } - -// simple ping pong protocol test for the pss devp2p emulation -func TestProtocol(t *testing.T) { - t.Run("32", testProtocol) - t.Run("8", testProtocol) - t.Run("0", testProtocol) -} - -func testProtocol(t *testing.T) { - - // address hint size - var addrsize int64 - paramstring := strings.Split(t.Name(), "/") - addrsize, _ = strconv.ParseInt(paramstring[1], 10, 0) - log.Info("protocol test", "addrsize", addrsize) - - topic := PingTopic.String() - - clients, err := setupNetwork(2) - if err != nil { - t.Fatal(err) - } - var loaddrhex string - err = clients[0].Call(&loaddrhex, "pss_baseAddr") - if err != nil { - t.Fatalf("rpc get node 1 baseaddr fail: %v", err) - } - loaddrhex = loaddrhex[:2+(addrsize*2)] - var roaddrhex string - err = clients[1].Call(&roaddrhex, "pss_baseAddr") - if err != nil { - t.Fatalf("rpc get node 2 baseaddr fail: %v", err) - } - roaddrhex = roaddrhex[:2+(addrsize*2)] - lnodeinfo := &p2p.NodeInfo{} - err = clients[0].Call(&lnodeinfo, "admin_nodeInfo") - if err != nil { - t.Fatalf("rpc nodeinfo node 11 fail: %v", err) - } - - var lpubkey string - err = clients[0].Call(&lpubkey, "pss_getPublicKey") - if err != nil { - t.Fatalf("rpc get node 1 pubkey fail: %v", err) - } - var rpubkey string - err = clients[1].Call(&rpubkey, "pss_getPublicKey") - if err != nil { - t.Fatalf("rpc get node 2 pubkey fail: %v", err) - } - - time.Sleep(time.Millisecond * 1000) // replace with hive healthy code - - lmsgC := make(chan APIMsg) - lctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic) - defer lsub.Unsubscribe() - rmsgC := make(chan APIMsg) - rctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic) - defer rsub.Unsubscribe() - - // set reciprocal public keys - err = clients[0].Call(nil, "pss_setPeerPublicKey", rpubkey, topic, roaddrhex) - if err != nil { - t.Fatal(err) - } - err = clients[1].Call(nil, "pss_setPeerPublicKey", lpubkey, topic, loaddrhex) - if err != nil { - t.Fatal(err) - } - - // add right peer's public key as protocol peer on left - nid, _ := discover.HexID("0x00") // this hack is needed to satisfy the p2p method - p := p2p.NewPeer(nid, fmt.Sprintf("%x", common.FromHex(loaddrhex)), []p2p.Cap{}) - _, err = pssprotocols[lnodeinfo.ID].protocol.AddPeer(p, pssprotocols[lnodeinfo.ID].run, PingTopic, true, rpubkey) - if err != nil { - t.Fatal(err) - } - - // sends ping asym, checks delivery - pssprotocols[lnodeinfo.ID].C <- false - select { - case <-lmsgC: - log.Debug("lnode ok") - case cerr := <-lctx.Done(): - t.Fatalf("test message timed out: %v", cerr) - } - select { - case <-rmsgC: - log.Debug("rnode ok") - case cerr := <-lctx.Done(): - t.Fatalf("test message timed out: %v", cerr) - } - - // sends ping asym, checks delivery - pssprotocols[lnodeinfo.ID].C <- false - select { - case <-lmsgC: - log.Debug("lnode ok") - case cerr := <-lctx.Done(): - t.Fatalf("test message timed out: %v", cerr) - } - select { - case <-rmsgC: - log.Debug("rnode ok") - case cerr := <-lctx.Done(): - t.Fatalf("test message timed out: %v", cerr) - } - -} diff --git a/swarm/pss/pss_go18plus_test.go b/swarm/pss/pss_go18plus_test.go index 30d94ee714..ed892deeeb 100644 --- a/swarm/pss/pss_go18plus_test.go +++ b/swarm/pss/pss_go18plus_test.go @@ -1,4 +1,4 @@ -// +build go1.8 +// +build foo package pss From 3a4875fc871eebca04e99017d475599a9929e019 Mon Sep 17 00:00:00 2001 From: lash Date: Tue, 16 Jan 2018 19:44:45 +0100 Subject: [PATCH 033/107] swarm/network: Fix kademlia param overflow on 32bit --- swarm/network/kademlia.go | 20 ++++++++++---------- swarm/network/kademlia_test.go | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/swarm/network/kademlia.go b/swarm/network/kademlia.go index d7bb7be6d7..ed1e01410d 100644 --- a/swarm/network/kademlia.go +++ b/swarm/network/kademlia.go @@ -54,14 +54,14 @@ var pof = pot.DefaultPof(256) // KadParams holds the config params for Kademlia type KadParams struct { // adjustable parameters - MaxProxDisplay int // number of rows the table shows - MinProxBinSize int // nearest neighbour core minimum cardinality - MinBinSize int // minimum number of peers in a row - MaxBinSize int // maximum number of peers in a row before pruning - RetryInterval int // initial interval before a peer is first redialed - RetryExponent int // exponent to multiply retry intervals with - MaxRetries int // maximum number of redial attempts - PruneInterval int // interval between peer pruning cycles + MaxProxDisplay int // number of rows the table shows + MinProxBinSize int // nearest neighbour core minimum cardinality + MinBinSize int // minimum number of peers in a row + MaxBinSize int // maximum number of peers in a row before pruning + RetryInterval uint // initial interval before a peer is first redialed + RetryExponent uint // exponent to multiply retry intervals with + MaxRetries int // maximum number of redial attempts + PruneInterval int // interval between peer pruning cycles // function to sanction or prevent suggesting a peer Reachable func(OverlayAddr) bool } @@ -400,10 +400,10 @@ func (k *Kademlia) callable(val pot.Val) OverlayAddr { } // calculate the allowed number of retries based on time lapsed since last seen timeAgo := int(time.Since(e.seenAt)) - div := k.RetryExponent + div := int(k.RetryExponent) div += (150000 - rand.Intn(300000)) * div / 1000000 var retries int - for delta := timeAgo; delta > k.RetryInterval; delta /= div { + for delta := timeAgo; uint(delta) > k.RetryInterval; delta /= div { retries++ } diff --git a/swarm/network/kademlia_test.go b/swarm/network/kademlia_test.go index 7e3c752dc2..16bbb62020 100644 --- a/swarm/network/kademlia_test.go +++ b/swarm/network/kademlia_test.go @@ -284,7 +284,7 @@ func TestSuggestPeerRetries(t *testing.T) { // 2 row gap, unsaturated proxbin, no callables -> want PO 0 k := newTestKademlia("00000000") cycle := time.Second - k.RetryInterval = int(cycle) + k.RetryInterval = uint(cycle) k.MaxRetries = 50 k.RetryExponent = 2 sleep := func(n int) { From 358783fe35e364988ff9115dca636fc3fbda0f92 Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 18 Jan 2018 03:37:23 +0100 Subject: [PATCH 034/107] swarm/storage: noop change to trigger travis (after outage) --- swarm/storage/localstore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index 7abfc9b086..bb55cc99fa 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -76,7 +76,7 @@ func (self *LocalStore) Get(key Key) (chunk *Chunk, err error) { return } -// Close local store +// Close the local store func (self *LocalStore) Close() { self.DbStore.Close() } From 2001752d78fea28718fa9c49e2a83c690e9f9a4b Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 18 Jan 2018 17:24:00 +0100 Subject: [PATCH 035/107] swarm/testutil: Updated NewLocalStore invocation --- swarm/testutil/http.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go index f2922fab00..b1dd4d4e65 100644 --- a/swarm/testutil/http.go +++ b/swarm/testutil/http.go @@ -38,7 +38,7 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { CacheCapacity: 5000, Radius: 0, } - localStore, err := storage.NewLocalStore(storage.MakeHashFunc("SHA3"), storeparams) + localStore, err := storage.NewLocalStore(storage.MakeHashFunc("SHA3"), storeparams, nil) if err != nil { os.RemoveAll(dir) t.Fatal(err) From 7a1099dfa8aaa79d0e511f0ec778fec29a4b510d Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 18 Jan 2018 22:28:07 +0100 Subject: [PATCH 036/107] swarm/storage/mock: Omit test on <1.8 --- swarm/storage/mock/test/test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/swarm/storage/mock/test/test.go b/swarm/storage/mock/test/test.go index 402634aa54..19af0a27bb 100644 --- a/swarm/storage/mock/test/test.go +++ b/swarm/storage/mock/test/test.go @@ -1,3 +1,5 @@ +// +build 1.8 +// // Copyright 2018 The go-ethereum Authors // This file is part of the go-ethereum library. // From 73ed8efc82e61e5697a66b0928b1d4fdc630aedb Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 19 Jan 2018 00:36:19 +0100 Subject: [PATCH 037/107] swarm/storage/mock: Correct build tag in wrong dir --- swarm/storage/mock/db/db_test.go | 2 ++ swarm/storage/mock/test/test.go | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/swarm/storage/mock/db/db_test.go b/swarm/storage/mock/db/db_test.go index 2855e2f298..782faaf35c 100644 --- a/swarm/storage/mock/db/db_test.go +++ b/swarm/storage/mock/db/db_test.go @@ -1,3 +1,5 @@ +// +build go1.8 +// // Copyright 2018 The go-ethereum Authors // This file is part of the go-ethereum library. // diff --git a/swarm/storage/mock/test/test.go b/swarm/storage/mock/test/test.go index 19af0a27bb..402634aa54 100644 --- a/swarm/storage/mock/test/test.go +++ b/swarm/storage/mock/test/test.go @@ -1,5 +1,3 @@ -// +build 1.8 -// // Copyright 2018 The go-ethereum Authors // This file is part of the go-ethereum library. // From 3ef02ccdbdd67be198e0398e82e67c7489378d40 Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 19 Jan 2018 00:38:09 +0100 Subject: [PATCH 038/107] cmd/swarm: Delint conversion --- cmd/swarm/config.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index b23d452869..c2209a3204 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -225,15 +225,15 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con } if storeCapacity := ctx.GlobalUint64(SwarmStoreCapacity.Name); storeCapacity != 0 { - currentConfig.StoreParams.DbCapacity = uint64(storeCapacity) + currentConfig.StoreParams.DbCapacity = storeCapacity } if storeCacheCapacity := ctx.GlobalUint(SwarmStoreCacheCapacity.Name); storeCacheCapacity != 0 { - currentConfig.StoreParams.CacheCapacity = uint(storeCacheCapacity) + currentConfig.StoreParams.CacheCapacity = storeCacheCapacity } if storeRadius := ctx.GlobalInt(SwarmStoreRadius.Name); storeRadius != 0 { - currentConfig.StoreParams.Radius = int(storeRadius) + currentConfig.StoreParams.Radius = storeRadius } return currentConfig From 56c59d57b12cd4ca25f3cfaa6229f47014e6ed12 Mon Sep 17 00:00:00 2001 From: lash Date: Wed, 17 Jan 2018 06:04:49 +0100 Subject: [PATCH 039/107] swarm/storage: External signing, chunk data verification --- swarm/storage/resource.go | 121 +++++++++++++++++++-------------- swarm/storage/resource_ens.go | 12 ++-- swarm/storage/resource_test.go | 82 +++++++++++++++++----- 3 files changed, 138 insertions(+), 77 deletions(-) diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 34444f92c2..c4b4e51597 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -93,7 +93,7 @@ type resource struct { // treated as a resource update chunk. type ResourceValidator interface { - isOwner(string) (bool, error) + isOwner(string, common.Address) (bool, error) nameHash(string) common.Hash } @@ -105,7 +105,6 @@ type ResourceHandler struct { hashLock sync.Mutex resourceLock sync.RWMutex hasher SwarmHash - privKey *ecdsa.PrivateKey maxChunkData int64 } @@ -127,7 +126,6 @@ func NewResourceHandler(privKey *ecdsa.PrivateKey, datadir string, cloudStore Cl rpcClient: rpcClient, resources: make(map[string]*resource), hasher: hasher(), - privKey: privKey, maxChunkData: DefaultBranches * int64(hasher().Size()), } @@ -187,9 +185,13 @@ func NewResource(name string, startBlock uint64, frequency uint64, nameHashFunc // Creates a new root entry for a mutable resource identified by `name` with the specified `frequency`. // // The start block of the resource update will be the actual current block height of the connected network. -func (self *ResourceHandler) NewResource(name string, frequency uint64) (*resource, error) { +func (self *ResourceHandler) NewResource(name string, frequency uint64, signature [signatureLength]byte) (*resource, error) { - ok, err := self.validator.isOwner(name) + addr, err := self.getAddressFromDataSig([]byte(name), signature) + if err != nil { + return nil, fmt.Errorf("Corrupt signature") + } + ok, err := self.validator.isOwner(name, addr) if err != nil { return nil, err } else if !ok { @@ -463,9 +465,13 @@ func parseUpdate(blob []byte) (period uint32, version uint32, ensname []byte, da // It is the caller's responsibility to make sure that this data is not stale. // // A resource update cannot span chunks, and thus has max length 4096 -func (self *ResourceHandler) Update(name string, data []byte) (Key, error) { +func (self *ResourceHandler) Update(name string, data []byte, signature [signatureLength]byte) (Key, error) { - ok, err := self.validator.isOwner(name) + addr, err := self.getAddressFromDataSig(data, signature) + if err != nil { + return nil, fmt.Errorf("Invalid data/signature: %v", err) + } + ok, err := self.validator.isOwner(name, addr) if err != nil { return nil, err } else if !ok { @@ -501,34 +507,36 @@ func (self *ResourceHandler) Update(name string, data []byte) (Key, error) { } version++ + // create the update chunk // prepend version and period to allow reverse lookups // data header length does NOT include the header length prefix bytes themselves headerlength := uint16(len(resource.nameHash) + 4 + 4) - fulldata := make([]byte, int(headerlength)+2+len(data)) - cursor := 0 - binary.LittleEndian.PutUint16(fulldata, headerlength) - cursor += 2 - - binary.LittleEndian.PutUint32(fulldata[cursor:], nextperiod) - cursor += 4 - - binary.LittleEndian.PutUint32(fulldata[cursor:], version) - cursor += 4 - - copy(fulldata[cursor:], resource.nameHash[:]) - cursor += len(resource.nameHash) - - copy(fulldata[cursor:], data) - - // create the update chunk and send it key := self.resourceHash(resource.nameHash, nextperiod, version) chunk := NewChunk(key, nil) - chunk.SData, err = self.signContent(fulldata) - if err != nil { - return nil, err - } - chunk.Size = int64(len(fulldata)) + chunk.SData = make([]byte, signatureLength+int(headerlength)+2+len(data)) + + cursor := 0 + copy(chunk.SData, signature[:]) + cursor += signatureLength + + binary.LittleEndian.PutUint16(chunk.SData[cursor:], headerlength) + cursor += 2 + + binary.LittleEndian.PutUint32(chunk.SData[cursor:], nextperiod) + cursor += 4 + + binary.LittleEndian.PutUint32(chunk.SData[cursor:], version) + cursor += 4 + + copy(chunk.SData[cursor:], resource.nameHash[:]) + cursor += len(resource.nameHash) + + copy(chunk.SData[cursor:], data) + + chunk.Size = int64(len(chunk.SData)) + + // send the chunk self.Put(chunk) log.Trace("resource update", "name", resource.name, "key", key, "currentblock", currentblock, "lastperiod", nextperiod, "version", version, "data", chunk.SData) @@ -594,33 +602,33 @@ func (self *ResourceHandler) resourceHash(namehash common.Hash, period uint32, v return self.hasher.Sum(nil) } -func (self *ResourceHandler) signContent(data []byte) ([]byte, error) { +func (self *ResourceHandler) getContentAccount(chunkdata []byte) (common.Address, error) { + if len(chunkdata) <= signatureLength { + return common.Address{}, fmt.Errorf("zero-length data") + } + var signaturetype [signatureLength]byte + copy(signaturetype[:], chunkdata[:signatureLength]) + return self.getAddressFromDataSig(chunkdata[signatureLength:], signaturetype) +} + +func (self *ResourceHandler) getContentName(chunkdata []byte) (string, error) { + nameoffset := signatureLength + 2 + 4 + 4 + if len(chunkdata) < nameoffset { + return "", fmt.Errorf("invalid chunk data") + } + namelength := binary.LittleEndian.Uint16(chunkdata[signatureLength : signatureLength+2]) + namebytes := make([]byte, namelength) + copy(namebytes, chunkdata[nameoffset:nameoffset+int(namelength)-2-4-4]) + return string(namebytes), nil +} + +func (self *ResourceHandler) getAddressFromDataSig(data []byte, signature [signatureLength]byte) (common.Address, error) { self.hashLock.Lock() self.hasher.Reset() self.hasher.Write(data) datahash := self.hasher.Sum(nil) self.hashLock.Unlock() - - signature, err := crypto.Sign(datahash, self.privKey) - if err != nil { - return nil, err - } - datawithsign := make([]byte, len(data)+signatureLength) - copy(datawithsign[:signatureLength], signature) - copy(datawithsign[signatureLength:], data) - return datawithsign, nil -} - -func (self *ResourceHandler) getContentAccount(chunkdata []byte) (common.Address, error) { - if len(chunkdata) <= signatureLength { - return common.Address{}, fmt.Errorf("zero-length data") - } - self.hashLock.Lock() - self.hasher.Reset() - self.hasher.Write(chunkdata[signatureLength:]) - datahash := self.hasher.Sum(nil) - self.hashLock.Unlock() - pub, err := crypto.SigToPub(datahash, chunkdata[:signatureLength]) + pub, err := crypto.SigToPub(datahash, signature[:]) if err != nil { return common.Address{}, err } @@ -632,7 +640,16 @@ func (self *ResourceHandler) verifyContent(chunkdata []byte) error { if err != nil { return err } - log.Warn("ens owner lookup not implemented, verify will return true in all cases", "address", address) + name, err := self.getContentName(chunkdata) + if err != nil { + return err + } + ok, err := self.validator.isOwner(name, address) + if err != nil { + return err + } else if !ok { + return fmt.Errorf("not owner") + } return nil } diff --git a/swarm/storage/resource_ens.go b/swarm/storage/resource_ens.go index a82311b08f..8a742e2716 100644 --- a/swarm/storage/resource_ens.go +++ b/swarm/storage/resource_ens.go @@ -8,27 +8,25 @@ import ( // ENS validation of mutable resource owners type ENSValidator struct { - owner common.Address - api *ens.ENS + api *ens.ENS } -func NewENSValidator(owneraddress common.Address, contractaddress common.Address, backend bind.ContractBackend, transactOpts *bind.TransactOpts) (*ENSValidator, error) { +func NewENSValidator(contractaddress common.Address, backend bind.ContractBackend, transactOpts *bind.TransactOpts) (*ENSValidator, error) { var err error validator := &ENSValidator{} validator.api, err = ens.NewENS(transactOpts, contractaddress, backend) if err != nil { return nil, err } - validator.owner = owneraddress return validator, nil } -func (self *ENSValidator) isOwner(name string) (bool, error) { +func (self *ENSValidator) isOwner(name string, address common.Address) (bool, error) { owneraddr, err := self.api.Owner(self.nameHash(name)) if err != nil { return false, err } - return owneraddr == self.owner, nil + return owneraddr == address, nil } func (self *ENSValidator) nameHash(name string) common.Hash { @@ -45,7 +43,7 @@ func NewGenericValidator(hashFunc func(string) common.Hash) *GenericValidator { hashFunc: hashFunc, } } -func (self *GenericValidator) isOwner(name string) (bool, error) { +func (self *GenericValidator) isOwner(name string, address common.Address) (bool, error) { return true, nil } diff --git a/swarm/storage/resource_test.go b/swarm/storage/resource_test.go index 951899ebfa..bc1fe37634 100644 --- a/swarm/storage/resource_test.go +++ b/swarm/storage/resource_test.go @@ -141,7 +141,12 @@ func TestResourceReverseLookup(t *testing.T) { } // create a new resource - rsrc, err := rh.NewResource(domainName, resourceFrequency) + signature, err := signContent(privkey, []byte(domainName)) + if err != nil { + teardownTest(t, err) + } + + rsrc, err := rh.NewResource(domainName, resourceFrequency, signature) if err != nil { teardownTest(t, err) } @@ -149,7 +154,12 @@ func TestResourceReverseLookup(t *testing.T) { // update data fwdBlocks(int(resourceFrequency+1), backend) data := []byte("foo") - resourcekey, err := rh.Update(domainName, data) + signature, err = signContent(privkey, data) + if err != nil { + teardownTest(t, err) + } + + resourcekey, err := rh.Update(domainName, data, signature) if err != nil { teardownTest(t, err) } @@ -205,7 +215,8 @@ func TestResourceHandler(t *testing.T) { if err != nil { teardownTest(t, err) } - _, err = rh.NewResource(domainName, resourceFrequency) + signature, err := signContent(privkey, []byte(resourcevalidname)) + _, err = rh.NewResource(domainName, resourceFrequency, signature) if err != nil { teardownTest(t, err) } @@ -230,28 +241,48 @@ func TestResourceHandler(t *testing.T) { // update halfway to first period resourcekey := make(map[string]Key) fwdBlocks(int(resourceFrequency/2), backend) - resourcekey["blinky"], err = rh.Update(domainName, []byte("blinky")) + data := []byte("blinky") + signature, err = signContent(privkey, data) + if err != nil { + teardownTest(t, err) + } + resourcekey["blinky"], err = rh.Update(domainName, data, signature) if err != nil { teardownTest(t, err) } // update on first period fwdBlocks(int(resourceFrequency/2), backend) - resourcekey["pinky"], err = rh.Update(domainName, []byte("pinky")) + data = []byte("pinky") + signature, err = signContent(privkey, data) + if err != nil { + teardownTest(t, err) + } + resourcekey["pinky"], err = rh.Update(domainName, data, signature) if err != nil { teardownTest(t, err) } // update on second period fwdBlocks(int(resourceFrequency), backend) - resourcekey["inky"], err = rh.Update(domainName, []byte("inky")) + data = []byte("inky") + signature, err = signContent(privkey, data) + if err != nil { + teardownTest(t, err) + } + resourcekey["inky"], err = rh.Update(domainName, data, signature) if err != nil { teardownTest(t, err) } // update just after second period fwdBlocks(1, backend) - resourcekey["clyde"], err = rh.Update(domainName, []byte("clyde")) + data = []byte("clyde") + signature, err = signContent(privkey, data) + if err != nil { + teardownTest(t, err) + } + resourcekey["clyde"], err = rh.Update(domainName, data, signature) if err != nil { teardownTest(t, err) } @@ -350,7 +381,7 @@ func TestResourceENSOwner(t *testing.T) { t.Fatal(err) } - validator, err := NewENSValidator(addr, contractAddr, contractbackend, transactOpts) + validator, err := NewENSValidator(contractAddr, contractbackend, transactOpts) if err != nil { t.Fatal(err) } @@ -361,28 +392,29 @@ func TestResourceENSOwner(t *testing.T) { teardownTest(t, err) } + signature, err := signContent(privkey, []byte(domainName)) + if err != nil { + teardownTest(t, err) + } // create new resource when we are owner = ok - _, err = rh.NewResource(domainName, 42) + _, err = rh.NewResource(domainName, 42, signature) if err != nil { teardownTest(t, fmt.Errorf("Create resource fail: %v", err)) } + data := []byte("foo") + signature, err = signContent(privkey, data) + // update resource when we are owner = ok - _, err = rh.Update(domainName, []byte("foo")) + _, err = rh.Update(domainName, data, signature) if err != nil { teardownTest(t, fmt.Errorf("Update resource fail: %v", err)) } // create new resource when we are NOT owner = !ok - addrtwo := crypto.PubkeyToAddress(privkeytwo.PublicKey) - validator.owner = addrtwo - - _, err = rh.NewResource(domainName, 42) - if err == nil { - teardownTest(t, fmt.Errorf("Expected resource create fail due to owner mismatch")) - } + signaturetwo, err := signContent(privkeytwo, data) // update resource when we are owner = ok - _, err = rh.Update(domainName, []byte("foo")) + _, err = rh.Update(domainName, data, signaturetwo) if err == nil { teardownTest(t, fmt.Errorf("Expected resource update fail due to owner mismatch")) } @@ -503,6 +535,20 @@ func setupENS(addr common.Address, transactOpts *bind.TransactOpts, sub string, return contractAddress, contractBackend, nil } +func signContent(privKey *ecdsa.PrivateKey, data []byte) ([signatureLength]byte, error) { + hasher.Reset() + hasher.Write(data) + datahash := hasher.Sum(nil) + + signature, err := crypto.Sign(datahash, privKey) + if err != nil { + return [signatureLength]byte{}, err + } + var signaturetype [signatureLength]byte + copy(signaturetype[:], signature) + return signaturetype, nil +} + type testCloudStore struct { } From f482ecc6de362489ef7ea9bdfb23397cc1d04297 Mon Sep 17 00:00:00 2001 From: lash Date: Wed, 17 Jan 2018 06:22:46 +0100 Subject: [PATCH 040/107] swarm/storage: Add signature type --- swarm/storage/resource.go | 25 +++++++++++++++++++------ swarm/storage/resource_test.go | 9 ++++----- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index c4b4e51597..1de6254545 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -22,6 +22,17 @@ const ( indexSize = 24 ) +type Signature [signatureLength]byte + +func NewSignature(b []byte) (Signature, error) { + var s Signature + if len(b) != signatureLength { + return [signatureLength]byte{}, fmt.Errorf("Must be %d bytes", signatureLength) + } + copy(s[:], b) + return s, nil +} + // Encapsulates an actual resource update. When synced it contains the most recent // version of the resource update data. type resource struct { @@ -185,7 +196,7 @@ func NewResource(name string, startBlock uint64, frequency uint64, nameHashFunc // Creates a new root entry for a mutable resource identified by `name` with the specified `frequency`. // // The start block of the resource update will be the actual current block height of the connected network. -func (self *ResourceHandler) NewResource(name string, frequency uint64, signature [signatureLength]byte) (*resource, error) { +func (self *ResourceHandler) NewResource(name string, frequency uint64, signature Signature) (*resource, error) { addr, err := self.getAddressFromDataSig([]byte(name), signature) if err != nil { @@ -465,7 +476,7 @@ func parseUpdate(blob []byte) (period uint32, version uint32, ensname []byte, da // It is the caller's responsibility to make sure that this data is not stale. // // A resource update cannot span chunks, and thus has max length 4096 -func (self *ResourceHandler) Update(name string, data []byte, signature [signatureLength]byte) (Key, error) { +func (self *ResourceHandler) Update(name string, data []byte, signature Signature) (Key, error) { addr, err := self.getAddressFromDataSig(data, signature) if err != nil { @@ -606,9 +617,11 @@ func (self *ResourceHandler) getContentAccount(chunkdata []byte) (common.Address if len(chunkdata) <= signatureLength { return common.Address{}, fmt.Errorf("zero-length data") } - var signaturetype [signatureLength]byte - copy(signaturetype[:], chunkdata[:signatureLength]) - return self.getAddressFromDataSig(chunkdata[signatureLength:], signaturetype) + signature, err := NewSignature(chunkdata[:signatureLength]) + if err != nil { + return zeroAddr, err + } + return self.getAddressFromDataSig(chunkdata[signatureLength:], signature) } func (self *ResourceHandler) getContentName(chunkdata []byte) (string, error) { @@ -622,7 +635,7 @@ func (self *ResourceHandler) getContentName(chunkdata []byte) (string, error) { return string(namebytes), nil } -func (self *ResourceHandler) getAddressFromDataSig(data []byte, signature [signatureLength]byte) (common.Address, error) { +func (self *ResourceHandler) getAddressFromDataSig(data []byte, signature Signature) (common.Address, error) { self.hashLock.Lock() self.hasher.Reset() self.hasher.Write(data) diff --git a/swarm/storage/resource_test.go b/swarm/storage/resource_test.go index bc1fe37634..a8b801b2f1 100644 --- a/swarm/storage/resource_test.go +++ b/swarm/storage/resource_test.go @@ -535,18 +535,17 @@ func setupENS(addr common.Address, transactOpts *bind.TransactOpts, sub string, return contractAddress, contractBackend, nil } -func signContent(privKey *ecdsa.PrivateKey, data []byte) ([signatureLength]byte, error) { +func signContent(privKey *ecdsa.PrivateKey, data []byte) (Signature, error) { hasher.Reset() hasher.Write(data) datahash := hasher.Sum(nil) - signature, err := crypto.Sign(datahash, privKey) + signaturebytes, err := crypto.Sign(datahash, privKey) if err != nil { return [signatureLength]byte{}, err } - var signaturetype [signatureLength]byte - copy(signaturetype[:], signature) - return signaturetype, nil + signature, err := NewSignature(signaturebytes) + return signature, err } type testCloudStore struct { From e1e867cdf9875e060aed1a5a5a91a0fb39ba1a4f Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 18 Jan 2018 01:13:13 +0100 Subject: [PATCH 041/107] swarm/storage: Simplify code, correct content hashing --- swarm/storage/resource.go | 417 +++++++++++++++------------------ swarm/storage/resource_ens.go | 48 +++- swarm/storage/resource_test.go | 328 +++++++++++--------------- 3 files changed, 361 insertions(+), 432 deletions(-) diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 1de6254545..04e8d06ac4 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -1,7 +1,6 @@ package storage import ( - "crypto/ecdsa" "encoding/binary" "fmt" "path/filepath" @@ -19,12 +18,18 @@ import ( const ( signatureLength = 65 - indexSize = 24 + indexSize = 16 + dbDirName = "resource" + chunkSize = 4096 // temporary until we implement DPA in the resourcehandler ) type Signature [signatureLength]byte -func NewSignature(b []byte) (Signature, error) { +var emptySignature Signature + +type SignFunc func(common.Hash) (Signature, error) + +func bytesToSignature(b []byte) (Signature, error) { var s Signature if len(b) != signatureLength { return [signatureLength]byte{}, fmt.Errorf("Must be %d bytes", signatureLength) @@ -46,6 +51,11 @@ type resource struct { updated time.Time } +// TODO Expire content after a defined period (to force resync) +func (r *resource) isSynced() bool { + return !r.updated.IsZero() +} + // Mutable resource is an entity which allows updates to a resource // without resorting to ENS on each update. // The update scheme is built on swarm chunks with chunk keys following @@ -104,8 +114,9 @@ type resource struct { // treated as a resource update chunk. type ResourceValidator interface { - isOwner(string, common.Address) (bool, error) + checkAccess(string, common.Address) (bool, error) nameHash(string) common.Hash + sign(common.Hash) (Signature, error) // SignFunc } type ResourceHandler struct { @@ -116,13 +127,15 @@ type ResourceHandler struct { hashLock sync.Mutex resourceLock sync.RWMutex hasher SwarmHash - maxChunkData int64 } // Create or open resource update chunk store -func NewResourceHandler(privKey *ecdsa.PrivateKey, datadir string, cloudStore CloudStore, rpcClient *rpc.Client, validator ResourceValidator) (*ResourceHandler, error) { - path := filepath.Join(datadir, "resource") - dbStore, err := NewDbStore(datadir, nil, singletonSwarmDbCapacity, 0) +func NewResourceHandler(datadir string, cloudStore CloudStore, rpcClient *rpc.Client, validator ResourceValidator) (*ResourceHandler, error) { + + hashfunc := MakeHashFunc(SHA3Hash) + + path := filepath.Join(datadir, dbDirName) + dbStore, err := NewDbStore(datadir, hashfunc, singletonSwarmDbCapacity, 0) if err != nil { return nil, err } @@ -130,14 +143,12 @@ func NewResourceHandler(privKey *ecdsa.PrivateKey, datadir string, cloudStore Cl memStore: NewMemStore(dbStore, singletonSwarmDbCapacity), DbStore: dbStore, } - hasher := MakeHashFunc("SHA3") rh := &ResourceHandler{ - ChunkStore: newResourceChunkStore(path, hasher, localStore, cloudStore), - rpcClient: rpcClient, - resources: make(map[string]*resource), - hasher: hasher(), - maxChunkData: DefaultBranches * int64(hasher().Size()), + ChunkStore: newResourceChunkStore(path, hashfunc, localStore, cloudStore), + rpcClient: rpcClient, + resources: make(map[string]*resource), + hasher: hashfunc(), } if validator != nil { @@ -149,72 +160,57 @@ func NewResourceHandler(privKey *ecdsa.PrivateKey, datadir string, cloudStore Cl rh.hasher.Reset() rh.hasher.Write([]byte(name)) return common.BytesToHash(rh.hasher.Sum(nil)) - }) + }, nil) } return rh, nil } -func validateInput(name string, frequency uint64) (string, error) { - // frequency 0 is invalid - if frequency == 0 { - return "", fmt.Errorf("Frequency cannot be 0") - } - - // must have name - if name == "" { - return "", fmt.Errorf("Name cannot be empty") - } - - // make sure our ens identifier is idna safe - validname, err := idna.ToASCII(name) - if err != nil { - return "", err - } - - return validname, nil -} - -// Creates a standalone resource object -// -// Can be passed to SetResource if external root data lookups are used -func NewResource(name string, startBlock uint64, frequency uint64, nameHashFunc func(name string) common.Hash) (*resource, error) { - - validname, err := validateInput(name, frequency) - if err != nil { - return nil, err - } - - return &resource{ - name: validname, - nameHash: nameHashFunc(validname), - startBlock: startBlock, - frequency: frequency, - }, nil +// \TODO should be hashsize * branches from the chosen chunker, implement with dpa +func (self *ResourceHandler) chunkSize() int64 { + return chunkSize } // Creates a new root entry for a mutable resource identified by `name` with the specified `frequency`. // +// The signature data should match the hash of the idna-converted name by the validator's namehash function, NOT the raw name bytes. +// // The start block of the resource update will be the actual current block height of the connected network. -func (self *ResourceHandler) NewResource(name string, frequency uint64, signature Signature) (*resource, error) { +func (self *ResourceHandler) NewResource(name string, frequency uint64, verify bool) (*resource, error) { - addr, err := self.getAddressFromDataSig([]byte(name), signature) - if err != nil { - return nil, fmt.Errorf("Corrupt signature") - } - ok, err := self.validator.isOwner(name, addr) - if err != nil { - return nil, err - } else if !ok { - return nil, fmt.Errorf("Not owner of '%s'", name) + // frequency 0 is invalid + if frequency == 0 { + return nil, fmt.Errorf("Frequency cannot be 0") } - validname, err := validateInput(name, frequency) + // must have name + if name == "" { + return nil, fmt.Errorf("Empty name") + } + + validName, err := toSafeName(name) if err != nil { return nil, err } - nameHash := self.validator.nameHash(validname) + nameHash := self.validator.nameHash(validName) + + if verify { + signature, err := self.validator.sign(nameHash) + if err != nil { + return nil, fmt.Errorf("Sign fail: %v", err) + } + addr, err := getAddressFromDataSig(nameHash, signature) + if err != nil { + return nil, fmt.Errorf("Retrieve address from signature fail: %v", err) + } + ok, err := self.validator.checkAccess(name, addr) + if err != nil { + return nil, err + } else if !ok { + return nil, fmt.Errorf("Not owner of '%s'", name) + } + } // get our blockheight at this time currentblock, err := self.getBlock() @@ -224,22 +220,19 @@ func (self *ResourceHandler) NewResource(name string, frequency uint64, signatur // chunk with key equal to namehash points to data of first blockheight + update frequency // from this we know from what blockheight we should look for updates, and how often - chunk := NewChunk(Key(nameHash[:]), nil) + chunk := NewChunk(Key(nameHash.Bytes()), nil) chunk.SData = make([]byte, indexSize) - // resource update root chunks follow same convention as "normal" chunks - // with 8 bytes prefix specifying size val := make([]byte, 8) - chunk.SData[0] = 16 // size, little-endian binary.LittleEndian.PutUint64(val, currentblock) - copy(chunk.SData[8:16], val) + copy(chunk.SData[:8], val) binary.LittleEndian.PutUint64(val, frequency) - copy(chunk.SData[16:], val) + copy(chunk.SData[8:], val) self.Put(chunk) - log.Debug("new resource", "name", validname, "key", nameHash, "startBlock", currentblock, "frequency", frequency) + log.Debug("new resource", "name", validName, "key", nameHash, "startBlock", currentblock, "frequency", frequency) rsrc := &resource{ - name: validname, + name: validName, nameHash: nameHash, startBlock: currentblock, frequency: frequency, @@ -250,42 +243,6 @@ func (self *ResourceHandler) NewResource(name string, frequency uint64, signatur return self.resources[name], nil } -// Set an externally defined resource object -// -// If the resource update root chunk is located externally (for example as a normal -// chunk looked up by ENS) the data would be manually added with this method). -// -// Method will fail if resource is already registered in this session, unless -// `allowOverwrite` is set -func (self *ResourceHandler) SetExternalResource(rsrc *resource, allowOverwrite bool) error { - - utfname, err := idna.ToUnicode(rsrc.name) - if err != nil { - return fmt.Errorf("Invalid IDNA rsrc name '%s'", rsrc.name) - } - if !allowOverwrite { - self.resourceLock.Lock() - _, ok := self.resources[utfname] - self.resourceLock.Unlock() - if ok { - return fmt.Errorf("Resource exists") - } - } - - // get our blockheight at this time - currentblock, err := self.getBlock() - if err != nil { - return err - } - - if rsrc.startBlock > currentblock { - return fmt.Errorf("Startblock cannot be higher than current block (%d > %d)", rsrc.startBlock, currentblock) - } - - self.resources[utfname] = rsrc - return nil -} - // Searches and retrieves the specific version of the resource update identified by `name` // at the specific block height // @@ -360,7 +317,7 @@ func (self *ResourceHandler) lookup(rsrc *resource, name string, period uint32, } for period > 0 { - key := self.resourceHash(rsrc.nameHash, period, version) + key := self.resourceHash(period, version, rsrc.nameHash) chunk, err := self.Get(key) if err == nil { if specificversion { @@ -370,7 +327,7 @@ func (self *ResourceHandler) lookup(rsrc *resource, name string, period uint32, log.Trace("rsrc update version 1 found, checking for version updates", "period", period, "key", key) for { newversion := version + 1 - key := self.resourceHash(rsrc.nameHash, period, newversion) + key := self.resourceHash(period, newversion, rsrc.nameHash) newchunk, err := self.Get(key) if err != nil { return self.updateResourceIndex(rsrc, chunk, &name) @@ -394,8 +351,8 @@ func (self *ResourceHandler) loadResource(name string, refresh bool) (*resource, rsrc := self.getResource(name) if rsrc == nil || refresh { rsrc = &resource{} - // make sure our ens identifier is idna safe - validname, err := idna.ToASCII(name) + // make sure our name is safe to use + validname, err := toSafeName(name) if err != nil { return nil, err } @@ -409,17 +366,11 @@ func (self *ResourceHandler) loadResource(name string, refresh bool) (*resource, } // sanity check for chunk data - // data is prefixed by 8 bytes of size - if len(chunk.SData) < indexSize { - return nil, fmt.Errorf("Invalid chunk length %d", len(chunk.SData)) - } else { - chunklength := binary.LittleEndian.Uint64(chunk.SData[:8]) - if chunklength != uint64(16) { - return nil, fmt.Errorf("Invalid chunk length header %d", chunklength) - } + if len(chunk.SData) != indexSize { + return nil, fmt.Errorf("Invalid chunk length %d, should be %d", len(chunk.SData), indexSize) } - rsrc.startBlock = binary.LittleEndian.Uint64(chunk.SData[8:16]) - rsrc.frequency = binary.LittleEndian.Uint64(chunk.SData[16:]) + rsrc.startBlock = binary.LittleEndian.Uint64(chunk.SData[:8]) + rsrc.frequency = binary.LittleEndian.Uint64(chunk.SData[8:]) } else { rsrc.name = self.resources[name].name rsrc.nameHash = self.resources[name].nameHash @@ -432,15 +383,21 @@ func (self *ResourceHandler) loadResource(name string, refresh bool) (*resource, // update mutable resource index map with specified content func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk, indexname *string) (*resource, error) { - // rsrc update data chunks are total hacks - // and have no size prefix :D - err := self.verifyContent(chunk.SData) - if err != nil { - return nil, err - } - // update our rsrcs entry map - period, version, _, data, err := parseUpdate(chunk.SData[signatureLength:]) + signature, period, version, name, data, err := parseUpdate(chunk.SData) + if rsrc.name != name { + return nil, fmt.Errorf("Update belongs to '%s', but have '%s'", name, rsrc.name) + } + self.hashLock.Lock() + self.hasher.Reset() + self.hasher.Write(chunk.Key[:]) + self.hasher.Write(data) + digest := self.hasher.Sum(nil) + self.hashLock.Unlock() + _, err = getAddressFromDataSig(common.BytesToHash(digest), signature) + if err != nil { + return nil, fmt.Errorf("Invalid signature: %v", err) + } rsrc.lastPeriod = period rsrc.version = version rsrc.updated = time.Now() @@ -451,22 +408,23 @@ func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk, i return rsrc, nil } -func parseUpdate(blob []byte) (period uint32, version uint32, ensname []byte, data []byte, err error) { - headerlength := binary.LittleEndian.Uint16(blob[:2]) - if int(headerlength+2) > len(blob) { - return 0, 0, nil, nil, fmt.Errorf("Reported header length %d longer than actual data length %d", headerlength, len(blob)) +func parseUpdate(chunkdata []byte) (signature Signature, period uint32, version uint32, name string, data []byte, err error) { + copy(signature[:], chunkdata[:signatureLength]) + cursor := signatureLength + headerlength := binary.LittleEndian.Uint16(chunkdata[cursor : cursor+2]) + if int(headerlength+2) > len(chunkdata) { + return emptySignature, 0, 0, "", nil, fmt.Errorf("Reported header length %d longer than actual data length %d", headerlength, len(chunkdata)) } - cursor := 2 - period = binary.LittleEndian.Uint32(blob[cursor : cursor+4]) + cursor += 2 + period = binary.LittleEndian.Uint32(chunkdata[cursor : cursor+4]) cursor += 4 - version = binary.LittleEndian.Uint32(blob[cursor : cursor+4]) + version = binary.LittleEndian.Uint32(chunkdata[cursor : cursor+4]) cursor += 4 - namelength := int(headerlength) - cursor + 2 - ensname = make([]byte, namelength) - copy(ensname, blob[cursor:]) + namelength := int(headerlength) - cursor + signatureLength + 2 + name = string(chunkdata[cursor : cursor+namelength]) cursor += namelength - data = make([]byte, len(blob)-cursor) - copy(data, blob[cursor:]) + data = make([]byte, len(chunkdata)-cursor) + copy(data, chunkdata[cursor:]) return } @@ -476,32 +434,18 @@ func parseUpdate(blob []byte) (period uint32, version uint32, ensname []byte, da // It is the caller's responsibility to make sure that this data is not stale. // // A resource update cannot span chunks, and thus has max length 4096 -func (self *ResourceHandler) Update(name string, data []byte, signature Signature) (Key, error) { - - addr, err := self.getAddressFromDataSig(data, signature) - if err != nil { - return nil, fmt.Errorf("Invalid data/signature: %v", err) - } - ok, err := self.validator.isOwner(name, addr) - if err != nil { - return nil, err - } else if !ok { - return nil, fmt.Errorf("Not owner of '%s'", name) - } - - // can be only one chunk long minus 65 byte signature - if int64(len(data)) > self.maxChunkData { - return nil, fmt.Errorf("Data overflow: %d / %d bytes", len(data), 4096-signatureLength) - } +func (self *ResourceHandler) Update(indexname string, data []byte) (Key, error) { // get the cached information - self.resourceLock.Lock() - defer self.resourceLock.Unlock() - resource, ok := self.resources[name] - if !ok { - return nil, fmt.Errorf("No such resource") - } else if resource.updated.IsZero() { - return nil, fmt.Errorf("Invalid resource") + rsrc := self.getResource(indexname) + if !rsrc.isSynced() { + return nil, fmt.Errorf("Resource object not in sync") + } + + // an update can be only one chunk long + datalimit := self.chunkSize() - int64(signatureLength-len(self.resources[indexname].name)-8) + if int64(len(data)) > datalimit { + return nil, fmt.Errorf("Data overflow: %d / %d bytes", len(data), datalimit) } // get our blockheight at this time and the next block of the update period @@ -509,21 +453,59 @@ func (self *ResourceHandler) Update(name string, data []byte, signature Signatur if err != nil { return nil, err } - nextperiod := getNextPeriod(resource.startBlock, currentblock, resource.frequency) + nextperiod := getNextPeriod(rsrc.startBlock, currentblock, rsrc.frequency) // if we already have an update for this block then increment version + // (resource object MUST be in sync for version to be correct) var version uint32 - if self.hasUpdate(name, nextperiod) { - version = resource.version + if self.hasUpdate(indexname, nextperiod) { + version = rsrc.version } version++ + // calculate the chunk key + key := self.resourceHash(nextperiod, version, rsrc.nameHash) + + // sign the data hash with the key + digest := self.keyDataHash(key, data) + signature, err := self.validator.sign(digest) + if err != nil { + return nil, err + } + + // get the address of the signer (which also checks that it's a valid signature) + addr, err := getAddressFromDataSig(digest, signature) + if err != nil { + return nil, fmt.Errorf("Invalid data/signature: %v", err) + } + + // check if the signer has access to update + ok, err := self.validator.checkAccess(indexname, addr) + if err != nil { + return nil, err + } else if !ok { + return nil, fmt.Errorf("Address %x does not have access to update %s", addr, indexname) + } + + chunk := newUpdateChunk(key, signature, nextperiod, version, self.resources[indexname].name, data) + + // send the chunk + self.Put(chunk) + log.Trace("resource update", "name", rsrc.name, "key", key, "currentblock", currentblock, "lastperiod", nextperiod, "version", version, "data", chunk.SData) + + // update our resources map entry and return the new key + rsrc.lastPeriod = nextperiod + rsrc.version = version + rsrc.data = make([]byte, len(data)) + copy(rsrc.data, data) + return key, nil +} + +func newUpdateChunk(key Key, signature Signature, period uint32, version uint32, name string, data []byte) *Chunk { // create the update chunk // prepend version and period to allow reverse lookups - // data header length does NOT include the header length prefix bytes themselves - headerlength := uint16(len(resource.nameHash) + 4 + 4) + headerlength := uint16(len(name) + 4 + 4) - key := self.resourceHash(resource.nameHash, nextperiod, version) chunk := NewChunk(key, nil) chunk.SData = make([]byte, signatureLength+int(headerlength)+2+len(data)) @@ -531,32 +513,24 @@ func (self *ResourceHandler) Update(name string, data []byte, signature Signatur copy(chunk.SData, signature[:]) cursor += signatureLength + // data header length does NOT include the header length prefix bytes themselves binary.LittleEndian.PutUint16(chunk.SData[cursor:], headerlength) cursor += 2 - binary.LittleEndian.PutUint32(chunk.SData[cursor:], nextperiod) + binary.LittleEndian.PutUint32(chunk.SData[cursor:], period) cursor += 4 binary.LittleEndian.PutUint32(chunk.SData[cursor:], version) cursor += 4 - copy(chunk.SData[cursor:], resource.nameHash[:]) - cursor += len(resource.nameHash) + namebytes := []byte(name) + copy(chunk.SData[cursor:], namebytes) + cursor += len(namebytes) copy(chunk.SData[cursor:], data) chunk.Size = int64(len(chunk.SData)) - - // send the chunk - self.Put(chunk) - log.Trace("resource update", "name", resource.name, "key", key, "currentblock", currentblock, "lastperiod", nextperiod, "version", version, "data", chunk.SData) - - // update our resources map entry and return the new key - resource.lastPeriod = nextperiod - resource.version = version - resource.data = make([]byte, len(data)) - copy(resource.data, data) - return key, nil + return chunk } // Closes the datastore. @@ -599,80 +573,37 @@ func (self *ResourceHandler) setResource(name string, rsrc *resource) { self.resources[name] = rsrc } -func (self *ResourceHandler) resourceHash(namehash common.Hash, period uint32, version uint32) Key { +func (self *ResourceHandler) resourceHash(period uint32, version uint32, namehash common.Hash) Key { // format is: hash(namehash|period|version) self.hashLock.Lock() defer self.hashLock.Unlock() self.hasher.Reset() - self.hasher.Write(namehash[:]) b := make([]byte, 4) binary.LittleEndian.PutUint32(b, period) self.hasher.Write(b) binary.LittleEndian.PutUint32(b, version) self.hasher.Write(b) + self.hasher.Write(namehash[:]) return self.hasher.Sum(nil) } -func (self *ResourceHandler) getContentAccount(chunkdata []byte) (common.Address, error) { - if len(chunkdata) <= signatureLength { - return common.Address{}, fmt.Errorf("zero-length data") - } - signature, err := NewSignature(chunkdata[:signatureLength]) - if err != nil { - return zeroAddr, err - } - return self.getAddressFromDataSig(chunkdata[signatureLength:], signature) -} - -func (self *ResourceHandler) getContentName(chunkdata []byte) (string, error) { - nameoffset := signatureLength + 2 + 4 + 4 - if len(chunkdata) < nameoffset { - return "", fmt.Errorf("invalid chunk data") - } - namelength := binary.LittleEndian.Uint16(chunkdata[signatureLength : signatureLength+2]) - namebytes := make([]byte, namelength) - copy(namebytes, chunkdata[nameoffset:nameoffset+int(namelength)-2-4-4]) - return string(namebytes), nil -} - -func (self *ResourceHandler) getAddressFromDataSig(data []byte, signature Signature) (common.Address, error) { - self.hashLock.Lock() - self.hasher.Reset() - self.hasher.Write(data) - datahash := self.hasher.Sum(nil) - self.hashLock.Unlock() - pub, err := crypto.SigToPub(datahash, signature[:]) +func getAddressFromDataSig(datahash common.Hash, signature Signature) (common.Address, error) { + pub, err := crypto.SigToPub(datahash.Bytes(), signature[:]) if err != nil { return common.Address{}, err } return crypto.PubkeyToAddress(*pub), nil } -func (self *ResourceHandler) verifyContent(chunkdata []byte) error { - address, err := self.getContentAccount(chunkdata) - if err != nil { - return err - } - name, err := self.getContentName(chunkdata) - if err != nil { - return err - } - ok, err := self.validator.isOwner(name, address) - if err != nil { - return err - } else if !ok { - return fmt.Errorf("not owner") - } - return nil -} - func (self *ResourceHandler) hasUpdate(name string, period uint32) bool { return self.resources[name].lastPeriod == period } +// \TODO chunkSize is a workaround until the ChunkStore interface exports a method to get the chunk size directly type resourceChunkStore struct { localStore ChunkStore netStore ChunkStore + chunkSize int64 } func newResourceChunkStore(path string, hasher SwarmHasher, localStore *LocalStore, cloudStore CloudStore) *resourceChunkStore { @@ -717,3 +648,21 @@ func getNextPeriod(start uint64, current uint64, frequency uint64) uint32 { period := blockdiff / frequency return uint32(period + 1) } + +func toSafeName(name string) (string, error) { + // make sure our ens identifier is idna safe + validname, err := idna.ToASCII(name) + if err != nil { + return "", err + } + return validname, nil +} + +func (self *ResourceHandler) keyDataHash(key Key, data []byte) common.Hash { + self.hashLock.Lock() + defer self.hashLock.Unlock() + self.hasher.Reset() + self.hasher.Write(key[:]) + self.hasher.Write(data) + return common.BytesToHash(self.hasher.Sum(nil)) +} diff --git a/swarm/storage/resource_ens.go b/swarm/storage/resource_ens.go index 8a742e2716..ccd4ed5344 100644 --- a/swarm/storage/resource_ens.go +++ b/swarm/storage/resource_ens.go @@ -1,27 +1,47 @@ package storage import ( + "fmt" + "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/contracts/ens" ) -// ENS validation of mutable resource owners -type ENSValidator struct { - api *ens.ENS +type baseValidator struct { + signFunc SignFunc } -func NewENSValidator(contractaddress common.Address, backend bind.ContractBackend, transactOpts *bind.TransactOpts) (*ENSValidator, error) { +func (b *baseValidator) sign(datahash common.Hash) (Signature, error) { + if b.signFunc == nil { + return emptySignature, fmt.Errorf("No signature function") + } + return b.signFunc(datahash) +} + +// ENS validation of mutable resource owners +type ENSValidator struct { + *baseValidator + api *ens.ENS + hashlength int +} + +func NewENSValidator(contractaddress common.Address, backend bind.ContractBackend, transactOpts *bind.TransactOpts, signFunc SignFunc) (*ENSValidator, error) { var err error - validator := &ENSValidator{} + validator := &ENSValidator{ + baseValidator: &baseValidator{ + signFunc: signFunc, + }, + } validator.api, err = ens.NewENS(transactOpts, contractaddress, backend) if err != nil { return nil, err } + validator.hashlength = len(ens.EnsNode(dbDirName).Bytes()) return validator, nil } -func (self *ENSValidator) isOwner(name string, address common.Address) (bool, error) { +func (self *ENSValidator) checkAccess(name string, address common.Address) (bool, error) { owneraddr, err := self.api.Owner(self.nameHash(name)) if err != nil { return false, err @@ -35,15 +55,23 @@ func (self *ENSValidator) nameHash(name string) common.Hash { // Default fallthrough validation of mutable resource ownership type GenericValidator struct { - hashFunc func(string) common.Hash + *baseValidator + hashFunc func(string) common.Hash + hashlength int } -func NewGenericValidator(hashFunc func(string) common.Hash) *GenericValidator { +func NewGenericValidator(hashFunc func(string) common.Hash, signFunc SignFunc) *GenericValidator { return &GenericValidator{ - hashFunc: hashFunc, + baseValidator: &baseValidator{ + signFunc: signFunc, + }, + hashFunc: hashFunc, + hashlength: len(hashFunc(dbDirName).Bytes()), } + } -func (self *GenericValidator) isOwner(name string, address common.Address) (bool, error) { + +func (self *GenericValidator) checkAccess(name string, address common.Address) (bool, error) { return true, nil } diff --git a/swarm/storage/resource_test.go b/swarm/storage/resource_test.go index a8b801b2f1..2a4f4ca4d5 100644 --- a/swarm/storage/resource_test.go +++ b/swarm/storage/resource_test.go @@ -29,7 +29,7 @@ import ( ) var ( - hasher = MakeHashFunc("SHA3")() + testHasher = MakeHashFunc(SHA3Hash)() zeroAddr = common.Address{} startBlock = uint64(4200) resourceFrequency = uint64(42) @@ -67,14 +67,8 @@ func (r *FakeRPC) BlockNumber() (string, error) { // check that signature address matches update signer address func TestResourceSignature(t *testing.T) { - // privkey for signing updates - privkey, err := crypto.GenerateKey() - if err != nil { - return - } - // set up rpc and create resourcehandler - rh, _, err, teardownTest := setupTest(privkey, nil, nil) + rh, _, signer, teardownTest, err := setupTest(nil, nil) if err != nil { teardownTest(t, err) } @@ -86,8 +80,7 @@ func TestResourceSignature(t *testing.T) { } // generate a hash for block 4200 version 1 - key := rh.resourceHash(ens.EnsNode(validname), 1, 1) - chunk := NewChunk(key, nil) + key := rh.resourceHash(1, 1, rh.validator.nameHash(validname)) // generate some bogus data for the chunk and sign it data := make([]byte, 8) @@ -95,26 +88,26 @@ func TestResourceSignature(t *testing.T) { if err != nil { teardownTest(t, err) } - hasher.Reset() - hasher.Write(data) - datahash := hasher.Sum(nil) - sig, err := crypto.Sign(datahash, privkey) + testHasher.Reset() + testHasher.Write(data) + digest := rh.keyDataHash(key, data) + sig, err := rh.validator.sign(digest) if err != nil { teardownTest(t, err) } - // put sig and data in the chunk - chunk.SData = make([]byte, 8+signatureLength) - copy(chunk.SData[:signatureLength], sig) - copy(chunk.SData[signatureLength:], data) + chunk := newUpdateChunk(key, sig, 1, 1, validname, data) + + log.Warn("key", "chunk", chunk.Key, "real", key) // check that we can recover the owner account from the update chunk's signature - // TODO: change this to verifyContent on ENS integration - recoveredaddress, err := rh.getContentAccount(chunk.SData) + checksig, _, _, _, newdata, err := parseUpdate(chunk.SData) + checkdigest := rh.keyDataHash(chunk.Key, newdata) + recoveredaddress, err := getAddressFromDataSig(checkdigest, checksig) if err != nil { teardownTest(t, err) } - originaladdress := crypto.PubkeyToAddress(privkey.PublicKey) + originaladdress := crypto.PubkeyToAddress(signer.privKey.PublicKey) if recoveredaddress != originaladdress { teardownTest(t, fmt.Errorf("addresses dont match: %x != %x", originaladdress, recoveredaddress)) @@ -122,90 +115,69 @@ func TestResourceSignature(t *testing.T) { teardownTest(t, nil) } -// determine resource update metadata from chunk data -func TestResourceReverseLookup(t *testing.T) { - - // privkey for signing updates - privkey, err := crypto.GenerateKey() - if err != nil { - return - } - - // make fake backend, set up rpc and create resourcehandler - backend := &fakeBackend{ - blocknumber: startBlock, - } - rh, _, err, teardownTest := setupTest(privkey, backend, nil) - if err != nil { - teardownTest(t, err) - } - - // create a new resource - signature, err := signContent(privkey, []byte(domainName)) - if err != nil { - teardownTest(t, err) - } - - rsrc, err := rh.NewResource(domainName, resourceFrequency, signature) - if err != nil { - teardownTest(t, err) - } - - // update data - fwdBlocks(int(resourceFrequency+1), backend) - data := []byte("foo") - signature, err = signContent(privkey, data) - if err != nil { - teardownTest(t, err) - } - - resourcekey, err := rh.Update(domainName, data, signature) - if err != nil { - teardownTest(t, err) - } - chunk, err := rh.ChunkStore.(*resourceChunkStore).localStore.(*LocalStore).memStore.Get(Key(resourcekey)) - if err != nil { - teardownTest(t, err) - } - - // check if data after header length offset is as expected - headerlength := binary.LittleEndian.Uint16(chunk.SData[signatureLength : signatureLength+2]) - if !bytes.Equal(chunk.SData[signatureLength+headerlength+2:], data) { - teardownTest(t, fmt.Errorf("Expected chunk data with header length %d (pos %d) to match %x, but was %x", headerlength, signatureLength+headerlength+2, data, chunk.SData[signatureLength+headerlength+2:])) - } - - // get name, period, version from chunk and check - revperiod, revversion, revname, revdata, err := parseUpdate(chunk.SData[signatureLength:]) - - if !bytes.Equal(revname, rsrc.nameHash.Bytes()) { - teardownTest(t, fmt.Errorf("Expected retrieved name from chunk data to be '%x', was '%x'", rsrc.nameHash.Bytes(), revname)) - } - if !bytes.Equal(revdata, data) { - teardownTest(t, fmt.Errorf("Expected retrieved data from chunk data to be '%x', was '%x'", data, revdata)) - } - - if revperiod != 2 { - teardownTest(t, fmt.Errorf("Expected retrieved period from chunk data to be 1, was %d", revperiod)) - } - if revversion != 1 { - teardownTest(t, fmt.Errorf("Expected retrieved version from chunk data to be 1, was %d", revversion)) - } -} - +// +//// determine resource update metadata from chunk data +//func TestResourceReverseLookup(t *testing.T) { +// +// // make fake backend, set up rpc and create resourcehandler +// backend := &fakeBackend{ +// blocknumber: startBlock, +// } +// rh, _, _, teardownTest, err := setupTest(backend, nil) +// if err != nil { +// teardownTest(t, err) +// } +// +// rsrc, err := rh.NewResource(domainName, resourceFrequency, false) +// if err != nil { +// teardownTest(t, err) +// } +// +// // update data +// fwdBlocks(int(resourceFrequency+1), backend) +// data := []byte("foo") +// resourcekey, err := rh.Update(domainName, data) +// if err != nil { +// teardownTest(t, err) +// } +// chunk, err := rh.ChunkStore.(*resourceChunkStore).localStore.(*LocalStore).memStore.Get(Key(resourcekey)) +// if err != nil { +// teardownTest(t, err) +// } +// +// // check if data after header length offset is as expected +// headerlength := binary.LittleEndian.Uint16(chunk.SData[signatureLength : signatureLength+2]) +// if !bytes.Equal(chunk.SData[signatureLength+headerlength+2:], data) { +// teardownTest(t, fmt.Errorf("Expected chunk data with header length %d (pos %d) to match %x, but was %x", headerlength, signatureLength+headerlength+2, data, chunk.SData[signatureLength+headerlength+2:])) +// } +// +// // get name, period, version from chunk and check +// _, revperiod, revversion, revname, revdata, err := parseUpdate(chunk.SData[signatureLength:]) +// +// //if !bytes.Equal(revname, rsrc.nameHash.Bytes()) { +// if revname == rsrc.name { +// teardownTest(t, fmt.Errorf("Expected retrieved name from chunk data to be '%x', was '%x'", rsrc.nameHash.Bytes(), revname)) +// } +// if !bytes.Equal(revdata, data) { +// teardownTest(t, fmt.Errorf("Expected retrieved data from chunk data to be '%x', was '%x'", data, revdata)) +// } +// +// if revperiod != 2 { +// teardownTest(t, fmt.Errorf("Expected retrieved period from chunk data to be 1, was %d", revperiod)) +// } +// if revversion != 1 { +// teardownTest(t, fmt.Errorf("Expected retrieved version from chunk data to be 1, was %d", revversion)) +// } +//} +// // make updates and retrieve them based on periods and versions func TestResourceHandler(t *testing.T) { - // privkey for signing updates - privkey, err := crypto.GenerateKey() - if err != nil { - return - } - // make fake backend, set up rpc and create resourcehandler backend := &fakeBackend{ blocknumber: startBlock, } - rh, datadir, err, teardownTest := setupTest(privkey, backend, nil) + rh, datadir, _, teardownTest, err := setupTest(backend, nil) if err != nil { teardownTest(t, err) } @@ -215,8 +187,7 @@ func TestResourceHandler(t *testing.T) { if err != nil { teardownTest(t, err) } - signature, err := signContent(privkey, []byte(resourcevalidname)) - _, err = rh.NewResource(domainName, resourceFrequency, signature) + _, err = rh.NewResource(domainName, resourceFrequency, false) if err != nil { teardownTest(t, err) } @@ -229,8 +200,8 @@ func TestResourceHandler(t *testing.T) { } else if len(chunk.SData) < 16 { teardownTest(t, fmt.Errorf("chunk data must be minimum 16 bytes, is %d", len(chunk.SData))) } - startblocknumber := binary.LittleEndian.Uint64(chunk.SData[8:16]) - chunkfrequency := binary.LittleEndian.Uint64(chunk.SData[16:]) + startblocknumber := binary.LittleEndian.Uint64(chunk.SData[:8]) + chunkfrequency := binary.LittleEndian.Uint64(chunk.SData[8:]) if startblocknumber != backend.blocknumber { teardownTest(t, fmt.Errorf("stored block number %d does not match provided block number %d", startblocknumber, backend.blocknumber)) } @@ -242,11 +213,7 @@ func TestResourceHandler(t *testing.T) { resourcekey := make(map[string]Key) fwdBlocks(int(resourceFrequency/2), backend) data := []byte("blinky") - signature, err = signContent(privkey, data) - if err != nil { - teardownTest(t, err) - } - resourcekey["blinky"], err = rh.Update(domainName, data, signature) + resourcekey["blinky"], err = rh.Update(domainName, data) if err != nil { teardownTest(t, err) } @@ -254,11 +221,7 @@ func TestResourceHandler(t *testing.T) { // update on first period fwdBlocks(int(resourceFrequency/2), backend) data = []byte("pinky") - signature, err = signContent(privkey, data) - if err != nil { - teardownTest(t, err) - } - resourcekey["pinky"], err = rh.Update(domainName, data, signature) + resourcekey["pinky"], err = rh.Update(domainName, data) if err != nil { teardownTest(t, err) } @@ -266,11 +229,7 @@ func TestResourceHandler(t *testing.T) { // update on second period fwdBlocks(int(resourceFrequency), backend) data = []byte("inky") - signature, err = signContent(privkey, data) - if err != nil { - teardownTest(t, err) - } - resourcekey["inky"], err = rh.Update(domainName, data, signature) + resourcekey["inky"], err = rh.Update(domainName, data) if err != nil { teardownTest(t, err) } @@ -278,11 +237,7 @@ func TestResourceHandler(t *testing.T) { // update just after second period fwdBlocks(1, backend) data = []byte("clyde") - signature, err = signContent(privkey, data) - if err != nil { - teardownTest(t, err) - } - resourcekey["clyde"], err = rh.Update(domainName, data, signature) + resourcekey["clyde"], err = rh.Update(domainName, data) if err != nil { teardownTest(t, err) } @@ -293,7 +248,7 @@ func TestResourceHandler(t *testing.T) { // it will match on second iteration startblocknumber + (resourceFrequency * 3) fwdBlocks(int(resourceFrequency*2)-1, backend) - rh2, err := NewResourceHandler(privkey, datadir, &testCloudStore{}, rh.rpcClient, nil) + rh2, err := NewResourceHandler(datadir, &testCloudStore{}, rh.rpcClient, nil) _, err = rh2.LookupLatest(domainName, true) if err != nil { teardownTest(t, err) @@ -310,45 +265,25 @@ func TestResourceHandler(t *testing.T) { teardownTest(t, fmt.Errorf("resource period was %d, expected 3", rh2.resources[domainName].lastPeriod)) } - rsrc, err := NewResource(domainName, startblocknumber, resourceFrequency, rh2.validator.nameHash) - if err != nil { - teardownTest(t, err) - } - err = rh2.SetExternalResource(rsrc, true) - if err != nil { - teardownTest(t, err) - } - - // latest block, latest version - resource, err := rh2.LookupLatest(domainName, false) // if key is specified, refresh is implicit - if err != nil { - teardownTest(t, err) - } - - // check data - if !bytes.Equal(resource.data, []byte("clyde")) { - teardownTest(t, fmt.Errorf("resource data (latest) was %v, expected %v", rh2.resources[domainName].data, []byte("clyde"))) - } - // specific block, latest version - resource, err = rh2.LookupHistorical(domainName, 3, true) + rsrc, err := rh2.LookupHistorical(domainName, 3, true) if err != nil { teardownTest(t, err) } // check data - if !bytes.Equal(resource.data, []byte("clyde")) { + if !bytes.Equal(rsrc.data, []byte("clyde")) { teardownTest(t, fmt.Errorf("resource data (historical) was %v, expected %v", rh2.resources[domainName].data, []byte("clyde"))) } // specific block, specific version - resource, err = rh2.LookupVersion(domainName, 3, 1, true) + rsrc, err = rh2.LookupVersion(domainName, 3, 1, true) if err != nil { teardownTest(t, err) } // check data - if !bytes.Equal(resource.data, []byte("inky")) { + if !bytes.Equal(rsrc.data, []byte("inky")) { teardownTest(t, fmt.Errorf("resource data (historical) was %v, expected %v", rh2.resources[domainName].data, []byte("inky"))) } teardownTest(t, nil) @@ -358,21 +293,15 @@ func TestResourceHandler(t *testing.T) { // create ENS enabled resource update, with and without valid owner func TestResourceENSOwner(t *testing.T) { - // privkey for signing updates - privkey, err := crypto.GenerateKey() + // signer containing private key + signer, err := newTestSigner() if err != nil { - return - } - - // privkey for checking wrong owner - privkeytwo, err := crypto.GenerateKey() - if err != nil { - return + t.Fatal(err) } // ens address and transact options - addr := crypto.PubkeyToAddress(privkey.PublicKey) - transactOpts := bind.NewKeyedTransactor(privkey) + addr := crypto.PubkeyToAddress(signer.privKey.PublicKey) + transactOpts := bind.NewKeyedTransactor(signer.privKey) // set up ENS sim domainparts := strings.Split(domainName, ".") @@ -381,40 +310,37 @@ func TestResourceENSOwner(t *testing.T) { t.Fatal(err) } - validator, err := NewENSValidator(contractAddr, contractbackend, transactOpts) + validator, err := NewENSValidator(contractAddr, contractbackend, transactOpts, signer.signContent) if err != nil { t.Fatal(err) } // set up rpc and create resourcehandler with ENS sim backend - rh, _, err, teardownTest := setupTest(privkey, contractbackend, validator) + rh, _, _, teardownTest, err := setupTest(contractbackend, validator) if err != nil { teardownTest(t, err) } - signature, err := signContent(privkey, []byte(domainName)) - if err != nil { - teardownTest(t, err) - } // create new resource when we are owner = ok - _, err = rh.NewResource(domainName, 42, signature) + _, err = rh.NewResource(domainName, resourceFrequency, true) if err != nil { teardownTest(t, fmt.Errorf("Create resource fail: %v", err)) } data := []byte("foo") - signature, err = signContent(privkey, data) - // update resource when we are owner = ok - _, err = rh.Update(domainName, data, signature) + _, err = rh.Update(domainName, data) if err != nil { teardownTest(t, fmt.Errorf("Update resource fail: %v", err)) } - // create new resource when we are NOT owner = !ok - signaturetwo, err := signContent(privkeytwo, data) // update resource when we are owner = ok - _, err = rh.Update(domainName, data, signaturetwo) + signertwo, err := newTestSigner() + if err != nil { + teardownTest(t, err) + } + rh.validator.(*ENSValidator).signFunc = signertwo.signContent + _, err = rh.Update(domainName, data) if err == nil { teardownTest(t, fmt.Errorf("Expected resource update fail due to owner mismatch")) } @@ -430,7 +356,7 @@ func fwdBlocks(count int, backend *fakeBackend) { } // create rpc and resourcehandler -func setupTest(privkey *ecdsa.PrivateKey, contractbackend bind.ContractBackend, validator ResourceValidator) (rh *ResourceHandler, datadir string, err error, teardown func(*testing.T, error)) { +func setupTest(contractbackend bind.ContractBackend, validator ResourceValidator) (rh *ResourceHandler, datadir string, signer *testSigner, teardown func(*testing.T, error), err error) { var fsClean func() var rpcClean func() @@ -443,6 +369,15 @@ func setupTest(privkey *ecdsa.PrivateKey, contractbackend bind.ContractBackend, } } + if validator == nil { + // create a new signer, which creates the private key + signer, err = newTestSigner() + if err != nil { + return + } + validator = NewGenericValidator(testHashFunc, signer.signContent) + } + // temp datadir datadir, err = ioutil.TempDir("", "rh") if err != nil { @@ -480,8 +415,7 @@ func setupTest(privkey *ecdsa.PrivateKey, contractbackend bind.ContractBackend, return } - // choose if with ens or not - rh, err = NewResourceHandler(privkey, datadir, &testCloudStore{}, rpcclient, validator) + rh, err = NewResourceHandler(datadir, &testCloudStore{}, rpcclient, validator) teardown = func(t *testing.T, err error) { cleanF() if err != nil { @@ -499,12 +433,12 @@ func setupENS(addr common.Address, transactOpts *bind.TransactOpts, sub string, var tophash [32]byte var subhash [32]byte - hasher.Reset() - hasher.Write([]byte(top)) - copy(tophash[:], hasher.Sum(nil)) - hasher.Reset() - hasher.Write([]byte(sub)) - copy(subhash[:], hasher.Sum(nil)) + testHasher.Reset() + testHasher.Write([]byte(top)) + copy(tophash[:], testHasher.Sum(nil)) + testHasher.Reset() + testHasher.Write([]byte(sub)) + copy(subhash[:], testHasher.Sum(nil)) // initialize contract backend and deploy contractBackend := &fakeBackend{ @@ -535,17 +469,35 @@ func setupENS(addr common.Address, transactOpts *bind.TransactOpts, sub string, return contractAddress, contractBackend, nil } -func signContent(privKey *ecdsa.PrivateKey, data []byte) (Signature, error) { - hasher.Reset() - hasher.Write(data) - datahash := hasher.Sum(nil) +func testHashFunc(name string) common.Hash { + testHasher.Reset() + testHasher.Write([]byte(name)) + return common.BytesToHash(testHasher.Sum(nil)) +} - signaturebytes, err := crypto.Sign(datahash, privKey) +type testSigner struct { + privKey *ecdsa.PrivateKey + hasher SwarmHash +} + +func newTestSigner() (*testSigner, error) { + privKey, err := crypto.GenerateKey() if err != nil { - return [signatureLength]byte{}, err + return nil, err } - signature, err := NewSignature(signaturebytes) - return signature, err + return &testSigner{ + privKey: privKey, + hasher: testHasher, + }, nil +} + +func (self *testSigner) signContent(data common.Hash) (signature Signature, err error) { + signaturebytes, err := crypto.Sign(data.Bytes(), self.privKey) + if err != nil { + return + } + signature, err = bytesToSignature(signaturebytes) + return } type testCloudStore struct { From c051bec50536390d4ed304644fdae1ad93b4f73a Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 18 Jan 2018 01:35:26 +0100 Subject: [PATCH 042/107] swarm/storage: Add test for reverse metadata retrieval --- swarm/storage/resource.go | 9 ---- swarm/storage/resource_test.go | 88 ++++++++++------------------------ 2 files changed, 25 insertions(+), 72 deletions(-) diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 04e8d06ac4..51010d7ca4 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -29,15 +29,6 @@ var emptySignature Signature type SignFunc func(common.Hash) (Signature, error) -func bytesToSignature(b []byte) (Signature, error) { - var s Signature - if len(b) != signatureLength { - return [signatureLength]byte{}, fmt.Errorf("Must be %d bytes", signatureLength) - } - copy(s[:], b) - return s, nil -} - // Encapsulates an actual resource update. When synced it contains the most recent // version of the resource update data. type resource struct { diff --git a/swarm/storage/resource_test.go b/swarm/storage/resource_test.go index 2a4f4ca4d5..f645522054 100644 --- a/swarm/storage/resource_test.go +++ b/swarm/storage/resource_test.go @@ -65,7 +65,10 @@ func (r *FakeRPC) BlockNumber() (string, error) { } // check that signature address matches update signer address -func TestResourceSignature(t *testing.T) { +func TestResourceReverse(t *testing.T) { + + period := uint32(4) + version := uint32(2) // set up rpc and create resourcehandler rh, _, signer, teardownTest, err := setupTest(nil, nil) @@ -80,7 +83,7 @@ func TestResourceSignature(t *testing.T) { } // generate a hash for block 4200 version 1 - key := rh.resourceHash(1, 1, rh.validator.nameHash(validname)) + key := rh.resourceHash(period, version, rh.validator.nameHash(validname)) // generate some bogus data for the chunk and sign it data := make([]byte, 8) @@ -96,13 +99,11 @@ func TestResourceSignature(t *testing.T) { teardownTest(t, err) } - chunk := newUpdateChunk(key, sig, 1, 1, validname, data) - - log.Warn("key", "chunk", chunk.Key, "real", key) + chunk := newUpdateChunk(key, sig, period, version, validname, data) // check that we can recover the owner account from the update chunk's signature - checksig, _, _, _, newdata, err := parseUpdate(chunk.SData) - checkdigest := rh.keyDataHash(chunk.Key, newdata) + checksig, checkperiod, checkversion, checkname, checkdata, err := parseUpdate(chunk.SData) + checkdigest := rh.keyDataHash(chunk.Key, checkdata) recoveredaddress, err := getAddressFromDataSig(checkdigest, checksig) if err != nil { teardownTest(t, err) @@ -112,64 +113,25 @@ func TestResourceSignature(t *testing.T) { if recoveredaddress != originaladdress { teardownTest(t, fmt.Errorf("addresses dont match: %x != %x", originaladdress, recoveredaddress)) } + + if !bytes.Equal(key[:], chunk.Key[:]) { + teardownTest(t, fmt.Errorf("Expected chunk key '%x', was '%x'", key, chunk.Key)) + } + if period != checkperiod { + teardownTest(t, fmt.Errorf("Expected period '%d', was '%d'", period, checkperiod)) + } + if version != checkversion { + teardownTest(t, fmt.Errorf("Expected version '%d', was '%d'", version, checkversion)) + } + if validname != checkname { + teardownTest(t, fmt.Errorf("Expected name '%s', was '%s'", validname, checkname)) + } + if !bytes.Equal(data, checkdata) { + teardownTest(t, fmt.Errorf("Expectedn data '%x', was '%x'", data, checkdata)) + } teardownTest(t, nil) } -// -//// determine resource update metadata from chunk data -//func TestResourceReverseLookup(t *testing.T) { -// -// // make fake backend, set up rpc and create resourcehandler -// backend := &fakeBackend{ -// blocknumber: startBlock, -// } -// rh, _, _, teardownTest, err := setupTest(backend, nil) -// if err != nil { -// teardownTest(t, err) -// } -// -// rsrc, err := rh.NewResource(domainName, resourceFrequency, false) -// if err != nil { -// teardownTest(t, err) -// } -// -// // update data -// fwdBlocks(int(resourceFrequency+1), backend) -// data := []byte("foo") -// resourcekey, err := rh.Update(domainName, data) -// if err != nil { -// teardownTest(t, err) -// } -// chunk, err := rh.ChunkStore.(*resourceChunkStore).localStore.(*LocalStore).memStore.Get(Key(resourcekey)) -// if err != nil { -// teardownTest(t, err) -// } -// -// // check if data after header length offset is as expected -// headerlength := binary.LittleEndian.Uint16(chunk.SData[signatureLength : signatureLength+2]) -// if !bytes.Equal(chunk.SData[signatureLength+headerlength+2:], data) { -// teardownTest(t, fmt.Errorf("Expected chunk data with header length %d (pos %d) to match %x, but was %x", headerlength, signatureLength+headerlength+2, data, chunk.SData[signatureLength+headerlength+2:])) -// } -// -// // get name, period, version from chunk and check -// _, revperiod, revversion, revname, revdata, err := parseUpdate(chunk.SData[signatureLength:]) -// -// //if !bytes.Equal(revname, rsrc.nameHash.Bytes()) { -// if revname == rsrc.name { -// teardownTest(t, fmt.Errorf("Expected retrieved name from chunk data to be '%x', was '%x'", rsrc.nameHash.Bytes(), revname)) -// } -// if !bytes.Equal(revdata, data) { -// teardownTest(t, fmt.Errorf("Expected retrieved data from chunk data to be '%x', was '%x'", data, revdata)) -// } -// -// if revperiod != 2 { -// teardownTest(t, fmt.Errorf("Expected retrieved period from chunk data to be 1, was %d", revperiod)) -// } -// if revversion != 1 { -// teardownTest(t, fmt.Errorf("Expected retrieved version from chunk data to be 1, was %d", revversion)) -// } -//} -// // make updates and retrieve them based on periods and versions func TestResourceHandler(t *testing.T) { @@ -496,7 +458,7 @@ func (self *testSigner) signContent(data common.Hash) (signature Signature, err if err != nil { return } - signature, err = bytesToSignature(signaturebytes) + copy(signature[:], signaturebytes) return } From 1b81fb85c29d2f7c4803622b0bb95f4194e5131c Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 18 Jan 2018 03:19:29 +0100 Subject: [PATCH 043/107] swarm/storage: Remove signatures from non-validated resources --- swarm/storage/resource.go | 319 +++++++++++++++++++-------------- swarm/storage/resource_ens.go | 34 +--- swarm/storage/resource_test.go | 122 +++++++------ 3 files changed, 260 insertions(+), 215 deletions(-) diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 51010d7ca4..342727ca5b 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -25,14 +25,14 @@ const ( type Signature [signatureLength]byte -var emptySignature Signature - type SignFunc func(common.Hash) (Signature, error) -// Encapsulates an actual resource update. When synced it contains the most recent +type nameHashFunc func(string) common.Hash + +// Encapsulates an specific resource update. When synced it contains the most recent // version of the resource update data. type resource struct { - name string + name *string nameHash common.Hash startBlock uint64 lastPeriod uint32 @@ -47,6 +47,14 @@ func (r *resource) isSynced() bool { return !r.updated.IsZero() } +// Implement to activate validation of resource updates +// Specifically signing data and verification of signatures +type ResourceValidator interface { + checkAccess(string, common.Address) (bool, error) + nameHash(string) common.Hash // nameHashFunc + sign(common.Hash) (Signature, error) // SignFunc +} + // Mutable resource is an entity which allows updates to a resource // without resorting to ENS on each update. // The update scheme is built on swarm chunks with chunk keys following @@ -56,8 +64,9 @@ func (r *resource) isSynced() bool { // expressed in terms of number of blocks. // // The root entry of a mutable resource is tied to a unique identifier, -// typically - but not necessarily - an ens name. It also contains the -// block number when the resource update was first registered, and +// typically - but not necessarily - an ens name. The identifier must be +// an valid IDNA string. It also contains the block number +// when the resource update was first registered, and // the block frequency with which the resource will be updated, both of // which are stored as little-endian uint64 values in the database (for a // total of 16 bytes). @@ -68,10 +77,6 @@ func (r *resource) isSynced() bool { // starting at block 4200 with frequency 42 will have updates on block 4242, // 4284, 4326 and so on. // -// The identifier is supplied as a string, but will be IDNA converted and -// passed through the ENS namehash function. Pure ascii identifiers without -// periods will thus merely be hashed. -// // Note that the root entry is not required for the resource update scheme to // work. A normal chunk of the blocknumber/frequency data can also be created, // and pointed to by an external resource (ENS or manifest entry) @@ -79,7 +84,7 @@ func (r *resource) isSynced() bool { // Actual data updates are also made in the form of swarm chunks. The keys // of the updates are the hash of a concatenation of properties as follows: // -// sha256(namehash|period|version) +// sha256(period|version|namehash) // // The period is (currentblock - startblock) / frequency // @@ -91,8 +96,12 @@ func (r *resource) isSynced() bool { // // A lookup agent need only know the identifier name in order to get the versions // -// the chunk data is: sign(resourcedata)|resourcedata -// the resourcedata is: headerlength|period|version|name|data +// the resourcedata is: +// headerlength|period|version|identifier|data +// +// if a validator is active, the chunk data is: +// sign(resourcedata)|resourcedata +// otherwise, the chunk data is the same as the resourcedata // // headerlength is a 16 bit value containing the byte length of period|version|name // period and version are both 32 bit values. name can have arbitrary length @@ -103,13 +112,6 @@ func (r *resource) isSynced() bool { // stored using a separate store, and forwarding/syncing protocols carry per-chunk // flags to tell whether the chunk can be validated or not; if not it is to be // treated as a resource update chunk. - -type ResourceValidator interface { - checkAccess(string, common.Address) (bool, error) - nameHash(string) common.Hash - sign(common.Hash) (Signature, error) // SignFunc -} - type ResourceHandler struct { ChunkStore validator ResourceValidator @@ -118,9 +120,12 @@ type ResourceHandler struct { hashLock sync.Mutex resourceLock sync.RWMutex hasher SwarmHash + nameHash nameHashFunc } // Create or open resource update chunk store +// +// If validator is nil, signature and access validation will be deactivated func NewResourceHandler(datadir string, cloudStore CloudStore, rpcClient *rpc.Client, validator ResourceValidator) (*ResourceHandler, error) { hashfunc := MakeHashFunc(SHA3Hash) @@ -140,18 +145,19 @@ func NewResourceHandler(datadir string, cloudStore CloudStore, rpcClient *rpc.Cl rpcClient: rpcClient, resources: make(map[string]*resource), hasher: hashfunc(), + validator: validator, } - if validator != nil { - rh.validator = validator + if rh.validator != nil { + rh.nameHash = rh.validator.nameHash } else { - rh.validator = NewGenericValidator(func(name string) common.Hash { + rh.nameHash = func(name string) common.Hash { rh.hashLock.Lock() defer rh.hashLock.Unlock() rh.hasher.Reset() rh.hasher.Write([]byte(name)) return common.BytesToHash(rh.hasher.Sum(nil)) - }, nil) + } } return rh, nil @@ -167,26 +173,20 @@ func (self *ResourceHandler) chunkSize() int64 { // The signature data should match the hash of the idna-converted name by the validator's namehash function, NOT the raw name bytes. // // The start block of the resource update will be the actual current block height of the connected network. -func (self *ResourceHandler) NewResource(name string, frequency uint64, verify bool) (*resource, error) { +func (self *ResourceHandler) NewResource(name string, frequency uint64) (*resource, error) { // frequency 0 is invalid if frequency == 0 { return nil, fmt.Errorf("Frequency cannot be 0") } - // must have name - if name == "" { - return nil, fmt.Errorf("Empty name") + if !isSafeName(name) { + return nil, fmt.Errorf("Invalid name: '%s'", name) } - validName, err := toSafeName(name) - if err != nil { - return nil, err - } + nameHash := self.nameHash(name) - nameHash := self.validator.nameHash(validName) - - if verify { + if self.validator != nil { signature, err := self.validator.sign(nameHash) if err != nil { return nil, fmt.Errorf("Sign fail: %v", err) @@ -220,10 +220,10 @@ func (self *ResourceHandler) NewResource(name string, frequency uint64, verify b binary.LittleEndian.PutUint64(val, frequency) copy(chunk.SData[8:], val) self.Put(chunk) - log.Debug("new resource", "name", validName, "key", nameHash, "startBlock", currentblock, "frequency", frequency) + log.Debug("new resource", "name", name, "key", nameHash, "startBlock", currentblock, "frequency", frequency) rsrc := &resource{ - name: validName, + name: &name, nameHash: nameHash, startBlock: currentblock, frequency: frequency, @@ -231,7 +231,7 @@ func (self *ResourceHandler) NewResource(name string, frequency uint64, verify b } self.setResource(name, rsrc) - return self.resources[name], nil + return rsrc, nil } // Searches and retrieves the specific version of the resource update identified by `name` @@ -247,7 +247,7 @@ func (self *ResourceHandler) LookupVersion(name string, period uint32, version u if err != nil { return nil, err } - return self.lookup(rsrc, name, period, version, refresh) + return self.lookup(rsrc, period, version, refresh) } // Retrieves the latest version of the resource update identified by `name` @@ -263,7 +263,7 @@ func (self *ResourceHandler) LookupHistorical(name string, period uint32, refres if err != nil { return nil, err } - return self.lookup(rsrc, name, period, 0, refresh) + return self.lookup(rsrc, period, 0, refresh) } // Retrieves the latest version of the resource update identified by `name` @@ -288,11 +288,11 @@ func (self *ResourceHandler) LookupLatest(name string, refresh bool) (*resource, return nil, err } nextperiod := getNextPeriod(rsrc.startBlock, currentblock, rsrc.frequency) - return self.lookup(rsrc, name, nextperiod, 0, refresh) + return self.lookup(rsrc, nextperiod, 0, refresh) } // base code for public lookup methods -func (self *ResourceHandler) lookup(rsrc *resource, name string, period uint32, version uint32, refresh bool) (*resource, error) { +func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint32, refresh bool) (*resource, error) { if period == 0 { return nil, fmt.Errorf("period must be >0") @@ -312,7 +312,7 @@ func (self *ResourceHandler) lookup(rsrc *resource, name string, period uint32, chunk, err := self.Get(key) if err == nil { if specificversion { - return self.updateResourceIndex(rsrc, chunk, &name) + return self.updateResourceIndex(rsrc, chunk) } // check if we have versions > 1. If a version fails, the previous version is used and returned. log.Trace("rsrc update version 1 found, checking for version updates", "period", period, "key", key) @@ -321,7 +321,7 @@ func (self *ResourceHandler) lookup(rsrc *resource, name string, period uint32, key := self.resourceHash(period, newversion, rsrc.nameHash) newchunk, err := self.Get(key) if err != nil { - return self.updateResourceIndex(rsrc, chunk, &name) + return self.updateResourceIndex(rsrc, chunk) } log.Trace("version update found, checking next", "version", version, "period", period, "key", key) chunk = newchunk @@ -336,19 +336,18 @@ func (self *ResourceHandler) lookup(rsrc *resource, name string, period uint32, // load existing mutable resource into resource struct func (self *ResourceHandler) loadResource(name string, refresh bool) (*resource, error) { + // if the resource is not known to this session we must load it // if refresh is set, we force load - rsrc := self.getResource(name) if rsrc == nil || refresh { rsrc = &resource{} // make sure our name is safe to use - validname, err := toSafeName(name) - if err != nil { - return nil, err + if !isSafeName(name) { + return nil, fmt.Errorf("Invalid name '%s'") } - rsrc.name = validname - rsrc.nameHash = self.validator.nameHash(validname) + rsrc.name = &name + rsrc.nameHash = self.nameHash(name) // get the root info chunk and update the cached value chunk, err := self.Get(Key(rsrc.nameHash[:])) @@ -356,7 +355,7 @@ func (self *ResourceHandler) loadResource(name string, refresh bool) (*resource, return nil, err } - // sanity check for chunk data + // minimum sanity check for chunk data if len(chunk.SData) != indexSize { return nil, fmt.Errorf("Invalid chunk length %d, should be %d", len(chunk.SData), indexSize) } @@ -372,46 +371,58 @@ func (self *ResourceHandler) loadResource(name string, refresh bool) (*resource, } // update mutable resource index map with specified content -func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk, indexname *string) (*resource, error) { +func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk) (*resource, error) { - // update our rsrcs entry map - signature, period, version, name, data, err := parseUpdate(chunk.SData) - if rsrc.name != name { + // retrieve metadata from chunk data and check that it matches this mutable resource + signature, period, version, name, data, err := self.parseUpdate(chunk.SData) + if *rsrc.name != name { return nil, fmt.Errorf("Update belongs to '%s', but have '%s'", name, rsrc.name) } - self.hashLock.Lock() - self.hasher.Reset() - self.hasher.Write(chunk.Key[:]) - self.hasher.Write(data) - digest := self.hasher.Sum(nil) - self.hashLock.Unlock() - _, err = getAddressFromDataSig(common.BytesToHash(digest), signature) - if err != nil { - return nil, fmt.Errorf("Invalid signature: %v", err) + // only check signature if validator is present + if self.validator != nil { + digest := self.keyDataHash(chunk.Key, data) + _, err = getAddressFromDataSig(digest, *signature) + if err != nil { + return nil, fmt.Errorf("Invalid signature: %v", err) + } } + + // update our rsrcs entry map rsrc.lastPeriod = period rsrc.version = version rsrc.updated = time.Now() rsrc.data = make([]byte, len(data)) copy(rsrc.data, data) - log.Debug("Resource synced", "name", rsrc.name, "key", chunk.Key, "period", rsrc.lastPeriod, "version", rsrc.version) - self.setResource(*indexname, rsrc) + log.Debug("Resource synced", "name", *rsrc.name, "key", chunk.Key, "period", rsrc.lastPeriod, "version", rsrc.version) + self.setResource(*rsrc.name, rsrc) return rsrc, nil } -func parseUpdate(chunkdata []byte) (signature Signature, period uint32, version uint32, name string, data []byte, err error) { - copy(signature[:], chunkdata[:signatureLength]) - cursor := signatureLength +// retrieve update metadata from chunk data +// mirrors newUpdateChunk() +func (self *ResourceHandler) parseUpdate(chunkdata []byte) (signature *Signature, period uint32, version uint32, name string, data []byte, err error) { + cursor := 0 + + // omit signatures if we have no validator + var sigoffset int + if self.validator != nil { + signature = &Signature{} + copy(signature[:], chunkdata[:signatureLength]) + sigoffset = signatureLength + cursor = sigoffset + } + headerlength := binary.LittleEndian.Uint16(chunkdata[cursor : cursor+2]) if int(headerlength+2) > len(chunkdata) { - return emptySignature, 0, 0, "", nil, fmt.Errorf("Reported header length %d longer than actual data length %d", headerlength, len(chunkdata)) + err = fmt.Errorf("Reported header length %d longer than actual data length %d", headerlength, len(chunkdata)) + return } cursor += 2 period = binary.LittleEndian.Uint32(chunkdata[cursor : cursor+4]) cursor += 4 version = binary.LittleEndian.Uint32(chunkdata[cursor : cursor+4]) cursor += 4 - namelength := int(headerlength) - cursor + signatureLength + 2 + namelength := int(headerlength) - cursor + sigoffset + 2 name = string(chunkdata[cursor : cursor+namelength]) cursor += namelength data = make([]byte, len(chunkdata)-cursor) @@ -425,16 +436,24 @@ func parseUpdate(chunkdata []byte) (signature Signature, period uint32, version // It is the caller's responsibility to make sure that this data is not stale. // // A resource update cannot span chunks, and thus has max length 4096 -func (self *ResourceHandler) Update(indexname string, data []byte) (Key, error) { +func (self *ResourceHandler) Update(name string, data []byte) (Key, error) { + + var sigoffset int + if self.validator != nil { + sigoffset = signatureLength + } // get the cached information - rsrc := self.getResource(indexname) + rsrc := self.getResource(name) + if rsrc == nil { + return nil, fmt.Errorf("Resource object not in index") + } if !rsrc.isSynced() { return nil, fmt.Errorf("Resource object not in sync") } // an update can be only one chunk long - datalimit := self.chunkSize() - int64(signatureLength-len(self.resources[indexname].name)-8) + datalimit := self.chunkSize() - int64(sigoffset-len(name)-8) if int64(len(data)) > datalimit { return nil, fmt.Errorf("Data overflow: %d / %d bytes", len(data), datalimit) } @@ -449,7 +468,7 @@ func (self *ResourceHandler) Update(indexname string, data []byte) (Key, error) // if we already have an update for this block then increment version // (resource object MUST be in sync for version to be correct) var version uint32 - if self.hasUpdate(indexname, nextperiod) { + if self.hasUpdate(name, nextperiod) { version = rsrc.version } version++ @@ -457,32 +476,36 @@ func (self *ResourceHandler) Update(indexname string, data []byte) (Key, error) // calculate the chunk key key := self.resourceHash(nextperiod, version, rsrc.nameHash) - // sign the data hash with the key - digest := self.keyDataHash(key, data) - signature, err := self.validator.sign(digest) - if err != nil { - return nil, err + var signature *Signature + if self.validator != nil { + // sign the data hash with the key + digest := self.keyDataHash(key, data) + sig, err := self.validator.sign(digest) + if err != nil { + return nil, err + } + signature = &sig + + // get the address of the signer (which also checks that it's a valid signature) + addr, err := getAddressFromDataSig(digest, *signature) + if err != nil { + return nil, fmt.Errorf("Invalid data/signature: %v", err) + } + + // check if the signer has access to update + ok, err := self.validator.checkAccess(name, addr) + if err != nil { + return nil, err + } else if !ok { + return nil, fmt.Errorf("Address %x does not have access to update %s", addr, name) + } } - // get the address of the signer (which also checks that it's a valid signature) - addr, err := getAddressFromDataSig(digest, signature) - if err != nil { - return nil, fmt.Errorf("Invalid data/signature: %v", err) - } - - // check if the signer has access to update - ok, err := self.validator.checkAccess(indexname, addr) - if err != nil { - return nil, err - } else if !ok { - return nil, fmt.Errorf("Address %x does not have access to update %s", addr, indexname) - } - - chunk := newUpdateChunk(key, signature, nextperiod, version, self.resources[indexname].name, data) + chunk := newUpdateChunk(key, signature, nextperiod, version, name, data) // send the chunk self.Put(chunk) - log.Trace("resource update", "name", rsrc.name, "key", key, "currentblock", currentblock, "lastperiod", nextperiod, "version", version, "data", chunk.SData) + log.Trace("resource update", "name", name, "key", key, "currentblock", currentblock, "lastperiod", nextperiod, "version", version, "data", chunk.SData) // update our resources map entry and return the new key rsrc.lastPeriod = nextperiod @@ -492,38 +515,6 @@ func (self *ResourceHandler) Update(indexname string, data []byte) (Key, error) return key, nil } -func newUpdateChunk(key Key, signature Signature, period uint32, version uint32, name string, data []byte) *Chunk { - // create the update chunk - // prepend version and period to allow reverse lookups - headerlength := uint16(len(name) + 4 + 4) - - chunk := NewChunk(key, nil) - chunk.SData = make([]byte, signatureLength+int(headerlength)+2+len(data)) - - cursor := 0 - copy(chunk.SData, signature[:]) - cursor += signatureLength - - // data header length does NOT include the header length prefix bytes themselves - binary.LittleEndian.PutUint16(chunk.SData[cursor:], headerlength) - cursor += 2 - - binary.LittleEndian.PutUint32(chunk.SData[cursor:], period) - cursor += 4 - - binary.LittleEndian.PutUint32(chunk.SData[cursor:], version) - cursor += 4 - - namebytes := []byte(name) - copy(chunk.SData[cursor:], namebytes) - cursor += len(namebytes) - - copy(chunk.SData[cursor:], data) - - chunk.Size = int64(len(chunk.SData)) - return chunk -} - // Closes the datastore. // Always call this at shutdown to avoid data corruption. func (self *ResourceHandler) Close() { @@ -543,10 +534,12 @@ func (self *ResourceHandler) getBlock() (uint64, error) { return strconv.ParseUint(currentblock, 10, 64) } +// Calculate the period index (aka major version number) from a given block number func (self *ResourceHandler) BlockToPeriod(name string, blocknumber uint64) uint32 { return getNextPeriod(self.resources[name].startBlock, blocknumber, self.resources[name].frequency) } +// Calculate the block number from a given period index (aka major version number) func (self *ResourceHandler) PeriodToBlock(name string, period uint32) uint64 { return self.resources[name].startBlock + (uint64(period) * self.resources[name].frequency) } @@ -564,8 +557,9 @@ func (self *ResourceHandler) setResource(name string, rsrc *resource) { self.resources[name] = rsrc } +// used for chunk keys func (self *ResourceHandler) resourceHash(period uint32, version uint32, namehash common.Hash) Key { - // format is: hash(namehash|period|version) + // format is: hash(period|version|namehash) self.hashLock.Lock() defer self.hashLock.Unlock() self.hasher.Reset() @@ -578,6 +572,13 @@ func (self *ResourceHandler) resourceHash(period uint32, version uint32, namehas return self.hasher.Sum(nil) } +func (self *ResourceHandler) hasUpdate(name string, period uint32) bool { + if self.resources[name].lastPeriod == period { + return true + } + return false +} + func getAddressFromDataSig(datahash common.Hash, signature Signature) (common.Address, error) { pub, err := crypto.SigToPub(datahash.Bytes(), signature[:]) if err != nil { @@ -586,8 +587,45 @@ func getAddressFromDataSig(datahash common.Hash, signature Signature) (common.Ad return crypto.PubkeyToAddress(*pub), nil } -func (self *ResourceHandler) hasUpdate(name string, period uint32) bool { - return self.resources[name].lastPeriod == period +// create an update chunk +func newUpdateChunk(key Key, signature *Signature, period uint32, version uint32, name string, data []byte) *Chunk { + + // no signatures if no validator + var sigoffset int + if signature != nil { + sigoffset = signatureLength + } + + // prepend version and period to allow reverse lookups + headerlength := uint16(len(name) + 4 + 4) + + chunk := NewChunk(key, nil) + chunk.SData = make([]byte, sigoffset+int(headerlength)+2+len(data)) + + cursor := 0 + if signature != nil { + copy(chunk.SData, (*signature)[:]) + cursor += signatureLength + } + + // data header length does NOT include the header length prefix bytes themselves + binary.LittleEndian.PutUint16(chunk.SData[cursor:], headerlength) + cursor += 2 + + binary.LittleEndian.PutUint32(chunk.SData[cursor:], period) + cursor += 4 + + binary.LittleEndian.PutUint32(chunk.SData[cursor:], version) + cursor += 4 + + namebytes := []byte(name) + copy(chunk.SData[cursor:], namebytes) + cursor += len(namebytes) + + copy(chunk.SData[cursor:], data) + + chunk.Size = int64(len(chunk.SData)) + return chunk } // \TODO chunkSize is a workaround until the ChunkStore interface exports a method to get the chunk size directly @@ -640,8 +678,7 @@ func getNextPeriod(start uint64, current uint64, frequency uint64) uint32 { return uint32(period + 1) } -func toSafeName(name string) (string, error) { - // make sure our ens identifier is idna safe +func ToSafeName(name string) (string, error) { validname, err := idna.ToASCII(name) if err != nil { return "", err @@ -649,6 +686,22 @@ func toSafeName(name string) (string, error) { return validname, nil } +// check that name identifiers contain valid bytes +func isSafeName(name string) bool { + if name == "" { + return false + } + validname, err := idna.ToASCII(name) + if err != nil { + return false + } + if validname != name { + return false + } + return true +} + +// convenience for creating signature hashes of update data func (self *ResourceHandler) keyDataHash(key Key, data []byte) common.Hash { self.hashLock.Lock() defer self.hashLock.Unlock() diff --git a/swarm/storage/resource_ens.go b/swarm/storage/resource_ens.go index ccd4ed5344..0a4500309d 100644 --- a/swarm/storage/resource_ens.go +++ b/swarm/storage/resource_ens.go @@ -12,9 +12,9 @@ type baseValidator struct { signFunc SignFunc } -func (b *baseValidator) sign(datahash common.Hash) (Signature, error) { +func (b *baseValidator) sign(datahash common.Hash) (signature Signature, err error) { if b.signFunc == nil { - return emptySignature, fmt.Errorf("No signature function") + return signature, fmt.Errorf("No signature function") } return b.signFunc(datahash) } @@ -22,8 +22,7 @@ func (b *baseValidator) sign(datahash common.Hash) (Signature, error) { // ENS validation of mutable resource owners type ENSValidator struct { *baseValidator - api *ens.ENS - hashlength int + api *ens.ENS } func NewENSValidator(contractaddress common.Address, backend bind.ContractBackend, transactOpts *bind.TransactOpts, signFunc SignFunc) (*ENSValidator, error) { @@ -37,7 +36,6 @@ func NewENSValidator(contractaddress common.Address, backend bind.ContractBacken if err != nil { return nil, err } - validator.hashlength = len(ens.EnsNode(dbDirName).Bytes()) return validator, nil } @@ -52,29 +50,3 @@ func (self *ENSValidator) checkAccess(name string, address common.Address) (bool func (self *ENSValidator) nameHash(name string) common.Hash { return ens.EnsNode(name) } - -// Default fallthrough validation of mutable resource ownership -type GenericValidator struct { - *baseValidator - hashFunc func(string) common.Hash - hashlength int -} - -func NewGenericValidator(hashFunc func(string) common.Hash, signFunc SignFunc) *GenericValidator { - return &GenericValidator{ - baseValidator: &baseValidator{ - signFunc: signFunc, - }, - hashFunc: hashFunc, - hashlength: len(hashFunc(dbDirName).Bytes()), - } - -} - -func (self *GenericValidator) checkAccess(name string, address common.Address) (bool, error) { - return true, nil -} - -func (self *GenericValidator) nameHash(name string) common.Hash { - return self.hashFunc(name) -} diff --git a/swarm/storage/resource_test.go b/swarm/storage/resource_test.go index f645522054..ffa0eec758 100644 --- a/swarm/storage/resource_test.go +++ b/swarm/storage/resource_test.go @@ -15,8 +15,6 @@ import ( "testing" "time" - "golang.org/x/net/idna" - "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/common" @@ -35,10 +33,16 @@ var ( resourceFrequency = uint64(42) cleanF func() domainName = "føø.bar" + safeName string ) func init() { + var err error log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) + safeName, err = ToSafeName(domainName) + if err != nil { + panic(err) + } } // simulated backend does not have the blocknumber call @@ -70,20 +74,20 @@ func TestResourceReverse(t *testing.T) { period := uint32(4) version := uint32(2) - // set up rpc and create resourcehandler - rh, _, signer, teardownTest, err := setupTest(nil, nil) + // signer containing private key + signer, err := newTestSigner() if err != nil { - teardownTest(t, err) + t.Fatal(err) } - // create a new resource - validname, err := idna.ToASCII(domainName) + // set up rpc and create resourcehandler + rh, _, _, teardownTest, err := setupTest(nil, newTestValidator(signer.signContent)) if err != nil { teardownTest(t, err) } // generate a hash for block 4200 version 1 - key := rh.resourceHash(period, version, rh.validator.nameHash(validname)) + key := rh.resourceHash(period, version, rh.nameHash(safeName)) // generate some bogus data for the chunk and sign it data := make([]byte, 8) @@ -99,17 +103,18 @@ func TestResourceReverse(t *testing.T) { teardownTest(t, err) } - chunk := newUpdateChunk(key, sig, period, version, validname, data) + chunk := newUpdateChunk(key, &sig, period, version, safeName, data) // check that we can recover the owner account from the update chunk's signature - checksig, checkperiod, checkversion, checkname, checkdata, err := parseUpdate(chunk.SData) + checksig, checkperiod, checkversion, checkname, checkdata, err := rh.parseUpdate(chunk.SData) checkdigest := rh.keyDataHash(chunk.Key, checkdata) - recoveredaddress, err := getAddressFromDataSig(checkdigest, checksig) + recoveredaddress, err := getAddressFromDataSig(checkdigest, *checksig) if err != nil { - teardownTest(t, err) + teardownTest(t, fmt.Errorf("Retrieve address from signature fail: %v", err)) } originaladdress := crypto.PubkeyToAddress(signer.privKey.PublicKey) + // check that the metadata retrieved from the chunk matches what we gave it if recoveredaddress != originaladdress { teardownTest(t, fmt.Errorf("addresses dont match: %x != %x", originaladdress, recoveredaddress)) } @@ -123,8 +128,8 @@ func TestResourceReverse(t *testing.T) { if version != checkversion { teardownTest(t, fmt.Errorf("Expected version '%d', was '%d'", version, checkversion)) } - if validname != checkname { - teardownTest(t, fmt.Errorf("Expected name '%s', was '%s'", validname, checkname)) + if safeName != checkname { + teardownTest(t, fmt.Errorf("Expected name '%s', was '%s'", safeName, checkname)) } if !bytes.Equal(data, checkdata) { teardownTest(t, fmt.Errorf("Expectedn data '%x', was '%x'", data, checkdata)) @@ -145,17 +150,16 @@ func TestResourceHandler(t *testing.T) { } // create a new resource - resourcevalidname, err := idna.ToASCII(domainName) if err != nil { teardownTest(t, err) } - _, err = rh.NewResource(domainName, resourceFrequency, false) + _, err = rh.NewResource(safeName, resourceFrequency) if err != nil { teardownTest(t, err) } // check that the new resource is stored correctly - namehash := rh.validator.nameHash(resourcevalidname) + namehash := rh.nameHash(safeName) chunk, err := rh.ChunkStore.(*resourceChunkStore).localStore.(*LocalStore).memStore.Get(Key(namehash[:])) if err != nil { teardownTest(t, err) @@ -175,7 +179,7 @@ func TestResourceHandler(t *testing.T) { resourcekey := make(map[string]Key) fwdBlocks(int(resourceFrequency/2), backend) data := []byte("blinky") - resourcekey["blinky"], err = rh.Update(domainName, data) + resourcekey["blinky"], err = rh.Update(safeName, data) if err != nil { teardownTest(t, err) } @@ -183,7 +187,7 @@ func TestResourceHandler(t *testing.T) { // update on first period fwdBlocks(int(resourceFrequency/2), backend) data = []byte("pinky") - resourcekey["pinky"], err = rh.Update(domainName, data) + resourcekey["pinky"], err = rh.Update(safeName, data) if err != nil { teardownTest(t, err) } @@ -191,7 +195,7 @@ func TestResourceHandler(t *testing.T) { // update on second period fwdBlocks(int(resourceFrequency), backend) data = []byte("inky") - resourcekey["inky"], err = rh.Update(domainName, data) + resourcekey["inky"], err = rh.Update(safeName, data) if err != nil { teardownTest(t, err) } @@ -199,7 +203,7 @@ func TestResourceHandler(t *testing.T) { // update just after second period fwdBlocks(1, backend) data = []byte("clyde") - resourcekey["clyde"], err = rh.Update(domainName, data) + resourcekey["clyde"], err = rh.Update(safeName, data) if err != nil { teardownTest(t, err) } @@ -211,43 +215,44 @@ func TestResourceHandler(t *testing.T) { fwdBlocks(int(resourceFrequency*2)-1, backend) rh2, err := NewResourceHandler(datadir, &testCloudStore{}, rh.rpcClient, nil) - _, err = rh2.LookupLatest(domainName, true) + _, err = rh2.LookupLatest(safeName, true) if err != nil { teardownTest(t, err) } // last update should be "clyde", version two, blockheight startblocknumber + (resourcefrequency * 3) - if !bytes.Equal(rh2.resources[domainName].data, []byte("clyde")) { - teardownTest(t, fmt.Errorf("resource data was %v, expected %v", rh2.resources[domainName].data, []byte("clyde"))) + if !bytes.Equal(rh2.resources[safeName].data, []byte("clyde")) { + teardownTest(t, fmt.Errorf("resource data was %v, expected %v", rh2.resources[safeName].data, []byte("clyde"))) } - if rh2.resources[domainName].version != 2 { - teardownTest(t, fmt.Errorf("resource version was %d, expected 2", rh2.resources[domainName].version)) + if rh2.resources[safeName].version != 2 { + teardownTest(t, fmt.Errorf("resource version was %d, expected 2", rh2.resources[safeName].version)) } - if rh2.resources[domainName].lastPeriod != 3 { - teardownTest(t, fmt.Errorf("resource period was %d, expected 3", rh2.resources[domainName].lastPeriod)) + if rh2.resources[safeName].lastPeriod != 3 { + teardownTest(t, fmt.Errorf("resource period was %d, expected 3", rh2.resources[safeName].lastPeriod)) } + log.Debug("Latest lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data) // specific block, latest version - rsrc, err := rh2.LookupHistorical(domainName, 3, true) + rsrc, err := rh2.LookupHistorical(safeName, 3, true) if err != nil { teardownTest(t, err) } - // check data if !bytes.Equal(rsrc.data, []byte("clyde")) { teardownTest(t, fmt.Errorf("resource data (historical) was %v, expected %v", rh2.resources[domainName].data, []byte("clyde"))) } + log.Debug("Historical lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data) // specific block, specific version - rsrc, err = rh2.LookupVersion(domainName, 3, 1, true) + rsrc, err = rh2.LookupVersion(safeName, 3, 1, true) if err != nil { teardownTest(t, err) } - // check data if !bytes.Equal(rsrc.data, []byte("inky")) { teardownTest(t, fmt.Errorf("resource data (historical) was %v, expected %v", rh2.resources[domainName].data, []byte("inky"))) } + log.Debug("Specific version lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data) teardownTest(t, nil) } @@ -266,7 +271,7 @@ func TestResourceENSOwner(t *testing.T) { transactOpts := bind.NewKeyedTransactor(signer.privKey) // set up ENS sim - domainparts := strings.Split(domainName, ".") + domainparts := strings.Split(safeName, ".") contractAddr, contractbackend, err := setupENS(addr, transactOpts, domainparts[0], domainparts[1]) if err != nil { t.Fatal(err) @@ -284,14 +289,14 @@ func TestResourceENSOwner(t *testing.T) { } // create new resource when we are owner = ok - _, err = rh.NewResource(domainName, resourceFrequency, true) + _, err = rh.NewResource(safeName, resourceFrequency) if err != nil { teardownTest(t, fmt.Errorf("Create resource fail: %v", err)) } data := []byte("foo") // update resource when we are owner = ok - _, err = rh.Update(domainName, data) + _, err = rh.Update(safeName, data) if err != nil { teardownTest(t, fmt.Errorf("Update resource fail: %v", err)) } @@ -302,7 +307,7 @@ func TestResourceENSOwner(t *testing.T) { teardownTest(t, err) } rh.validator.(*ENSValidator).signFunc = signertwo.signContent - _, err = rh.Update(domainName, data) + _, err = rh.Update(safeName, data) if err == nil { teardownTest(t, fmt.Errorf("Expected resource update fail due to owner mismatch")) } @@ -331,15 +336,6 @@ func setupTest(contractbackend bind.ContractBackend, validator ResourceValidator } } - if validator == nil { - // create a new signer, which creates the private key - signer, err = newTestSigner() - if err != nil { - return - } - validator = NewGenericValidator(testHashFunc, signer.signContent) - } - // temp datadir datadir, err = ioutil.TempDir("", "rh") if err != nil { @@ -431,12 +427,7 @@ func setupENS(addr common.Address, transactOpts *bind.TransactOpts, sub string, return contractAddress, contractBackend, nil } -func testHashFunc(name string) common.Hash { - testHasher.Reset() - testHasher.Write([]byte(name)) - return common.BytesToHash(testHasher.Sum(nil)) -} - +// implementation of an external signer to pass to validator type testSigner struct { privKey *ecdsa.PrivateKey hasher SwarmHash @@ -453,6 +444,7 @@ func newTestSigner() (*testSigner, error) { }, nil } +// matches the SignFunc type func (self *testSigner) signContent(data common.Hash) (signature Signature, err error) { signaturebytes, err := crypto.Sign(data.Bytes(), self.privKey) if err != nil { @@ -473,3 +465,31 @@ func (c *testCloudStore) Deliver(*Chunk) { func (c *testCloudStore) Retrieve(*Chunk) { } + +// Default fallthrough validation of mutable resource ownership +type testValidator struct { + *baseValidator + hashFunc func(string) common.Hash +} + +func newTestValidator(signFunc SignFunc) *testValidator { + return &testValidator{ + baseValidator: &baseValidator{ + signFunc: signFunc, + }, + hashFunc: func(name string) common.Hash { + testHasher.Reset() + testHasher.Write([]byte(name)) + return common.BytesToHash(testHasher.Sum(nil)) + }, + } + +} + +func (self *testValidator) checkAccess(name string, address common.Address) (bool, error) { + return true, nil +} + +func (self *testValidator) nameHash(name string) common.Hash { + return self.hashFunc(name) +} From d224cd119cd85b0bde00844e6595c04923242d0b Mon Sep 17 00:00:00 2001 From: lash Date: Sat, 20 Jan 2018 18:35:21 +0100 Subject: [PATCH 044/107] swarm/storage: Add store timeout --- swarm/storage/resource.go | 45 ++++++++++++++++++++++------------ swarm/storage/resource_test.go | 3 --- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 342727ca5b..3d6c653716 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -17,10 +17,11 @@ import ( ) const ( - signatureLength = 65 - indexSize = 16 - dbDirName = "resource" - chunkSize = 4096 // temporary until we implement DPA in the resourcehandler + signatureLength = 65 + indexSize = 16 + dbDirName = "resource" + chunkSize = 4096 // temporary until we implement DPA in the resourcehandler + defaultStoreTimeout = 4000 * time.Millisecond ) type Signature [signatureLength]byte @@ -121,6 +122,7 @@ type ResourceHandler struct { resourceLock sync.RWMutex hasher SwarmHash nameHash nameHashFunc + storeTimeout time.Duration } // Create or open resource update chunk store @@ -141,11 +143,12 @@ func NewResourceHandler(datadir string, cloudStore CloudStore, rpcClient *rpc.Cl } rh := &ResourceHandler{ - ChunkStore: newResourceChunkStore(path, hashfunc, localStore, cloudStore), - rpcClient: rpcClient, - resources: make(map[string]*resource), - hasher: hashfunc(), - validator: validator, + ChunkStore: newResourceChunkStore(path, hashfunc, localStore, cloudStore), + rpcClient: rpcClient, + resources: make(map[string]*resource), + hasher: hashfunc(), + validator: validator, + storeTimeout: defaultStoreTimeout, } if rh.validator != nil { @@ -344,7 +347,7 @@ func (self *ResourceHandler) loadResource(name string, refresh bool) (*resource, rsrc = &resource{} // make sure our name is safe to use if !isSafeName(name) { - return nil, fmt.Errorf("Invalid name '%s'") + return nil, fmt.Errorf("Invalid name '%s'", name) } rsrc.name = &name rsrc.nameHash = self.nameHash(name) @@ -376,7 +379,7 @@ func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk) ( // retrieve metadata from chunk data and check that it matches this mutable resource signature, period, version, name, data, err := self.parseUpdate(chunk.SData) if *rsrc.name != name { - return nil, fmt.Errorf("Update belongs to '%s', but have '%s'", name, rsrc.name) + return nil, fmt.Errorf("Update belongs to '%s', but have '%s'", name, *rsrc.name) } // only check signature if validator is present if self.validator != nil { @@ -400,9 +403,10 @@ func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk) ( // retrieve update metadata from chunk data // mirrors newUpdateChunk() -func (self *ResourceHandler) parseUpdate(chunkdata []byte) (signature *Signature, period uint32, version uint32, name string, data []byte, err error) { +func (self *ResourceHandler) parseUpdate(chunkdata []byte) (*Signature, uint32, uint32, string, []byte, error) { + var err error cursor := 0 - + var signature *Signature // omit signatures if we have no validator var sigoffset int if self.validator != nil { @@ -415,8 +419,13 @@ func (self *ResourceHandler) parseUpdate(chunkdata []byte) (signature *Signature headerlength := binary.LittleEndian.Uint16(chunkdata[cursor : cursor+2]) if int(headerlength+2) > len(chunkdata) { err = fmt.Errorf("Reported header length %d longer than actual data length %d", headerlength, len(chunkdata)) - return + return nil, 0, 0, "", nil, err } + + var period uint32 + var version uint32 + var name string + var data []byte cursor += 2 period = binary.LittleEndian.Uint32(chunkdata[cursor : cursor+4]) cursor += 4 @@ -427,7 +436,7 @@ func (self *ResourceHandler) parseUpdate(chunkdata []byte) (signature *Signature cursor += namelength data = make([]byte, len(chunkdata)-cursor) copy(data, chunkdata[cursor:]) - return + return signature, period, version, name, data, err } // Adds an actual data update @@ -505,6 +514,12 @@ func (self *ResourceHandler) Update(name string, data []byte) (Key, error) { // send the chunk self.Put(chunk) + timeout := time.NewTimer(self.storeTimeout) + select { + case <-chunk.dbStored: + case <-timeout.C: + + } log.Trace("resource update", "name", name, "key", key, "currentblock", currentblock, "lastperiod", nextperiod, "version", version, "data", chunk.SData) // update our resources map entry and return the new key diff --git a/swarm/storage/resource_test.go b/swarm/storage/resource_test.go index ffa0eec758..d130709428 100644 --- a/swarm/storage/resource_test.go +++ b/swarm/storage/resource_test.go @@ -150,9 +150,6 @@ func TestResourceHandler(t *testing.T) { } // create a new resource - if err != nil { - teardownTest(t, err) - } _, err = rh.NewResource(safeName, resourceFrequency) if err != nil { teardownTest(t, err) From 84be00915416872b6136b06a6f7b12b095585e36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 22 Jan 2018 14:07:47 +0200 Subject: [PATCH 045/107] core: sorted reorg insertion order for proper head header updating --- core/blockchain.go | 13 ++++++----- core/blockchain_test.go | 48 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 737fbe3eee..f886ffe4ed 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -465,7 +465,7 @@ func (bc *BlockChain) insert(block *types.Block) { } bc.currentBlock = block - // If the block is better than out head or is on a different chain, force update heads + // If the block is better than our head or is on a different chain, force update heads if updateHeads { bc.hc.SetCurrentHeader(block.Header()) @@ -1140,18 +1140,17 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { } else { log.Error("Impossible reorg, please file an issue", "oldnum", oldBlock.Number(), "oldhash", oldBlock.Hash(), "newnum", newBlock.Number(), "newhash", newBlock.Hash()) } + // Insert the new chain, taking care of the proper incremental order var addedTxs types.Transactions - // insert blocks. Order does not matter. Last block will be written in ImportChain itself which creates the new head properly - for _, block := range newChain { + for i := len(newChain) - 1; i >= 0; i-- { // insert the block in the canonical way, re-writing history - bc.insert(block) + bc.insert(newChain[i]) // write lookup entries for hash based transaction/receipt searches - if err := WriteTxLookupEntries(bc.chainDb, block); err != nil { + if err := WriteTxLookupEntries(bc.chainDb, newChain[i]); err != nil { return err } - addedTxs = append(addedTxs, block.Transactions()...) + addedTxs = append(addedTxs, newChain[i].Transactions()...) } - // calculate the difference between deleted and added transactions diff := types.TxDifference(deletedTxs, addedTxs) // When transactions get deleted from the database that means the diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 26c816027f..cbde3bcd2d 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -1197,3 +1197,51 @@ func TestEIP161AccountRemoval(t *testing.T) { t.Error("account should not exist") } } + +// This is a regression test (i.e. as weird as it is, don't delete it ever), which +// tests that under weird reorg conditions the blockchain and its internal header- +// chain return the same latest block/header. +// +// https://github.com/ethereum/go-ethereum/pull/15941 +func TestBlockchainHeaderchainReorgConsistency(t *testing.T) { + // Generate a canonical chain to act as the main dataset + engine := ethash.NewFaker() + + db, _ := ethdb.NewMemDatabase() + genesis := new(Genesis).MustCommit(db) + blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) + + // Generate a bunch of fork blocks, each side forking from the canonical chain + forks := make([]*types.Block, len(blocks)) + for i := 0; i < len(forks); i++ { + parent := genesis + if i > 0 { + parent = blocks[i-1] + } + fork, _ := GenerateChain(params.TestChainConfig, parent, engine, db, 1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) }) + forks[i] = fork[0] + } + // Import the canonical and fork chain side by side, verifying the current block + // and current header consistency + diskdb, _ := ethdb.NewMemDatabase() + new(Genesis).MustCommit(diskdb) + + chain, err := NewBlockChain(diskdb, params.TestChainConfig, engine, vm.Config{}) + if err != nil { + t.Fatalf("failed to create tester chain: %v", err) + } + for i := 0; i < len(blocks); i++ { + if _, err := chain.InsertChain(blocks[i : i+1]); err != nil { + t.Fatalf("block %d: failed to insert into chain: %v", i, err) + } + if chain.CurrentBlock().Hash() != chain.CurrentHeader().Hash() { + t.Errorf("block %d: current block/header mismatch: block #%d [%x…], header #%d [%x…]", i, chain.CurrentBlock().Number(), chain.CurrentBlock().Hash().Bytes()[:4], chain.CurrentHeader().Number, chain.CurrentHeader().Hash().Bytes()[:4]) + } + if _, err := chain.InsertChain(forks[i : i+1]); err != nil { + t.Fatalf(" fork %d: failed to insert into chain: %v", i, err) + } + if chain.CurrentBlock().Hash() != chain.CurrentHeader().Hash() { + t.Errorf(" fork %d: current block/header mismatch: block #%d [%x…], header #%d [%x…]", i, chain.CurrentBlock().Number(), chain.CurrentBlock().Hash().Bytes()[:4], chain.CurrentHeader().Number, chain.CurrentHeader().Hash().Bytes()[:4]) + } + } +} From 92580d69d3156e5d2f0788eb5d48664cc3fb0ef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Mon, 22 Jan 2018 13:38:34 +0100 Subject: [PATCH 046/107] p2p, p2p/discover, p2p/discv5: implement UDP port sharing (#15200) This commit affects p2p/discv5 "topic discovery" by running it on the same UDP port where the old discovery works. This is realized by giving an "unhandled" packet channel to the old v4 discovery packet handler where all invalid packets are sent. These packets are then processed by v5. v5 packets are always invalid when interpreted by v4 and vice versa. This is ensured by adding one to the first byte of the packet hash in v5 packets. DiscoveryV5Bootnodes is also changed to point to new bootnodes that are implementing the changed packet format with modified hash. Existing and new v5 bootnodes are both running on different ports ATM. --- cmd/bootnode/main.go | 25 ++++++++++++-- cmd/faucet/faucet.go | 1 - cmd/utils/flags.go | 10 ------ les/backend.go | 5 ++- les/protocol.go | 5 +-- les/server.go | 4 +-- mobile/geth.go | 1 - node/defaults.go | 7 ++-- p2p/discover/udp.go | 44 +++++++++++------------ p2p/discover/udp_test.go | 3 +- p2p/discv5/net.go | 3 +- p2p/discv5/net_test.go | 2 +- p2p/discv5/sim_test.go | 2 +- p2p/discv5/ticket.go | 2 +- p2p/discv5/udp.go | 46 ++++++++++-------------- p2p/server.go | 75 +++++++++++++++++++++++++++++++++++++--- params/bootnodes.go | 6 ++-- 17 files changed, 150 insertions(+), 91 deletions(-) diff --git a/cmd/bootnode/main.go b/cmd/bootnode/main.go index e1734d89ac..ecfc6fc24e 100644 --- a/cmd/bootnode/main.go +++ b/cmd/bootnode/main.go @@ -21,6 +21,7 @@ import ( "crypto/ecdsa" "flag" "fmt" + "net" "os" "github.com/ethereum/go-ethereum/cmd/utils" @@ -96,12 +97,32 @@ func main() { } } + addr, err := net.ResolveUDPAddr("udp", *listenAddr) + if err != nil { + utils.Fatalf("-ResolveUDPAddr: %v", err) + } + conn, err := net.ListenUDP("udp", addr) + if err != nil { + utils.Fatalf("-ListenUDP: %v", err) + } + + realaddr := conn.LocalAddr().(*net.UDPAddr) + if natm != nil { + if !realaddr.IP.IsLoopback() { + go nat.Map(natm, nil, "udp", realaddr.Port, realaddr.Port, "ethereum discovery") + } + // TODO: react to external IP changes over time. + if ext, err := natm.ExternalIP(); err == nil { + realaddr = &net.UDPAddr{IP: ext, Port: realaddr.Port} + } + } + if *runv5 { - if _, err := discv5.ListenUDP(nodeKey, *listenAddr, natm, "", restrictList); err != nil { + if _, err := discv5.ListenUDP(nodeKey, conn, realaddr, "", restrictList); err != nil { utils.Fatalf("%v", err) } } else { - if _, err := discover.ListenUDP(nodeKey, *listenAddr, natm, "", restrictList); err != nil { + if _, err := discover.ListenUDP(nodeKey, conn, realaddr, nil, "", restrictList); err != nil { utils.Fatalf("%v", err) } } diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index e92924fc9c..99527f9d1e 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -223,7 +223,6 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network u NoDiscovery: true, DiscoveryV5: true, ListenAddr: fmt.Sprintf(":%d", port), - DiscoveryV5Addr: fmt.Sprintf(":%d", port+1), MaxPeers: 25, BootstrapNodesV5: enodes, }, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 3766ea4a60..89dcd230c4 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -636,14 +636,6 @@ func setListenAddress(ctx *cli.Context, cfg *p2p.Config) { } } -// setDiscoveryV5Address creates a UDP listening address string from set command -// line flags for the V5 discovery protocol. -func setDiscoveryV5Address(ctx *cli.Context, cfg *p2p.Config) { - if ctx.GlobalIsSet(ListenPortFlag.Name) { - cfg.DiscoveryV5Addr = fmt.Sprintf(":%d", ctx.GlobalInt(ListenPortFlag.Name)+1) - } -} - // setNAT creates a port mapper from command line flags. func setNAT(ctx *cli.Context, cfg *p2p.Config) { if ctx.GlobalIsSet(NATFlag.Name) { @@ -794,7 +786,6 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) { setNodeKey(ctx, cfg) setNAT(ctx, cfg) setListenAddress(ctx, cfg) - setDiscoveryV5Address(ctx, cfg) setBootstrapNodes(ctx, cfg) setBootstrapNodesV5(ctx, cfg) @@ -830,7 +821,6 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) { // --dev mode can't use p2p networking. cfg.MaxPeers = 0 cfg.ListenAddr = ":0" - cfg.DiscoveryV5Addr = ":0" cfg.NoDiscovery = true cfg.DiscoveryV5 = false } diff --git a/les/backend.go b/les/backend.go index 7180b81d76..798e44e85c 100644 --- a/les/backend.go +++ b/les/backend.go @@ -221,9 +221,8 @@ func (s *LightEthereum) Start(srvr *p2p.Server) error { s.startBloomHandlers() log.Warn("Light client mode is an experimental feature") s.netRPCService = ethapi.NewPublicNetAPI(srvr, s.networkId) - // search the topic belonging to the oldest supported protocol because - // servers always advertise all supported protocols - protocolVersion := ClientProtocolVersions[len(ClientProtocolVersions)-1] + // clients are searching for the first advertised protocol in the list + protocolVersion := AdvertiseProtocolVersions[0] s.serverPool.start(srvr, lesTopic(s.blockchain.Genesis().Hash(), protocolVersion)) s.protocolManager.Start() return nil diff --git a/les/protocol.go b/les/protocol.go index 05e6654d6e..6a7354d1c2 100644 --- a/les/protocol.go +++ b/les/protocol.go @@ -41,8 +41,9 @@ const ( // Supported versions of the les protocol (first is primary) var ( - ClientProtocolVersions = []uint{lpv2, lpv1} - ServerProtocolVersions = []uint{lpv2, lpv1} + ClientProtocolVersions = []uint{lpv2, lpv1} + ServerProtocolVersions = []uint{lpv2, lpv1} + AdvertiseProtocolVersions = []uint{lpv2} // clients are searching for the first advertised protocol in the list ) // Number of implemented message corresponding to different protocol versions. diff --git a/les/server.go b/les/server.go index d8f93cd87b..ec2e44fecc 100644 --- a/les/server.go +++ b/les/server.go @@ -56,8 +56,8 @@ func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { return nil, err } - lesTopics := make([]discv5.Topic, len(ServerProtocolVersions)) - for i, pv := range ServerProtocolVersions { + lesTopics := make([]discv5.Topic, len(AdvertiseProtocolVersions)) + for i, pv := range AdvertiseProtocolVersions { lesTopics[i] = lesTopic(eth.BlockChain().Genesis().Hash(), pv) } diff --git a/mobile/geth.go b/mobile/geth.go index 7b39faadec..7e3b8f4915 100644 --- a/mobile/geth.go +++ b/mobile/geth.go @@ -116,7 +116,6 @@ func NewNode(datadir string, config *NodeConfig) (stack *Node, _ error) { P2P: p2p.Config{ NoDiscovery: true, DiscoveryV5: true, - DiscoveryV5Addr: ":0", BootstrapNodesV5: config.BootstrapNodes.nodes, ListenAddr: ":0", NAT: nat.Any(), diff --git a/node/defaults.go b/node/defaults.go index 848f08e05c..d4e1486834 100644 --- a/node/defaults.go +++ b/node/defaults.go @@ -41,10 +41,9 @@ var DefaultConfig = Config{ WSPort: DefaultWSPort, WSModules: []string{"net", "web3"}, P2P: p2p.Config{ - ListenAddr: ":30303", - DiscoveryV5Addr: ":30304", - MaxPeers: 25, - NAT: nat.Any(), + ListenAddr: ":30303", + MaxPeers: 25, + NAT: nat.Any(), }, } diff --git a/p2p/discover/udp.go b/p2p/discover/udp.go index f9eb99ee36..60436952d8 100644 --- a/p2p/discover/udp.go +++ b/p2p/discover/udp.go @@ -210,17 +210,15 @@ type reply struct { matched chan<- bool } +// ReadPacket is sent to the unhandled channel when it could not be processed +type ReadPacket struct { + Data []byte + Addr *net.UDPAddr +} + // ListenUDP returns a new table that listens for UDP packets on laddr. -func ListenUDP(priv *ecdsa.PrivateKey, laddr string, natm nat.Interface, nodeDBPath string, netrestrict *netutil.Netlist) (*Table, error) { - addr, err := net.ResolveUDPAddr("udp", laddr) - if err != nil { - return nil, err - } - conn, err := net.ListenUDP("udp", addr) - if err != nil { - return nil, err - } - tab, _, err := newUDP(priv, conn, natm, nodeDBPath, netrestrict) +func ListenUDP(priv *ecdsa.PrivateKey, conn conn, realaddr *net.UDPAddr, unhandled chan ReadPacket, nodeDBPath string, netrestrict *netutil.Netlist) (*Table, error) { + tab, _, err := newUDP(priv, conn, realaddr, unhandled, nodeDBPath, netrestrict) if err != nil { return nil, err } @@ -228,7 +226,7 @@ func ListenUDP(priv *ecdsa.PrivateKey, laddr string, natm nat.Interface, nodeDBP return tab, nil } -func newUDP(priv *ecdsa.PrivateKey, c conn, natm nat.Interface, nodeDBPath string, netrestrict *netutil.Netlist) (*Table, *udp, error) { +func newUDP(priv *ecdsa.PrivateKey, c conn, realaddr *net.UDPAddr, unhandled chan ReadPacket, nodeDBPath string, netrestrict *netutil.Netlist) (*Table, *udp, error) { udp := &udp{ conn: c, priv: priv, @@ -237,16 +235,6 @@ func newUDP(priv *ecdsa.PrivateKey, c conn, natm nat.Interface, nodeDBPath strin gotreply: make(chan reply), addpending: make(chan *pending), } - realaddr := c.LocalAddr().(*net.UDPAddr) - if natm != nil { - if !realaddr.IP.IsLoopback() { - go nat.Map(natm, udp.closing, "udp", realaddr.Port, realaddr.Port, "ethereum discovery") - } - // TODO: react to external IP changes over time. - if ext, err := natm.ExternalIP(); err == nil { - realaddr = &net.UDPAddr{IP: ext, Port: realaddr.Port} - } - } // TODO: separate TCP port udp.ourEndpoint = makeEndpoint(realaddr, uint16(realaddr.Port)) tab, err := newTable(udp, PubkeyID(&priv.PublicKey), realaddr, nodeDBPath) @@ -256,7 +244,7 @@ func newUDP(priv *ecdsa.PrivateKey, c conn, natm nat.Interface, nodeDBPath strin udp.Table = tab go udp.loop() - go udp.readLoop() + go udp.readLoop(unhandled) return udp.Table, udp, nil } @@ -492,8 +480,11 @@ func encodePacket(priv *ecdsa.PrivateKey, ptype byte, req interface{}) ([]byte, } // readLoop runs in its own goroutine. it handles incoming UDP packets. -func (t *udp) readLoop() { +func (t *udp) readLoop(unhandled chan ReadPacket) { defer t.conn.Close() + if unhandled != nil { + defer close(unhandled) + } // Discovery packets are defined to be no larger than 1280 bytes. // Packets larger than this size will be cut at the end and treated // as invalid because their hash won't match. @@ -509,7 +500,12 @@ func (t *udp) readLoop() { log.Debug("UDP read error", "err", err) return } - t.handlePacket(from, buf[:nbytes]) + if t.handlePacket(from, buf[:nbytes]) != nil && unhandled != nil { + select { + case unhandled <- ReadPacket{buf[:nbytes], from}: + default: + } + } } } diff --git a/p2p/discover/udp_test.go b/p2p/discover/udp_test.go index 21e8b561da..b81caf8392 100644 --- a/p2p/discover/udp_test.go +++ b/p2p/discover/udp_test.go @@ -70,7 +70,8 @@ func newUDPTest(t *testing.T) *udpTest { remotekey: newkey(), remoteaddr: &net.UDPAddr{IP: net.IP{10, 0, 1, 99}, Port: 30303}, } - test.table, test.udp, _ = newUDP(test.localkey, test.pipe, nil, "", nil) + realaddr := test.pipe.LocalAddr().(*net.UDPAddr) + test.table, test.udp, _ = newUDP(test.localkey, test.pipe, realaddr, nil, "", nil) return test } diff --git a/p2p/discv5/net.go b/p2p/discv5/net.go index cd9981584d..f9baf126f1 100644 --- a/p2p/discv5/net.go +++ b/p2p/discv5/net.go @@ -29,7 +29,6 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/p2p/nat" "github.com/ethereum/go-ethereum/p2p/netutil" "github.com/ethereum/go-ethereum/rlp" ) @@ -134,7 +133,7 @@ type timeoutEvent struct { node *Node } -func newNetwork(conn transport, ourPubkey ecdsa.PublicKey, natm nat.Interface, dbPath string, netrestrict *netutil.Netlist) (*Network, error) { +func newNetwork(conn transport, ourPubkey ecdsa.PublicKey, dbPath string, netrestrict *netutil.Netlist) (*Network, error) { ourID := PubkeyID(&ourPubkey) var db *nodeDB diff --git a/p2p/discv5/net_test.go b/p2p/discv5/net_test.go index bd234f5ba6..369282ca9c 100644 --- a/p2p/discv5/net_test.go +++ b/p2p/discv5/net_test.go @@ -28,7 +28,7 @@ import ( func TestNetwork_Lookup(t *testing.T) { key, _ := crypto.GenerateKey() - network, err := newNetwork(lookupTestnet, key.PublicKey, nil, "", nil) + network, err := newNetwork(lookupTestnet, key.PublicKey, "", nil) if err != nil { t.Fatal(err) } diff --git a/p2p/discv5/sim_test.go b/p2p/discv5/sim_test.go index bf57872e2d..543faecd48 100644 --- a/p2p/discv5/sim_test.go +++ b/p2p/discv5/sim_test.go @@ -282,7 +282,7 @@ func (s *simulation) launchNode(log bool) *Network { addr := &net.UDPAddr{IP: ip, Port: 30303} transport := &simTransport{joinTime: time.Now(), sender: id, senderAddr: addr, sim: s, priv: key} - net, err := newNetwork(transport, key.PublicKey, nil, "", nil) + net, err := newNetwork(transport, key.PublicKey, "", nil) if err != nil { panic("cannot launch new node: " + err.Error()) } diff --git a/p2p/discv5/ticket.go b/p2p/discv5/ticket.go index b45ec4d2be..023c5000d2 100644 --- a/p2p/discv5/ticket.go +++ b/p2p/discv5/ticket.go @@ -642,7 +642,7 @@ func (s *ticketStore) gotTopicNodes(from *Node, hash common.Hash, nodes []rpcNod if ip.IsUnspecified() || ip.IsLoopback() { ip = from.IP } - n := NewNode(node.ID, ip, node.UDP-1, node.TCP-1) // subtract one from port while discv5 is running in test mode on UDPport+1 + n := NewNode(node.ID, ip, node.UDP, node.TCP) select { case chn <- n: default: diff --git a/p2p/discv5/udp.go b/p2p/discv5/udp.go index 26087cd8e5..e921520768 100644 --- a/p2p/discv5/udp.go +++ b/p2p/discv5/udp.go @@ -37,7 +37,7 @@ const Version = 4 // Errors var ( errPacketTooSmall = errors.New("too small") - errBadHash = errors.New("bad hash") + errBadPrefix = errors.New("bad prefix") errExpired = errors.New("expired") errUnsolicitedReply = errors.New("unsolicited reply") errUnknownNode = errors.New("unknown node") @@ -145,10 +145,11 @@ type ( } ) -const ( - macSize = 256 / 8 - sigSize = 520 / 8 - headSize = macSize + sigSize // space of packet frame data +var ( + versionPrefix = []byte("temporary discovery v5") + versionPrefixSize = len(versionPrefix) + sigSize = 520 / 8 + headSize = versionPrefixSize + sigSize // space of packet frame data ) // Neighbors replies are sent across multiple packets to @@ -237,12 +238,12 @@ type udp struct { } // ListenUDP returns a new table that listens for UDP packets on laddr. -func ListenUDP(priv *ecdsa.PrivateKey, laddr string, natm nat.Interface, nodeDBPath string, netrestrict *netutil.Netlist) (*Network, error) { - transport, err := listenUDP(priv, laddr) +func ListenUDP(priv *ecdsa.PrivateKey, conn conn, realaddr *net.UDPAddr, nodeDBPath string, netrestrict *netutil.Netlist) (*Network, error) { + transport, err := listenUDP(priv, conn, realaddr) if err != nil { return nil, err } - net, err := newNetwork(transport, priv.PublicKey, natm, nodeDBPath, netrestrict) + net, err := newNetwork(transport, priv.PublicKey, nodeDBPath, netrestrict) if err != nil { return nil, err } @@ -251,16 +252,8 @@ func ListenUDP(priv *ecdsa.PrivateKey, laddr string, natm nat.Interface, nodeDBP return net, nil } -func listenUDP(priv *ecdsa.PrivateKey, laddr string) (*udp, error) { - addr, err := net.ResolveUDPAddr("udp", laddr) - if err != nil { - return nil, err - } - conn, err := net.ListenUDP("udp", addr) - if err != nil { - return nil, err - } - return &udp{conn: conn, priv: priv, ourEndpoint: makeEndpoint(addr, uint16(addr.Port))}, nil +func listenUDP(priv *ecdsa.PrivateKey, conn conn, realaddr *net.UDPAddr) (*udp, error) { + return &udp{conn: conn, priv: priv, ourEndpoint: makeEndpoint(realaddr, uint16(realaddr.Port))}, nil } func (t *udp) localAddr() *net.UDPAddr { @@ -372,11 +365,9 @@ func encodePacket(priv *ecdsa.PrivateKey, ptype byte, req interface{}) (p, hash log.Error(fmt.Sprint("could not sign packet:", err)) return nil, nil, err } - copy(packet[macSize:], sig) - // add the hash to the front. Note: this doesn't protect the - // packet in any way. - hash = crypto.Keccak256(packet[macSize:]) - copy(packet, hash) + copy(packet, versionPrefix) + copy(packet[versionPrefixSize:], sig) + hash = crypto.Keccak256(packet[versionPrefixSize:]) return packet, hash, nil } @@ -420,17 +411,16 @@ func decodePacket(buffer []byte, pkt *ingressPacket) error { } buf := make([]byte, len(buffer)) copy(buf, buffer) - hash, sig, sigdata := buf[:macSize], buf[macSize:headSize], buf[headSize:] - shouldhash := crypto.Keccak256(buf[macSize:]) - if !bytes.Equal(hash, shouldhash) { - return errBadHash + prefix, sig, sigdata := buf[:versionPrefixSize], buf[versionPrefixSize:headSize], buf[headSize:] + if !bytes.Equal(prefix, versionPrefix) { + return errBadPrefix } fromID, err := recoverNodeID(crypto.Keccak256(buf[headSize:]), sig) if err != nil { return err } pkt.rawData = buf - pkt.hash = hash + pkt.hash = crypto.Keccak256(buf[versionPrefixSize:]) pkt.remoteID = fromID switch pkt.ev = nodeEvent(sigdata[0]); pkt.ev { case pingPacket: diff --git a/p2p/server.go b/p2p/server.go index 922df55ba5..2cff94ea5b 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -78,9 +78,6 @@ type Config struct { // protocol should be started or not. DiscoveryV5 bool `toml:",omitempty"` - // Listener address for the V5 discovery protocol UDP traffic. - DiscoveryV5Addr string `toml:",omitempty"` - // Name sets the node name of this server. // Use common.MakeName to create a name that follows existing conventions. Name string `toml:"-"` @@ -354,6 +351,32 @@ func (srv *Server) Stop() { srv.loopWG.Wait() } +// sharedUDPConn implements a shared connection. Write sends messages to the underlying connection while read returns +// messages that were found unprocessable and sent to the unhandled channel by the primary listener. +type sharedUDPConn struct { + *net.UDPConn + unhandled chan discover.ReadPacket +} + +// ReadFromUDP implements discv5.conn +func (s *sharedUDPConn) ReadFromUDP(b []byte) (n int, addr *net.UDPAddr, err error) { + packet, ok := <-s.unhandled + if !ok { + return 0, nil, fmt.Errorf("Connection was closed") + } + l := len(packet.Data) + if l > len(b) { + l = len(b) + } + copy(b[:l], packet.Data[:l]) + return l, packet.Addr, nil +} + +// Close implements discv5.conn +func (s *sharedUDPConn) Close() error { + return nil +} + // Start starts running the server. // Servers can not be re-used after stopping. func (srv *Server) Start() (err error) { @@ -388,9 +411,43 @@ func (srv *Server) Start() (err error) { srv.peerOp = make(chan peerOpFunc) srv.peerOpDone = make(chan struct{}) + var ( + conn *net.UDPConn + sconn *sharedUDPConn + realaddr *net.UDPAddr + unhandled chan discover.ReadPacket + ) + + if !srv.NoDiscovery || srv.DiscoveryV5 { + addr, err := net.ResolveUDPAddr("udp", srv.ListenAddr) + if err != nil { + return err + } + conn, err = net.ListenUDP("udp", addr) + if err != nil { + return err + } + + realaddr = conn.LocalAddr().(*net.UDPAddr) + if srv.NAT != nil { + if !realaddr.IP.IsLoopback() { + go nat.Map(srv.NAT, srv.quit, "udp", realaddr.Port, realaddr.Port, "ethereum discovery") + } + // TODO: react to external IP changes over time. + if ext, err := srv.NAT.ExternalIP(); err == nil { + realaddr = &net.UDPAddr{IP: ext, Port: realaddr.Port} + } + } + } + + if !srv.NoDiscovery && srv.DiscoveryV5 { + unhandled = make(chan discover.ReadPacket, 100) + sconn = &sharedUDPConn{conn, unhandled} + } + // node table if !srv.NoDiscovery { - ntab, err := discover.ListenUDP(srv.PrivateKey, srv.ListenAddr, srv.NAT, srv.NodeDatabase, srv.NetRestrict) + ntab, err := discover.ListenUDP(srv.PrivateKey, conn, realaddr, unhandled, srv.NodeDatabase, srv.NetRestrict) if err != nil { return err } @@ -401,7 +458,15 @@ func (srv *Server) Start() (err error) { } if srv.DiscoveryV5 { - ntab, err := discv5.ListenUDP(srv.PrivateKey, srv.DiscoveryV5Addr, srv.NAT, "", srv.NetRestrict) //srv.NodeDatabase) + var ( + ntab *discv5.Network + err error + ) + if sconn != nil { + ntab, err = discv5.ListenUDP(srv.PrivateKey, sconn, realaddr, "", srv.NetRestrict) //srv.NodeDatabase) + } else { + ntab, err = discv5.ListenUDP(srv.PrivateKey, conn, realaddr, "", srv.NetRestrict) //srv.NodeDatabase) + } if err != nil { return err } diff --git a/params/bootnodes.go b/params/bootnodes.go index ecb1acd4f1..849b569200 100644 --- a/params/bootnodes.go +++ b/params/bootnodes.go @@ -56,7 +56,7 @@ var RinkebyV5Bootnodes = []string{ // DiscoveryV5Bootnodes are the enode URLs of the P2P bootstrap nodes for the // experimental RLPx v5 topic-discovery network. var DiscoveryV5Bootnodes = []string{ - "enode://0cc5f5ffb5d9098c8b8c62325f3797f56509bff942704687b6530992ac706e2cb946b90a34f1f19548cd3c7baccbcaea354531e5983c7d1bc0dee16ce4b6440b@40.118.3.223:30305", - "enode://1c7a64d76c0334b0418c004af2f67c50e36a3be60b5e4790bdac0439d21603469a85fad36f2473c9a80eb043ae60936df905fa28f1ff614c3e5dc34f15dcd2dc@40.118.3.223:30308", - "enode://85c85d7143ae8bb96924f2b54f1b3e70d8c4d367af305325d30a61385a432f247d2c75c45c6b4a60335060d072d7f5b35dd1d4c45f76941f62a4f83b6e75daaf@40.118.3.223:30309", + "enode://0cc5f5ffb5d9098c8b8c62325f3797f56509bff942704687b6530992ac706e2cb946b90a34f1f19548cd3c7baccbcaea354531e5983c7d1bc0dee16ce4b6440b@40.118.3.223:30304", + "enode://1c7a64d76c0334b0418c004af2f67c50e36a3be60b5e4790bdac0439d21603469a85fad36f2473c9a80eb043ae60936df905fa28f1ff614c3e5dc34f15dcd2dc@40.118.3.223:30306", + "enode://85c85d7143ae8bb96924f2b54f1b3e70d8c4d367af305325d30a61385a432f247d2c75c45c6b4a60335060d072d7f5b35dd1d4c45f76941f62a4f83b6e75daaf@40.118.3.223:30307", } From 48641d7308daa2956197a77208692f0c20cad7c9 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 23 Jan 2018 08:50:11 +0100 Subject: [PATCH 047/107] p2p/discv5: logs info about discv5 node info at bind time --- p2p/discv5/udp.go | 1 + 1 file changed, 1 insertion(+) diff --git a/p2p/discv5/udp.go b/p2p/discv5/udp.go index e921520768..5437718173 100644 --- a/p2p/discv5/udp.go +++ b/p2p/discv5/udp.go @@ -247,6 +247,7 @@ func ListenUDP(priv *ecdsa.PrivateKey, conn conn, realaddr *net.UDPAddr, nodeDBP if err != nil { return nil, err } + log.Info("UDP listener up", "net", net.tab.self) transport.net = net go transport.readLoop() return net, nil From 924065e19d08cc7e6af0b3a5b5b1ef3785b79bd4 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 23 Jan 2018 11:05:30 +0100 Subject: [PATCH 048/107] consensus/ethash: improve cache/dataset handling (#15864) * consensus/ethash: add maxEpoch constant * consensus/ethash: improve cache/dataset handling There are two fixes in this commit: Unmap the memory through a finalizer like the libethash wrapper did. The release logic was incorrect and freed the memory while it was being used, leading to crashes like in #14495 or #14943. Track caches and datasets using simplelru instead of reinventing LRU logic. This should make it easier to see whether it's correct. * consensus/ethash: restore 'future item' logic in lru * consensus/ethash: use mmap even in test mode This makes it possible to shorten the time taken for TestCacheFileEvict. * consensus/ethash: shuffle func calc*Size comments around * consensus/ethash: ensure future cache/dataset is in the lru cache * consensus/ethash: add issue link to the new test * consensus/ethash: fix vet * consensus/ethash: fix test * consensus: tiny issue + nitpick fixes --- consensus/ethash/algorithm.go | 6 +- consensus/ethash/algorithm_go1.7.go | 4 +- consensus/ethash/algorithm_go1.8.go | 34 +-- consensus/ethash/algorithm_go1.8_test.go | 23 +- consensus/ethash/consensus.go | 10 +- consensus/ethash/ethash.go | 288 ++++++++++------------- consensus/ethash/ethash_test.go | 39 +++ consensus/ethash/sealer.go | 17 +- 8 files changed, 208 insertions(+), 213 deletions(-) diff --git a/consensus/ethash/algorithm.go b/consensus/ethash/algorithm.go index 76f19252fa..10767bb312 100644 --- a/consensus/ethash/algorithm.go +++ b/consensus/ethash/algorithm.go @@ -355,9 +355,11 @@ func hashimotoFull(dataset []uint32, hash []byte, nonce uint64) ([]byte, []byte) return hashimoto(hash, nonce, uint64(len(dataset))*4, lookup) } +const maxEpoch = 2048 + // datasetSizes is a lookup table for the ethash dataset size for the first 2048 // epochs (i.e. 61440000 blocks). -var datasetSizes = []uint64{ +var datasetSizes = [maxEpoch]uint64{ 1073739904, 1082130304, 1090514816, 1098906752, 1107293056, 1115684224, 1124070016, 1132461952, 1140849536, 1149232768, 1157627776, 1166013824, 1174404736, 1182786944, 1191180416, @@ -771,7 +773,7 @@ var datasetSizes = []uint64{ // cacheSizes is a lookup table for the ethash verification cache size for the // first 2048 epochs (i.e. 61440000 blocks). -var cacheSizes = []uint64{ +var cacheSizes = [maxEpoch]uint64{ 16776896, 16907456, 17039296, 17170112, 17301056, 17432512, 17563072, 17693888, 17824192, 17955904, 18087488, 18218176, 18349504, 18481088, 18611392, 18742336, 18874304, 19004224, 19135936, 19267264, 19398208, diff --git a/consensus/ethash/algorithm_go1.7.go b/consensus/ethash/algorithm_go1.7.go index c34d041c32..c7f7f48e41 100644 --- a/consensus/ethash/algorithm_go1.7.go +++ b/consensus/ethash/algorithm_go1.7.go @@ -25,7 +25,7 @@ package ethash func cacheSize(block uint64) uint64 { // If we have a pre-generated value, use that epoch := int(block / epochLength) - if epoch < len(cacheSizes) { + if epoch < maxEpoch { return cacheSizes[epoch] } // We don't have a way to verify primes fast before Go 1.8 @@ -39,7 +39,7 @@ func cacheSize(block uint64) uint64 { func datasetSize(block uint64) uint64 { // If we have a pre-generated value, use that epoch := int(block / epochLength) - if epoch < len(datasetSizes) { + if epoch < maxEpoch { return datasetSizes[epoch] } // We don't have a way to verify primes fast before Go 1.8 diff --git a/consensus/ethash/algorithm_go1.8.go b/consensus/ethash/algorithm_go1.8.go index d691b758f0..975fdffe51 100644 --- a/consensus/ethash/algorithm_go1.8.go +++ b/consensus/ethash/algorithm_go1.8.go @@ -20,17 +20,20 @@ package ethash import "math/big" -// cacheSize calculates and returns the size of the ethash verification cache that -// belongs to a certain block number. The cache size grows linearly, however, we -// always take the highest prime below the linearly growing threshold in order to -// reduce the risk of accidental regularities leading to cyclic behavior. +// cacheSize returns the size of the ethash verification cache that belongs to a certain +// block number. func cacheSize(block uint64) uint64 { - // If we have a pre-generated value, use that epoch := int(block / epochLength) - if epoch < len(cacheSizes) { + if epoch < maxEpoch { return cacheSizes[epoch] } - // No known cache size, calculate manually (sanity branch only) + return calcCacheSize(epoch) +} + +// calcCacheSize calculates the cache size for epoch. The cache size grows linearly, +// however, we always take the highest prime below the linearly growing threshold in order +// to reduce the risk of accidental regularities leading to cyclic behavior. +func calcCacheSize(epoch int) uint64 { size := cacheInitBytes + cacheGrowthBytes*uint64(epoch) - hashBytes for !new(big.Int).SetUint64(size / hashBytes).ProbablyPrime(1) { // Always accurate for n < 2^64 size -= 2 * hashBytes @@ -38,17 +41,20 @@ func cacheSize(block uint64) uint64 { return size } -// datasetSize calculates and returns the size of the ethash mining dataset that -// belongs to a certain block number. The dataset size grows linearly, however, we -// always take the highest prime below the linearly growing threshold in order to -// reduce the risk of accidental regularities leading to cyclic behavior. +// datasetSize returns the size of the ethash mining dataset that belongs to a certain +// block number. func datasetSize(block uint64) uint64 { - // If we have a pre-generated value, use that epoch := int(block / epochLength) - if epoch < len(datasetSizes) { + if epoch < maxEpoch { return datasetSizes[epoch] } - // No known dataset size, calculate manually (sanity branch only) + return calcDatasetSize(epoch) +} + +// calcDatasetSize calculates the dataset size for epoch. The dataset size grows linearly, +// however, we always take the highest prime below the linearly growing threshold in order +// to reduce the risk of accidental regularities leading to cyclic behavior. +func calcDatasetSize(epoch int) uint64 { size := datasetInitBytes + datasetGrowthBytes*uint64(epoch) - mixBytes for !new(big.Int).SetUint64(size / mixBytes).ProbablyPrime(1) { // Always accurate for n < 2^64 size -= 2 * mixBytes diff --git a/consensus/ethash/algorithm_go1.8_test.go b/consensus/ethash/algorithm_go1.8_test.go index a822944a60..6648bd6a97 100644 --- a/consensus/ethash/algorithm_go1.8_test.go +++ b/consensus/ethash/algorithm_go1.8_test.go @@ -23,24 +23,15 @@ import "testing" // Tests whether the dataset size calculator works correctly by cross checking the // hard coded lookup table with the value generated by it. func TestSizeCalculations(t *testing.T) { - var tests []uint64 - - // Verify all the cache sizes from the lookup table - defer func(sizes []uint64) { cacheSizes = sizes }(cacheSizes) - tests, cacheSizes = cacheSizes, []uint64{} - - for i, test := range tests { - if size := cacheSize(uint64(i*epochLength) + 1); size != test { - t.Errorf("cache %d: cache size mismatch: have %d, want %d", i, size, test) + // Verify all the cache and dataset sizes from the lookup table. + for epoch, want := range cacheSizes { + if size := calcCacheSize(epoch); size != want { + t.Errorf("cache %d: cache size mismatch: have %d, want %d", epoch, size, want) } } - // Verify all the dataset sizes from the lookup table - defer func(sizes []uint64) { datasetSizes = sizes }(datasetSizes) - tests, datasetSizes = datasetSizes, []uint64{} - - for i, test := range tests { - if size := datasetSize(uint64(i*epochLength) + 1); size != test { - t.Errorf("dataset %d: dataset size mismatch: have %d, want %d", i, size, test) + for epoch, want := range datasetSizes { + if size := calcDatasetSize(epoch); size != want { + t.Errorf("dataset %d: dataset size mismatch: have %d, want %d", epoch, size, want) } } } diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 82d23c92b6..92a23d4a4d 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -476,7 +476,7 @@ func (ethash *Ethash) VerifySeal(chain consensus.ChainReader, header *types.Head } // Sanity check that the block number is below the lookup table size (60M blocks) number := header.Number.Uint64() - if number/epochLength >= uint64(len(cacheSizes)) { + if number/epochLength >= maxEpoch { // Go < 1.7 cannot calculate new cache/dataset sizes (no fast prime check) return errNonceOutOfRange } @@ -484,14 +484,18 @@ func (ethash *Ethash) VerifySeal(chain consensus.ChainReader, header *types.Head if header.Difficulty.Sign() <= 0 { return errInvalidDifficulty } + // Recompute the digest and PoW value and verify against the header cache := ethash.cache(number) - size := datasetSize(number) if ethash.config.PowMode == ModeTest { size = 32 * 1024 } - digest, result := hashimotoLight(size, cache, header.HashNoNonce().Bytes(), header.Nonce.Uint64()) + digest, result := hashimotoLight(size, cache.cache, header.HashNoNonce().Bytes(), header.Nonce.Uint64()) + // Caches are unmapped in a finalizer. Ensure that the cache stays live + // until after the call to hashimotoLight so it's not unmapped while being used. + runtime.KeepAlive(cache) + if !bytes.Equal(header.MixDigest[:], digest) { return errInvalidMixDigest } diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index a78b3a895d..91e20112ae 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -26,6 +26,7 @@ import ( "os" "path/filepath" "reflect" + "runtime" "strconv" "sync" "time" @@ -35,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" + "github.com/hashicorp/golang-lru/simplelru" metrics "github.com/rcrowley/go-metrics" ) @@ -142,32 +144,82 @@ func memoryMapAndGenerate(path string, size uint64, generator func(buffer []uint return memoryMap(path) } +// lru tracks caches or datasets by their last use time, keeping at most N of them. +type lru struct { + what string + new func(epoch uint64) interface{} + mu sync.Mutex + // Items are kept in a LRU cache, but there is a special case: + // We always keep an item for (highest seen epoch) + 1 as the 'future item'. + cache *simplelru.LRU + future uint64 + futureItem interface{} +} + +// newlru create a new least-recently-used cache for ither the verification caches +// or the mining datasets. +func newlru(what string, maxItems int, new func(epoch uint64) interface{}) *lru { + if maxItems <= 0 { + maxItems = 1 + } + cache, _ := simplelru.NewLRU(maxItems, func(key, value interface{}) { + log.Trace("Evicted ethash "+what, "epoch", key) + }) + return &lru{what: what, new: new, cache: cache} +} + +// get retrieves or creates an item for the given epoch. The first return value is always +// non-nil. The second return value is non-nil if lru thinks that an item will be useful in +// the near future. +func (lru *lru) get(epoch uint64) (item, future interface{}) { + lru.mu.Lock() + defer lru.mu.Unlock() + + // Get or create the item for the requested epoch. + item, ok := lru.cache.Get(epoch) + if !ok { + if lru.future > 0 && lru.future == epoch { + item = lru.futureItem + } else { + log.Trace("Requiring new ethash "+lru.what, "epoch", epoch) + item = lru.new(epoch) + } + lru.cache.Add(epoch, item) + } + // Update the 'future item' if epoch is larger than previously seen. + if epoch < maxEpoch-1 && lru.future < epoch+1 { + log.Trace("Requiring new future ethash "+lru.what, "epoch", epoch+1) + future = lru.new(epoch + 1) + lru.future = epoch + 1 + lru.futureItem = future + } + return item, future +} + // cache wraps an ethash cache with some metadata to allow easier concurrent use. type cache struct { - epoch uint64 // Epoch for which this cache is relevant + epoch uint64 // Epoch for which this cache is relevant + dump *os.File // File descriptor of the memory mapped cache + mmap mmap.MMap // Memory map itself to unmap before releasing + cache []uint32 // The actual cache data content (may be memory mapped) + once sync.Once // Ensures the cache is generated only once +} - dump *os.File // File descriptor of the memory mapped cache - mmap mmap.MMap // Memory map itself to unmap before releasing - - cache []uint32 // The actual cache data content (may be memory mapped) - used time.Time // Timestamp of the last use for smarter eviction - once sync.Once // Ensures the cache is generated only once - lock sync.Mutex // Ensures thread safety for updating the usage time +// newCache creates a new ethash verification cache and returns it as a plain Go +// interface to be usable in an LRU cache. +func newCache(epoch uint64) interface{} { + return &cache{epoch: epoch} } // generate ensures that the cache content is generated before use. func (c *cache) generate(dir string, limit int, test bool) { c.once.Do(func() { - // If we have a testing cache, generate and return - if test { - c.cache = make([]uint32, 1024/4) - generateCache(c.cache, c.epoch, seedHash(c.epoch*epochLength+1)) - return - } - // If we don't store anything on disk, generate and return size := cacheSize(c.epoch*epochLength + 1) seed := seedHash(c.epoch*epochLength + 1) - + if test { + size = 1024 + } + // If we don't store anything on disk, generate and return. if dir == "" { c.cache = make([]uint32, size/4) generateCache(c.cache, c.epoch, seed) @@ -181,6 +233,10 @@ func (c *cache) generate(dir string, limit int, test bool) { path := filepath.Join(dir, fmt.Sprintf("cache-R%d-%x%s", algorithmRevision, seed[:8], endian)) logger := log.New("epoch", c.epoch) + // We're about to mmap the file, ensure that the mapping is cleaned up when the + // cache becomes unused. + runtime.SetFinalizer(c, (*cache).finalizer) + // Try to load the file from disk and memory map it var err error c.dump, c.mmap, c.cache, err = memoryMap(path) @@ -207,49 +263,41 @@ func (c *cache) generate(dir string, limit int, test bool) { }) } -// release closes any file handlers and memory maps open. -func (c *cache) release() { +// finalizer unmaps the memory and closes the file. +func (c *cache) finalizer() { if c.mmap != nil { c.mmap.Unmap() - c.mmap = nil - } - if c.dump != nil { c.dump.Close() - c.dump = nil + c.mmap, c.dump = nil, nil } } // dataset wraps an ethash dataset with some metadata to allow easier concurrent use. type dataset struct { - epoch uint64 // Epoch for which this cache is relevant + epoch uint64 // Epoch for which this cache is relevant + dump *os.File // File descriptor of the memory mapped cache + mmap mmap.MMap // Memory map itself to unmap before releasing + dataset []uint32 // The actual cache data content + once sync.Once // Ensures the cache is generated only once +} - dump *os.File // File descriptor of the memory mapped cache - mmap mmap.MMap // Memory map itself to unmap before releasing - - dataset []uint32 // The actual cache data content - used time.Time // Timestamp of the last use for smarter eviction - once sync.Once // Ensures the cache is generated only once - lock sync.Mutex // Ensures thread safety for updating the usage time +// newDataset creates a new ethash mining dataset and returns it as a plain Go +// interface to be usable in an LRU cache. +func newDataset(epoch uint64) interface{} { + return &dataset{epoch: epoch} } // generate ensures that the dataset content is generated before use. func (d *dataset) generate(dir string, limit int, test bool) { d.once.Do(func() { - // If we have a testing dataset, generate and return - if test { - cache := make([]uint32, 1024/4) - generateCache(cache, d.epoch, seedHash(d.epoch*epochLength+1)) - - d.dataset = make([]uint32, 32*1024/4) - generateDataset(d.dataset, d.epoch, cache) - - return - } - // If we don't store anything on disk, generate and return csize := cacheSize(d.epoch*epochLength + 1) dsize := datasetSize(d.epoch*epochLength + 1) seed := seedHash(d.epoch*epochLength + 1) - + if test { + csize = 1024 + dsize = 32 * 1024 + } + // If we don't store anything on disk, generate and return if dir == "" { cache := make([]uint32, csize/4) generateCache(cache, d.epoch, seed) @@ -265,6 +313,10 @@ func (d *dataset) generate(dir string, limit int, test bool) { path := filepath.Join(dir, fmt.Sprintf("full-R%d-%x%s", algorithmRevision, seed[:8], endian)) logger := log.New("epoch", d.epoch) + // We're about to mmap the file, ensure that the mapping is cleaned up when the + // cache becomes unused. + runtime.SetFinalizer(d, (*dataset).finalizer) + // Try to load the file from disk and memory map it var err error d.dump, d.mmap, d.dataset, err = memoryMap(path) @@ -294,15 +346,12 @@ func (d *dataset) generate(dir string, limit int, test bool) { }) } -// release closes any file handlers and memory maps open. -func (d *dataset) release() { +// finalizer closes any file handlers and memory maps open. +func (d *dataset) finalizer() { if d.mmap != nil { d.mmap.Unmap() - d.mmap = nil - } - if d.dump != nil { d.dump.Close() - d.dump = nil + d.mmap, d.dump = nil, nil } } @@ -310,14 +359,12 @@ func (d *dataset) release() { func MakeCache(block uint64, dir string) { c := cache{epoch: block / epochLength} c.generate(dir, math.MaxInt32, false) - c.release() } // MakeDataset generates a new ethash dataset and optionally stores it to disk. func MakeDataset(block uint64, dir string) { d := dataset{epoch: block / epochLength} d.generate(dir, math.MaxInt32, false) - d.release() } // Mode defines the type and amount of PoW verification an ethash engine makes. @@ -347,10 +394,8 @@ type Config struct { type Ethash struct { config Config - caches map[uint64]*cache // In memory caches to avoid regenerating too often - fcache *cache // Pre-generated cache for the estimated future epoch - datasets map[uint64]*dataset // In memory datasets to avoid regenerating too often - fdataset *dataset // Pre-generated dataset for the estimated future epoch + caches *lru // In memory caches to avoid regenerating too often + datasets *lru // In memory datasets to avoid regenerating too often // Mining related fields rand *rand.Rand // Properly seeded random source for nonces @@ -380,8 +425,8 @@ func New(config Config) *Ethash { } return &Ethash{ config: config, - caches: make(map[uint64]*cache), - datasets: make(map[uint64]*dataset), + caches: newlru("cache", config.CachesInMem, newCache), + datasets: newlru("dataset", config.DatasetsInMem, newDataset), update: make(chan struct{}), hashrate: metrics.NewMeter(), } @@ -390,16 +435,7 @@ func New(config Config) *Ethash { // NewTester creates a small sized ethash PoW scheme useful only for testing // purposes. func NewTester() *Ethash { - return &Ethash{ - config: Config{ - CachesInMem: 1, - PowMode: ModeTest, - }, - caches: make(map[uint64]*cache), - datasets: make(map[uint64]*dataset), - update: make(chan struct{}), - hashrate: metrics.NewMeter(), - } + return New(Config{CachesInMem: 1, PowMode: ModeTest}) } // NewFaker creates a ethash consensus engine with a fake PoW scheme that accepts @@ -456,126 +492,40 @@ func NewShared() *Ethash { // cache tries to retrieve a verification cache for the specified block number // by first checking against a list of in-memory caches, then against caches // stored on disk, and finally generating one if none can be found. -func (ethash *Ethash) cache(block uint64) []uint32 { +func (ethash *Ethash) cache(block uint64) *cache { epoch := block / epochLength + currentI, futureI := ethash.caches.get(epoch) + current := currentI.(*cache) - // If we have a PoW for that epoch, use that - ethash.lock.Lock() - - current, future := ethash.caches[epoch], (*cache)(nil) - if current == nil { - // No in-memory cache, evict the oldest if the cache limit was reached - for len(ethash.caches) > 0 && len(ethash.caches) >= ethash.config.CachesInMem { - var evict *cache - for _, cache := range ethash.caches { - if evict == nil || evict.used.After(cache.used) { - evict = cache - } - } - delete(ethash.caches, evict.epoch) - evict.release() - - log.Trace("Evicted ethash cache", "epoch", evict.epoch, "used", evict.used) - } - // If we have the new cache pre-generated, use that, otherwise create a new one - if ethash.fcache != nil && ethash.fcache.epoch == epoch { - log.Trace("Using pre-generated cache", "epoch", epoch) - current, ethash.fcache = ethash.fcache, nil - } else { - log.Trace("Requiring new ethash cache", "epoch", epoch) - current = &cache{epoch: epoch} - } - ethash.caches[epoch] = current - - // If we just used up the future cache, or need a refresh, regenerate - if ethash.fcache == nil || ethash.fcache.epoch <= epoch { - if ethash.fcache != nil { - ethash.fcache.release() - } - log.Trace("Requiring new future ethash cache", "epoch", epoch+1) - future = &cache{epoch: epoch + 1} - ethash.fcache = future - } - // New current cache, set its initial timestamp - current.used = time.Now() - } - ethash.lock.Unlock() - - // Wait for generation finish, bump the timestamp and finalize the cache + // Wait for generation finish. current.generate(ethash.config.CacheDir, ethash.config.CachesOnDisk, ethash.config.PowMode == ModeTest) - current.lock.Lock() - current.used = time.Now() - current.lock.Unlock() - - // If we exhausted the future cache, now's a good time to regenerate it - if future != nil { + // If we need a new future cache, now's a good time to regenerate it. + if futureI != nil { + future := futureI.(*cache) go future.generate(ethash.config.CacheDir, ethash.config.CachesOnDisk, ethash.config.PowMode == ModeTest) } - return current.cache + return current } // dataset tries to retrieve a mining dataset for the specified block number // by first checking against a list of in-memory datasets, then against DAGs // stored on disk, and finally generating one if none can be found. -func (ethash *Ethash) dataset(block uint64) []uint32 { +func (ethash *Ethash) dataset(block uint64) *dataset { epoch := block / epochLength + currentI, futureI := ethash.datasets.get(epoch) + current := currentI.(*dataset) - // If we have a PoW for that epoch, use that - ethash.lock.Lock() - - current, future := ethash.datasets[epoch], (*dataset)(nil) - if current == nil { - // No in-memory dataset, evict the oldest if the dataset limit was reached - for len(ethash.datasets) > 0 && len(ethash.datasets) >= ethash.config.DatasetsInMem { - var evict *dataset - for _, dataset := range ethash.datasets { - if evict == nil || evict.used.After(dataset.used) { - evict = dataset - } - } - delete(ethash.datasets, evict.epoch) - evict.release() - - log.Trace("Evicted ethash dataset", "epoch", evict.epoch, "used", evict.used) - } - // If we have the new cache pre-generated, use that, otherwise create a new one - if ethash.fdataset != nil && ethash.fdataset.epoch == epoch { - log.Trace("Using pre-generated dataset", "epoch", epoch) - current = &dataset{epoch: ethash.fdataset.epoch} // Reload from disk - ethash.fdataset = nil - } else { - log.Trace("Requiring new ethash dataset", "epoch", epoch) - current = &dataset{epoch: epoch} - } - ethash.datasets[epoch] = current - - // If we just used up the future dataset, or need a refresh, regenerate - if ethash.fdataset == nil || ethash.fdataset.epoch <= epoch { - if ethash.fdataset != nil { - ethash.fdataset.release() - } - log.Trace("Requiring new future ethash dataset", "epoch", epoch+1) - future = &dataset{epoch: epoch + 1} - ethash.fdataset = future - } - // New current dataset, set its initial timestamp - current.used = time.Now() - } - ethash.lock.Unlock() - - // Wait for generation finish, bump the timestamp and finalize the cache + // Wait for generation finish. current.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest) - current.lock.Lock() - current.used = time.Now() - current.lock.Unlock() - - // If we exhausted the future dataset, now's a good time to regenerate it - if future != nil { + // If we need a new future dataset, now's a good time to regenerate it. + if futureI != nil { + future := futureI.(*dataset) go future.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest) } - return current.dataset + + return current } // Threads returns the number of mining threads currently enabled. This doesn't diff --git a/consensus/ethash/ethash_test.go b/consensus/ethash/ethash_test.go index b3a2f32f70..31116da437 100644 --- a/consensus/ethash/ethash_test.go +++ b/consensus/ethash/ethash_test.go @@ -17,7 +17,11 @@ package ethash import ( + "io/ioutil" "math/big" + "math/rand" + "os" + "sync" "testing" "github.com/ethereum/go-ethereum/core/types" @@ -38,3 +42,38 @@ func TestTestMode(t *testing.T) { t.Fatalf("unexpected verification error: %v", err) } } + +// This test checks that cache lru logic doesn't crash under load. +// It reproduces https://github.com/ethereum/go-ethereum/issues/14943 +func TestCacheFileEvict(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "ethash-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + e := New(Config{CachesInMem: 3, CachesOnDisk: 10, CacheDir: tmpdir, PowMode: ModeTest}) + + workers := 8 + epochs := 100 + var wg sync.WaitGroup + wg.Add(workers) + for i := 0; i < workers; i++ { + go verifyTest(&wg, e, i, epochs) + } + wg.Wait() +} + +func verifyTest(wg *sync.WaitGroup, e *Ethash, workerIndex, epochs int) { + defer wg.Done() + + const wiggle = 4 * epochLength + r := rand.New(rand.NewSource(int64(workerIndex))) + for epoch := 0; epoch < epochs; epoch++ { + block := int64(epoch)*epochLength - wiggle/2 + r.Int63n(wiggle) + if block < 0 { + block = 0 + } + head := &types.Header{Number: big.NewInt(block), Difficulty: big.NewInt(100)} + e.VerifySeal(nil, head) + } +} diff --git a/consensus/ethash/sealer.go b/consensus/ethash/sealer.go index c2447e4730..b5e742d8bb 100644 --- a/consensus/ethash/sealer.go +++ b/consensus/ethash/sealer.go @@ -97,10 +97,9 @@ func (ethash *Ethash) Seal(chain consensus.ChainReader, block *types.Block, stop func (ethash *Ethash) mine(block *types.Block, id int, seed uint64, abort chan struct{}, found chan *types.Block) { // Extract some data from the header var ( - header = block.Header() - hash = header.HashNoNonce().Bytes() - target = new(big.Int).Div(maxUint256, header.Difficulty) - + header = block.Header() + hash = header.HashNoNonce().Bytes() + target = new(big.Int).Div(maxUint256, header.Difficulty) number = header.Number.Uint64() dataset = ethash.dataset(number) ) @@ -111,13 +110,14 @@ func (ethash *Ethash) mine(block *types.Block, id int, seed uint64, abort chan s ) logger := log.New("miner", id) logger.Trace("Started ethash search for new nonces", "seed", seed) +search: for { select { case <-abort: // Mining terminated, update stats and abort logger.Trace("Ethash nonce search aborted", "attempts", nonce-seed) ethash.hashrate.Mark(attempts) - return + break search default: // We don't have to update hash rate on every nonce, so update after after 2^X nonces @@ -127,7 +127,7 @@ func (ethash *Ethash) mine(block *types.Block, id int, seed uint64, abort chan s attempts = 0 } // Compute the PoW value of this nonce - digest, result := hashimotoFull(dataset, hash, nonce) + digest, result := hashimotoFull(dataset.dataset, hash, nonce) if new(big.Int).SetBytes(result).Cmp(target) <= 0 { // Correct nonce found, create a new header with it header = types.CopyHeader(header) @@ -141,9 +141,12 @@ func (ethash *Ethash) mine(block *types.Block, id int, seed uint64, abort chan s case <-abort: logger.Trace("Ethash nonce found but discarded", "attempts", nonce-seed, "nonce", nonce) } - return + break search } nonce++ } } + // Datasets are unmapped in a finalizer. Ensure that the dataset stays live + // during sealing so it's not unmapped while being read. + runtime.KeepAlive(dataset) } From 397c6cde1e2fd3636024cad5d23d5e06796772dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Tue, 23 Jan 2018 11:53:09 +0100 Subject: [PATCH 049/107] p2p/discv5: fix topic register panic at shutdown (#15946) --- p2p/discv5/ticket.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/discv5/ticket.go b/p2p/discv5/ticket.go index 023c5000d2..1ecef37e40 100644 --- a/p2p/discv5/ticket.go +++ b/p2p/discv5/ticket.go @@ -350,7 +350,7 @@ func (s *ticketStore) nextFilteredTicket() (*ticketRef, time.Duration) { regTime := now + mclock.AbsTime(wait) topic := ticket.t.topics[ticket.idx] - if regTime >= s.tickets[topic].nextReg { + if s.tickets[topic] != nil && regTime >= s.tickets[topic].nextReg { return ticket, wait } s.removeTicketRef(*ticket) From ec96216d1696bca2671bb7d043ba6af02c20738d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Tue, 23 Jan 2018 12:10:49 +0100 Subject: [PATCH 050/107] Chain indexer fix + new CHT (#15934) * core, light: fix chain indexer bug * light: add new CHT --- light/lightchain.go | 2 +- light/postprocess.go | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/light/lightchain.go b/light/lightchain.go index 03c7c1f0d0..0d97ce1a23 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -393,7 +393,7 @@ func (self *LightChain) InsertHeaderChain(chain []*types.Header, checkFreq int) return err } i, err := self.hc.InsertHeaderChain(chain, whFunc, start) - go self.postChainEvents(events) + self.postChainEvents(events) return i, err } diff --git a/light/postprocess.go b/light/postprocess.go index e7e513880f..32dbc102be 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -53,18 +53,18 @@ type trustedCheckpoint struct { var ( mainnetCheckpoint = trustedCheckpoint{ name: "ETH mainnet", - sectionIdx: 129, - sectionHead: common.HexToHash("64100587c8ec9a76870056d07cb0f58622552d16de6253a59cac4b580c899501"), - chtRoot: common.HexToHash("bb4fb4076cbe6923c8a8ce8f158452bbe19564959313466989fda095a60884ca"), - bloomTrieRoot: common.HexToHash("0db524b2c4a2a9520a42fd842b02d2e8fb58ff37c75cf57bd0eb82daeace6716"), + sectionIdx: 150, + sectionHead: common.HexToHash("1e2e67f289565cbe7bd4367f7960dbd73a3f7c53439e1047cd7ba331c8109e39"), + chtRoot: common.HexToHash("f2a6c9ca143d647b44523cc249f1072c8912358ab873a77a5fdc792b8df99e80"), + bloomTrieRoot: common.HexToHash("c018952fa1513c97857e79fbb9a37acaf8432d5b85e52a78eca7dff5fd5900ee"), } ropstenCheckpoint = trustedCheckpoint{ name: "Ropsten testnet", - sectionIdx: 50, - sectionHead: common.HexToHash("00bd65923a1aa67f85e6b4ae67835784dd54be165c37f056691723c55bf016bd"), - chtRoot: common.HexToHash("6f56dc61936752cc1f8c84b4addabdbe6a1c19693de3f21cb818362df2117f03"), - bloomTrieRoot: common.HexToHash("aca7d7c504d22737242effc3fdc604a762a0af9ced898036b5986c3a15220208"), + sectionIdx: 75, + sectionHead: common.HexToHash("12e68324f4578ea3e8e7fb3968167686729396c9279287fa1f1a8b51bb2d05b4"), + chtRoot: common.HexToHash("3e51dc095c69fa654a4cac766e0afff7357515b4b3c3a379c675f810363e54be"), + bloomTrieRoot: common.HexToHash("33e3a70b33c1d73aa698d496a80615e98ed31fa8f56969876180553b32333339"), } ) From 3d92c9388873406dc57abe15a8e2d80c65910f05 Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 18 Jan 2018 01:13:13 +0100 Subject: [PATCH 051/107] swarm/storage: Simplify code, correct content hashing --- swarm/storage/resource_ens.go | 1 + swarm/storage/resource_test.go | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/swarm/storage/resource_ens.go b/swarm/storage/resource_ens.go index 0a4500309d..f8f34919a2 100644 --- a/swarm/storage/resource_ens.go +++ b/swarm/storage/resource_ens.go @@ -36,6 +36,7 @@ func NewENSValidator(contractaddress common.Address, backend bind.ContractBacken if err != nil { return nil, err } + validator.hashlength = len(ens.EnsNode(dbDirName).Bytes()) return validator, nil } diff --git a/swarm/storage/resource_test.go b/swarm/storage/resource_test.go index d130709428..205edfe3b6 100644 --- a/swarm/storage/resource_test.go +++ b/swarm/storage/resource_test.go @@ -333,6 +333,15 @@ func setupTest(contractbackend bind.ContractBackend, validator ResourceValidator } } + if validator == nil { + // create a new signer, which creates the private key + signer, err = newTestSigner() + if err != nil { + return + } + validator = NewGenericValidator(testHashFunc, signer.signContent) + } + // temp datadir datadir, err = ioutil.TempDir("", "rh") if err != nil { From cf191d30a9703d5e57756507b2724bedc57d80f6 Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 18 Jan 2018 03:19:29 +0100 Subject: [PATCH 052/107] swarm/storage: Remove signatures from non-validated resources --- swarm/storage/resource_ens.go | 1 - swarm/storage/resource_test.go | 9 --------- 2 files changed, 10 deletions(-) diff --git a/swarm/storage/resource_ens.go b/swarm/storage/resource_ens.go index f8f34919a2..0a4500309d 100644 --- a/swarm/storage/resource_ens.go +++ b/swarm/storage/resource_ens.go @@ -36,7 +36,6 @@ func NewENSValidator(contractaddress common.Address, backend bind.ContractBacken if err != nil { return nil, err } - validator.hashlength = len(ens.EnsNode(dbDirName).Bytes()) return validator, nil } diff --git a/swarm/storage/resource_test.go b/swarm/storage/resource_test.go index 205edfe3b6..d130709428 100644 --- a/swarm/storage/resource_test.go +++ b/swarm/storage/resource_test.go @@ -333,15 +333,6 @@ func setupTest(contractbackend bind.ContractBackend, validator ResourceValidator } } - if validator == nil { - // create a new signer, which creates the private key - signer, err = newTestSigner() - if err != nil { - return - } - validator = NewGenericValidator(testHashFunc, signer.signContent) - } - // temp datadir datadir, err = ioutil.TempDir("", "rh") if err != nil { From 13543f40c9afe9d9536bb782fc44185b90eca87a Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 18 Jan 2018 05:09:49 +0100 Subject: [PATCH 053/107] swarm: Add base api for mutable resources --- swarm/api/api.go | 30 +++++++++++--- swarm/api/api_test.go | 2 +- swarm/api/config_test.go | 4 -- swarm/api/http/server.go | 58 +++++++++++++++++++++++++++ swarm/api/http/server_test.go | 41 ++++++++++++++++++- swarm/api/uri.go | 6 ++- swarm/storage/resource.go | 28 ++++++++++++- swarm/testutil/http.go | 74 ++++++++++++++++++++++++++++++++--- 8 files changed, 224 insertions(+), 19 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 8c4bca2ec0..cc10fa2134 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -46,15 +46,17 @@ on top of the dpa it is the public interface of the dpa which is included in the ethereum stack */ type Api struct { - dpa *storage.DPA - dns Resolver + dpa *storage.DPA + dns Resolver + resource *storage.ResourceHandler } //the api constructor initialises -func NewApi(dpa *storage.DPA, dns Resolver) (self *Api) { +func NewApi(dpa *storage.DPA, dns Resolver, resourceHandler *storage.ResourceHandler) (self *Api) { self = &Api{ - dpa: dpa, - dns: dns, + dpa: dpa, + dns: dns, + resource: resourceHandler, } return } @@ -361,3 +363,21 @@ func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storag } return key, manifestEntryMap, nil } + +func (self *Api) DbLookupLatest(name string) (io.ReadSeeker, error) { + _, err := self.resource.LookupLatest(name, true) + if err != nil { + return nil, err + } + return bytes.NewReader(self.resource.GetData(name)), nil +} + +func (self *Api) DbCreate(name string, frequency uint64) (err error) { + _, err = self.resource.NewResource(name, frequency) + return err +} + +func (self *Api) DbUpdate(name string, data []byte) (storage.Key, uint32, uint32, error) { + key, err := self.resource.Update(name, data) + return key, self.resource.GetLastPeriod(name), self.resource.GetVersion(name), err +} diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index e673f76c42..57c8e88f29 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -40,7 +40,7 @@ func testApi(t *testing.T, f func(*Api)) { if err != nil { return } - api := NewApi(dpa, nil) + api := NewApi(dpa, nil, nil) dpa.Start() f(api) dpa.Stop() diff --git a/swarm/api/config_test.go b/swarm/api/config_test.go index 4851f19fc5..993388686b 100644 --- a/swarm/api/config_test.go +++ b/swarm/api/config_test.go @@ -55,10 +55,6 @@ func TestConfig(t *testing.T) { t.Fatal("Failed to correctly initialize SwapParams") } - if one.HiveParams.MaxPeersPerRequest != 5 { - t.Fatal("Failed to correctly initialize HiveParams") - } - if one.StoreParams.ChunkDbPath == one.Path { t.Fatal("Failed to correctly initialize StoreParams") } diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 74341899d2..440cb04cb8 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -21,6 +21,7 @@ package http import ( "archive/tar" + "bytes" "encoding/json" "errors" "fmt" @@ -290,6 +291,56 @@ func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) { fmt.Fprint(w, newKey) } +func (s *Server) HandlePostDb(w http.ResponseWriter, r *Request) { + if r.ContentLength == 0 { + frequency, err := strconv.ParseUint(r.uri.Path, 10, 64) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + http.ServeContent(w, &r.Request, "", time.Now(), bytes.NewReader([]byte(err.Error()))) + return + } + err = s.api.DbCreate(r.uri.Addr, frequency) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + } else { + data, err := ioutil.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + _, _, _, err = s.api.DbUpdate(r.uri.Addr, data) + if err != nil { + w.WriteHeader(http.StatusUnauthorized) + http.ServeContent(w, &r.Request, "", time.Now(), bytes.NewReader([]byte(err.Error()))) + return + } + } + w.WriteHeader(http.StatusOK) +} + +func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { + w.Header().Set("Content-Type", "application/octet-stream") + + var params []string + if len(r.uri.Path) > 0 { + params = strings.Split(r.uri.Path, "/") + } + switch len(params) { + case 0: + data, err := s.api.DbLookupLatest(r.uri.Addr) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + break + } + http.ServeContent(w, &r.Request, "", time.Now(), data) + break + default: + w.WriteHeader(http.StatusBadRequest) + } +} + // HandleGet handles a GET request to // - bzz-raw:// and responds with the raw content stored at the // given storage key @@ -604,6 +655,8 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { case "POST": if uri.Raw() || uri.DeprecatedRaw() { s.HandlePostRaw(w, req) + } else if uri.Db() { + s.HandlePostDb(w, req) } else { s.HandlePostFiles(w, req) } @@ -644,6 +697,11 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + if uri.Db() { + s.HandleGetDb(w, req) + return + } + s.HandleGetFile(w, req) default: diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 305d5cf7db..06d4660d6b 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -22,17 +22,57 @@ import ( "fmt" "io/ioutil" "net/http" + "os" "strings" "sync" "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" swarm "github.com/ethereum/go-ethereum/swarm/api/client" "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/testutil" ) +func init() { + log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) +} + +func TestBzzGetDb(t *testing.T) { + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() + + url := srv.URL + "/bzz-db:/foo/42" + resp, err := http.Post(url, "application/octet-stream", nil) + if err != nil { + fmt.Printf("err: %v\n", err) + return + } + b, err := ioutil.ReadAll(resp.Body) + fmt.Printf("Create: %s : %s\n", resp.Status, b) + + url = srv.URL + "/bzz-db:/foo" + data := []byte("foo") + resp, err = http.Post(url, "application/octet-stream", bytes.NewReader(data)) + if err != nil { + fmt.Printf("err: %v\n", err) + return + } + b, err = ioutil.ReadAll(resp.Body) + fmt.Printf("Update: %s : %s\n", resp.Status, b) + + url = srv.URL + "/bzz-db:/foo" + resp, err = http.Get(url) + if err != nil { + fmt.Printf("err: %v\n", err) + return + } + b, err = ioutil.ReadAll(resp.Body) + fmt.Printf("Get: %s : %s\n", resp.Status, b) + +} + func TestBzzGetPath(t *testing.T) { var err error @@ -258,7 +298,6 @@ func TestBzzGetPath(t *testing.T) { t.Fatalf("Non-Hash response body does not match, expected: %v, got: %v", nonhashresponses[i], string(respbody)) } } - } // TestBzzRootRedirect tests that getting the root path of a manifest without diff --git a/swarm/api/uri.go b/swarm/api/uri.go index d8aafedf41..9b786045c7 100644 --- a/swarm/api/uri.go +++ b/swarm/api/uri.go @@ -69,7 +69,7 @@ func Parse(rawuri string) (*URI, error) { // check the scheme is valid switch uri.Scheme { - case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzz-hash", "bzzr", "bzzi": + case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzz-hash", "bzzr", "bzzi", "bzz-db": default: return nil, fmt.Errorf("unknown scheme %q", u.Scheme) } @@ -92,6 +92,10 @@ func Parse(rawuri string) (*URI, error) { return uri, nil } +func (u *URI) Db() bool { + return u.Scheme == "bzz-db" +} + func (u *URI) Raw() bool { return u.Scheme == "bzz-raw" } diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 3d6c653716..1e00c1dcf7 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -44,8 +44,8 @@ type resource struct { } // TODO Expire content after a defined period (to force resync) -func (r *resource) isSynced() bool { - return !r.updated.IsZero() +func (self *resource) isSynced() bool { + return !self.updated.IsZero() } // Implement to activate validation of resource updates @@ -166,6 +166,30 @@ func NewResourceHandler(datadir string, cloudStore CloudStore, rpcClient *rpc.Cl return rh, nil } +func (self *ResourceHandler) GetData(name string) []byte { + rsrc := self.getResource(name) + if rsrc == nil { + return nil + } + return rsrc.data +} + +func (self *ResourceHandler) GetLastPeriod(name string) uint32 { + rsrc := self.getResource(name) + if rsrc == nil { + return 0 + } + return rsrc.lastPeriod +} + +func (self *ResourceHandler) GetVersion(name string) uint32 { + rsrc := self.getResource(name) + if rsrc == nil { + return 0 + } + return rsrc.version +} + // \TODO should be hashsize * branches from the chosen chunker, implement with dpa func (self *ResourceHandler) chunkSize() int64 { return chunkSize diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go index b1dd4d4e65..fbe52e1111 100644 --- a/swarm/testutil/http.go +++ b/swarm/testutil/http.go @@ -17,11 +17,15 @@ package testutil import ( + "crypto/ecdsa" "io/ioutil" "net/http/httptest" "os" + "path/filepath" + "strconv" "testing" + "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm/api" httpapi "github.com/ethereum/go-ethereum/swarm/api/http" "github.com/ethereum/go-ethereum/swarm/storage" @@ -38,7 +42,7 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { CacheCapacity: 5000, Radius: 0, } - localStore, err := storage.NewLocalStore(storage.MakeHashFunc("SHA3"), storeparams, nil) + localStore, err := storage.NewLocalStore(storage.MakeHashFunc(storage.SHA3Hash), storeparams) if err != nil { os.RemoveAll(dir) t.Fatal(err) @@ -49,20 +53,59 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { ChunkStore: localStore, } dpa.Start() - a := api.NewApi(dpa, nil) + + // mutable resources test setup + resourcedir, err := ioutil.TempDir("", "swarm-resource-test") + if err != nil { + t.Fatal(err) + } + ipcpath := filepath.Join(resourcedir, "test.ipc") + ipcl, err := rpc.CreateIPCListener(ipcpath) + if err != nil { + t.Fatal(err) + } + rpcserver := rpc.NewServer() + rpcserver.RegisterName("eth", &FakeRPC{}) + go func() { + rpcserver.ServeListener(ipcl) + }() + rpcClean := func() { + rpcserver.Stop() + } + + // connect to fake rpc + rpcclient, err := rpc.Dial(ipcpath) + if err != nil { + t.Fatal(err) + } + rh, err := storage.NewResourceHandler(resourcedir, &testCloudStore{}, rpcclient, nil) + if err != nil { + t.Fatal(err) + } + + a := api.NewApi(dpa, nil, rh) srv := httptest.NewServer(httpapi.NewServer(a)) return &TestSwarmServer{ Server: srv, Dpa: dpa, dir: dir, + hasher: storage.MakeHashFunc("SHA3")(), + cleanup: func() { + rh.Close() + rpcClean() + os.RemoveAll(dir) + os.RemoveAll(resourcedir) + }, } } type TestSwarmServer struct { *httptest.Server - - Dpa *storage.DPA - dir string + hasher storage.SwarmHash + privatekey *ecdsa.PrivateKey + Dpa *storage.DPA + dir string + cleanup func() } func (t *TestSwarmServer) Close() { @@ -70,3 +113,24 @@ func (t *TestSwarmServer) Close() { t.Dpa.Stop() os.RemoveAll(t.dir) } + +type testCloudStore struct { +} + +func (c *testCloudStore) Store(*storage.Chunk) { +} + +func (c *testCloudStore) Deliver(*storage.Chunk) { +} + +func (c *testCloudStore) Retrieve(*storage.Chunk) { +} + +// for faking the rpc service, since we don't need the whole node stack +type FakeRPC struct { + blocknumber uint64 +} + +func (r *FakeRPC) BlockNumber() (string, error) { + return strconv.FormatUint(r.blocknumber, 10), nil +} From 81ec1f9694f84178895c134634d0702eae7d8e6f Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 19 Jan 2018 17:08:04 +0100 Subject: [PATCH 054/107] swarm/api: Add all lookup types + public getblock --- swarm/api/api.go | 19 +++++++++++++++++-- swarm/api/http/server.go | 35 +++++++++++++++++++++++++++++------ swarm/api/uri.go | 6 +++++- swarm/storage/resource.go | 8 ++++---- 4 files changed, 55 insertions(+), 13 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index cc10fa2134..827e89ad1d 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -364,8 +364,23 @@ func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storag return key, manifestEntryMap, nil } -func (self *Api) DbLookupLatest(name string) (io.ReadSeeker, error) { - _, err := self.resource.LookupLatest(name, true) +// Look up mutable resource updates at specific periods and versions +func (self *Api) DbLookup(name string, period uint32, version uint32) (io.ReadSeeker, error) { + var err error + if version != 0 { + if period == 0 { + currentblocknumber, err := self.resource.GetBlock() + if err != nil { + return nil, fmt.Errorf("Could not determine latest block: %v", err) + } + period = self.resource.BlockToPeriod(name, currentblocknumber) + } + _, err = self.resource.LookupVersion(name, period, version, true) + } else if period != 0 { + _, err = self.resource.LookupHistorical(name, period, true) + } else { + _, err = self.resource.LookupLatest(name, true) + } if err != nil { return nil, err } diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 440cb04cb8..bcdb4d2e59 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -320,25 +320,48 @@ func (s *Server) HandlePostDb(w http.ResponseWriter, r *Request) { w.WriteHeader(http.StatusOK) } +// Retrieve mutable resource updates: +// bzz-db[-[immutable|-raw]]:// - get latest update +// bzz-db[-[immutable|-raw]]:/// - get latest update on period n +// bzz-db[-[immutable|-raw]]://// - get update version m of period n func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { w.Header().Set("Content-Type", "application/octet-stream") + key, err := s.api.Resolve(r.uri) + if err != nil { + s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + return + } + _ = key + var params []string if len(r.uri.Path) > 0 { params = strings.Split(r.uri.Path, "/") } switch len(params) { case 0: - data, err := s.api.DbLookupLatest(r.uri.Addr) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - break - } - http.ServeContent(w, &r.Request, "", time.Now(), data) + data, err := s.api.DbLookup(r.uri.Addr) + break + case 2: + strconv.ParseUint(params[1], 10, 32) + case 1: + strconv.ParseUint(params[0], 10, 32) break default: w.WriteHeader(http.StatusBadRequest) + err = "params 0-2" } + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + wrappedData := wrapDbContent(data, s.uri.Scheme) + http.ServeContent(w, &r.Request, "", time.Now(), data) + +} + +func wrapDbContent(data io.Reader, scheme *string) io.Reader { + } // HandleGet handles a GET request to diff --git a/swarm/api/uri.go b/swarm/api/uri.go index 9b786045c7..c7a929f0f2 100644 --- a/swarm/api/uri.go +++ b/swarm/api/uri.go @@ -69,7 +69,7 @@ func Parse(rawuri string) (*URI, error) { // check the scheme is valid switch uri.Scheme { - case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzz-hash", "bzzr", "bzzi", "bzz-db": + case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzz-hash", "bzzr", "bzzi", "bzz-db", "bzz-db-raw": default: return nil, fmt.Errorf("unknown scheme %q", u.Scheme) } @@ -96,6 +96,10 @@ func (u *URI) Db() bool { return u.Scheme == "bzz-db" } +func (u *URI) DbRaw() bool { + return u.Scheme == "bzz-db-raw" +} + func (u *URI) Raw() bool { return u.Scheme == "bzz-raw" } diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 1e00c1dcf7..33845da40d 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -231,7 +231,7 @@ func (self *ResourceHandler) NewResource(name string, frequency uint64) (*resour } // get our blockheight at this time - currentblock, err := self.getBlock() + currentblock, err := self.GetBlock() if err != nil { return nil, err } @@ -310,7 +310,7 @@ func (self *ResourceHandler) LookupLatest(name string, refresh bool) (*resource, if err != nil { return nil, err } - currentblock, err := self.getBlock() + currentblock, err := self.GetBlock() if err != nil { return nil, err } @@ -492,7 +492,7 @@ func (self *ResourceHandler) Update(name string, data []byte) (Key, error) { } // get our blockheight at this time and the next block of the update period - currentblock, err := self.getBlock() + currentblock, err := self.GetBlock() if err != nil { return nil, err } @@ -560,7 +560,7 @@ func (self *ResourceHandler) Close() { self.ChunkStore.Close() } -func (self *ResourceHandler) getBlock() (uint64, error) { +func (self *ResourceHandler) GetBlock() (uint64, error) { // get the block height and convert to uint64 var currentblock string err := self.rpcClient.Call(¤tblock, "eth_blockNumber") From 1bf4f4a37f02b57e1a90929c10413fa47b1d42d1 Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 19 Jan 2018 18:28:10 +0100 Subject: [PATCH 055/107] swarm/api: Add raw form to api+server WIP --- swarm/api/api.go | 24 ++++++++---- swarm/api/http/server.go | 39 ++++++++++++-------- swarm/api/http/server_test.go | 15 ++++++-- swarm/storage/resource.go | 69 ++++++++++++++++++++++++----------- swarm/storage/resource_ens.go | 6 +++ swarm/testutil/http.go | 2 +- 6 files changed, 105 insertions(+), 50 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 827e89ad1d..cfc26aeff5 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -46,9 +46,9 @@ on top of the dpa it is the public interface of the dpa which is included in the ethereum stack */ type Api struct { + resource *storage.ResourceHandler dpa *storage.DPA dns Resolver - resource *storage.ResourceHandler } //the api constructor initialises @@ -365,7 +365,7 @@ func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storag } // Look up mutable resource updates at specific periods and versions -func (self *Api) DbLookup(name string, period uint32, version uint32) (io.ReadSeeker, error) { +func (self *Api) DbLookup(key storage.Key, name string, period uint32, version uint32) (io.ReadSeeker, error) { var err error if version != 0 { if period == 0 { @@ -375,16 +375,20 @@ func (self *Api) DbLookup(name string, period uint32, version uint32) (io.ReadSe } period = self.resource.BlockToPeriod(name, currentblocknumber) } - _, err = self.resource.LookupVersion(name, period, version, true) + _, err = self.resource.LookupVersionByName(name, period, version, true) } else if period != 0 { - _, err = self.resource.LookupHistorical(name, period, true) + _, err = self.resource.LookupHistoricalByName(name, period, true) } else { - _, err = self.resource.LookupLatest(name, true) + _, err = self.resource.LookupLatestByName(name, true) } if err != nil { return nil, err } - return bytes.NewReader(self.resource.GetData(name)), nil + data, err := self.resource.GetData(name) + if err != nil { + return nil, err + } + return bytes.NewReader(data), nil } func (self *Api) DbCreate(name string, frequency uint64) (err error) { @@ -394,5 +398,11 @@ func (self *Api) DbCreate(name string, frequency uint64) (err error) { func (self *Api) DbUpdate(name string, data []byte) (storage.Key, uint32, uint32, error) { key, err := self.resource.Update(name, data) - return key, self.resource.GetLastPeriod(name), self.resource.GetVersion(name), err + period, _ := self.resource.GetLastPeriod(name) + version, _ := self.resource.GetVersion(name) + return key, period, version, err +} + +func (self *Api) DbHashSize() int { + return self.resource.HashSize() } diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index bcdb4d2e59..9e3c7711c1 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -332,36 +332,42 @@ func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) return } - _ = key var params []string if len(r.uri.Path) > 0 { params = strings.Split(r.uri.Path, "/") } + var period uint64 + var version uint64 + var data io.ReadSeeker switch len(params) { case 0: - data, err := s.api.DbLookup(r.uri.Addr) + data, err = s.api.DbLookup(key, r.uri.Addr, 0, 0) break case 2: - strconv.ParseUint(params[1], 10, 32) + version, err = strconv.ParseUint(params[1], 10, 32) + if err != nil { + break + } case 1: - strconv.ParseUint(params[0], 10, 32) + period, err = strconv.ParseUint(params[0], 10, 32) + if err != nil { + break + } + data, err = s.api.DbLookup(key, r.uri.Addr, uint32(period), uint32(version)) break default: w.WriteHeader(http.StatusBadRequest) - err = "params 0-2" + err = fmt.Errorf("params 0-2") } if err != nil { w.WriteHeader(http.StatusInternalServerError) return } - wrappedData := wrapDbContent(data, s.uri.Scheme) + if !r.uri.DbRaw() { + + } http.ServeContent(w, &r.Request, "", time.Now(), data) - -} - -func wrapDbContent(data io.Reader, scheme *string) io.Reader { - } // HandleGet handles a GET request to @@ -705,6 +711,12 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.HandleDelete(w, req) case "GET": + + if uri.Db() || uri.DbRaw() { + s.HandleGetDb(w, req) + return + } + if uri.Raw() || uri.Hash() || uri.DeprecatedRaw() { s.HandleGet(w, req) return @@ -720,11 +732,6 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - if uri.Db() { - s.HandleGetDb(w, req) - return - } - s.HandleGetFile(w, req) default: diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 06d4660d6b..888445d88d 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -18,6 +18,7 @@ package http_test import ( "bytes" + "crypto/rand" "errors" "fmt" "io/ioutil" @@ -43,7 +44,14 @@ func TestBzzGetDb(t *testing.T) { srv := testutil.NewTestSwarmServer(t) defer srv.Close() - url := srv.URL + "/bzz-db:/foo/42" + keybytes := make([]byte, common.HashLength) // nearest we get to source of info + _, err := rand.Read(keybytes) + if err != nil { + fmt.Printf("err: %v\n", err) + return + } + + url := fmt.Sprintf("%s/bzz-db:/%s/42", srv.URL, fmt.Sprintf("%x", keybytes)) resp, err := http.Post(url, "application/octet-stream", nil) if err != nil { fmt.Printf("err: %v\n", err) @@ -52,7 +60,7 @@ func TestBzzGetDb(t *testing.T) { b, err := ioutil.ReadAll(resp.Body) fmt.Printf("Create: %s : %s\n", resp.Status, b) - url = srv.URL + "/bzz-db:/foo" + url = fmt.Sprintf("%s/bzz-db:/%s", srv.URL, fmt.Sprintf("%x", keybytes)) data := []byte("foo") resp, err = http.Post(url, "application/octet-stream", bytes.NewReader(data)) if err != nil { @@ -62,7 +70,7 @@ func TestBzzGetDb(t *testing.T) { b, err = ioutil.ReadAll(resp.Body) fmt.Printf("Update: %s : %s\n", resp.Status, b) - url = srv.URL + "/bzz-db:/foo" + url = fmt.Sprintf("%s/bzz-db-raw:/%s", srv.URL, fmt.Sprintf("%x", keybytes)) resp, err = http.Get(url) if err != nil { fmt.Printf("err: %v\n", err) @@ -70,7 +78,6 @@ func TestBzzGetDb(t *testing.T) { } b, err = ioutil.ReadAll(resp.Body) fmt.Printf("Get: %s : %s\n", resp.Status, b) - } func TestBzzGetPath(t *testing.T) { diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 33845da40d..b76117b336 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -51,6 +51,7 @@ func (self *resource) isSynced() bool { // Implement to activate validation of resource updates // Specifically signing data and verification of signatures type ResourceValidator interface { + hashSize() int checkAccess(string, common.Address) (bool, error) nameHash(string) common.Hash // nameHashFunc sign(common.Hash) (Signature, error) // SignFunc @@ -166,28 +167,34 @@ func NewResourceHandler(datadir string, cloudStore CloudStore, rpcClient *rpc.Cl return rh, nil } -func (self *ResourceHandler) GetData(name string) []byte { - rsrc := self.getResource(name) - if rsrc == nil { - return nil - } - return rsrc.data +func (self *ResourceHandler) HashSize() int { + return self.validator.hashSize() } -func (self *ResourceHandler) GetLastPeriod(name string) uint32 { +// get data from current resource +func (self *ResourceHandler) GetData(name string) ([]byte, error) { rsrc := self.getResource(name) - if rsrc == nil { - return 0 + if rsrc == nil || !rsrc.isSynced() { + return nil, fmt.Errorf("Resource does not exist or is not synced") } - return rsrc.lastPeriod + return rsrc.data, nil } -func (self *ResourceHandler) GetVersion(name string) uint32 { +func (self *ResourceHandler) GetLastPeriod(name string) (uint32, error) { rsrc := self.getResource(name) - if rsrc == nil { - return 0 + + if rsrc == nil || !rsrc.isSynced() { + return 0, fmt.Errorf("Resource does not exist or is not synced") } - return rsrc.version + return rsrc.lastPeriod, nil +} + +func (self *ResourceHandler) GetVersion(name string) (uint32, error) { + rsrc := self.getResource(name) + if rsrc == nil || !rsrc.isSynced() { + return 0, fmt.Errorf("Resource does not exist or is not synced") + } + return rsrc.version, nil } // \TODO should be hashsize * branches from the chosen chunker, implement with dpa @@ -269,8 +276,14 @@ func (self *ResourceHandler) NewResource(name string, frequency uint64) (*resour // root chunk. // It is the callers responsibility to make sure that this chunk exists (if the resource // update root data was retrieved externally, it typically doesn't) -func (self *ResourceHandler) LookupVersion(name string, period uint32, version uint32, refresh bool) (*resource, error) { - rsrc, err := self.loadResource(name, refresh) +// +// +func (self *ResourceHandler) LookupVersionByName(name string, period uint32, version uint32, refresh bool) (*resource, error) { + return self.LookupVersion(self.nameHash(name), name, period, version, refresh) +} + +func (self *ResourceHandler) LookupVersion(nameHash common.Hash, name string, period uint32, version uint32, refresh bool) (*resource, error) { + rsrc, err := self.loadResource(nameHash, name, refresh) if err != nil { return nil, err } @@ -285,8 +298,12 @@ func (self *ResourceHandler) LookupVersion(name string, period uint32, version u // and returned. // // See also (*ResourceHandler).LookupVersion -func (self *ResourceHandler) LookupHistorical(name string, period uint32, refresh bool) (*resource, error) { - rsrc, err := self.loadResource(name, refresh) +func (self *ResourceHandler) LookupHistoricalByName(name string, period uint32, refresh bool) (*resource, error) { + return self.LookupHistorical(self.nameHash(name), name, period, refresh) +} + +func (self *ResourceHandler) LookupHistorical(nameHash common.Hash, name string, period uint32, refresh bool) (*resource, error) { + rsrc, err := self.loadResource(nameHash, name, refresh) if err != nil { return nil, err } @@ -303,10 +320,14 @@ func (self *ResourceHandler) LookupHistorical(name string, period uint32, refres // Version iteration is done as in (*ResourceHandler).LookupHistorical // // See also (*ResourceHandler).LookupHistorical -func (self *ResourceHandler) LookupLatest(name string, refresh bool) (*resource, error) { +func (self *ResourceHandler) LookupLatestByName(name string, refresh bool) (*resource, error) { + return self.LookupLatest(self.nameHash(name), name, refresh) +} + +func (self *ResourceHandler) LookupLatest(nameHash common.Hash, name string, refresh bool) (*resource, error) { // get our blockheight at this time and the next block of the update period - rsrc, err := self.loadResource(name, refresh) + rsrc, err := self.loadResource(nameHash, name, refresh) if err != nil { return nil, err } @@ -362,7 +383,11 @@ func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint3 } // load existing mutable resource into resource struct -func (self *ResourceHandler) loadResource(name string, refresh bool) (*resource, error) { +func (self *ResourceHandler) loadResource(nameHash common.Hash, name string, refresh bool) (*resource, error) { + + if name == "" { + name = nameHash.Hex() + } // if the resource is not known to this session we must load it // if refresh is set, we force load @@ -374,7 +399,7 @@ func (self *ResourceHandler) loadResource(name string, refresh bool) (*resource, return nil, fmt.Errorf("Invalid name '%s'", name) } rsrc.name = &name - rsrc.nameHash = self.nameHash(name) + rsrc.nameHash = nameHash // get the root info chunk and update the cached value chunk, err := self.Get(Key(rsrc.nameHash[:])) diff --git a/swarm/storage/resource_ens.go b/swarm/storage/resource_ens.go index 0a4500309d..df008efa17 100644 --- a/swarm/storage/resource_ens.go +++ b/swarm/storage/resource_ens.go @@ -10,6 +10,7 @@ import ( type baseValidator struct { signFunc SignFunc + hashsize int } func (b *baseValidator) sign(datahash common.Hash) (signature Signature, err error) { @@ -19,6 +20,10 @@ func (b *baseValidator) sign(datahash common.Hash) (signature Signature, err err return b.signFunc(datahash) } +func (b *baseValidator) hashSize() int { + return b.hashsize +} + // ENS validation of mutable resource owners type ENSValidator struct { *baseValidator @@ -30,6 +35,7 @@ func NewENSValidator(contractaddress common.Address, backend bind.ContractBacken validator := &ENSValidator{ baseValidator: &baseValidator{ signFunc: signFunc, + hashsize: common.HashLength, }, } validator.api, err = ens.NewENS(transactOpts, contractaddress, backend) diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go index fbe52e1111..dcba90c9c5 100644 --- a/swarm/testutil/http.go +++ b/swarm/testutil/http.go @@ -89,7 +89,7 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { Server: srv, Dpa: dpa, dir: dir, - hasher: storage.MakeHashFunc("SHA3")(), + hasher: storage.MakeHashFunc(storage.SHA3Hash)(), cleanup: func() { rh.Close() rpcClean() From 249e1a8aaa6c384ee11fe0a58744d084fa6c6315 Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 19 Jan 2018 20:39:49 +0100 Subject: [PATCH 056/107] swarm, cmd/swarm, ethclient: Fullstack mut.rsrc. w api Add ethclient.Client.BlockNumber method for access to current block number (needed by ResourceHandler) Add placeholder manifest support --- cmd/swarm/main.go | 2 +- ethclient/ethclient.go | 10 ++++++ swarm/api/api.go | 16 ++++++---- swarm/api/config.go | 58 ++++++++++++++++++---------------- swarm/api/http/server.go | 36 ++++++++++++++++++--- swarm/api/http/server_test.go | 10 ++++++ swarm/api/manifest.go | 3 +- swarm/storage/resource.go | 51 ++++++++++++++++++++---------- swarm/storage/resource_sign.go | 20 ++++++++++++ swarm/storage/resource_test.go | 35 +++++++++----------- swarm/swarm.go | 24 ++++++++++++-- swarm/testutil/http.go | 23 ++++++++------ 12 files changed, 197 insertions(+), 91 deletions(-) create mode 100644 swarm/storage/resource_sign.go diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 886895852f..4233c7c982 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -560,7 +560,7 @@ func registerBzzService(bzzconfig *bzzapi.Config, ctx *cli.Context, stack *node. } // In production, mockStore must be always nil. - return swarm.NewSwarm(ctx, swapClient, ensClient, bzzconfig, bzzconfig.SwapEnabled, bzzconfig.SyncEnabled, bzzconfig.Cors, bzzconfig.PssEnabled, nil) + return swarm.NewSwarm(ctx, swapClient, ensClient, bzzconfig, bzzconfig.SwapEnabled, bzzconfig.SyncEnabled, bzzconfig.Cors, bzzconfig.PssEnabled, bzzconfig.ResourceEnabled, nil) } //register within the ethereum node if err := stack.Register(boot); err != nil { diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 87a912901a..4b53e785d6 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -23,6 +23,7 @@ import ( "errors" "fmt" "math/big" + "strconv" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" @@ -76,6 +77,15 @@ type rpcBlock struct { UncleHashes []common.Hash `json:"uncles"` } +func (ec *Client) BlockNumber(ctx context.Context) (uint64, error) { + var number string + err := ec.c.CallContext(ctx, &number, "eth_blockNumber") + if err != nil { + return 0, err + } + return strconv.ParseUint(number, 10, 64) +} + func (ec *Client) getBlock(ctx context.Context, method string, args ...interface{}) (*types.Block, error) { var raw json.RawMessage err := ec.c.CallContext(ctx, &raw, method, args...) diff --git a/swarm/api/api.go b/swarm/api/api.go index cfc26aeff5..0e11028ec4 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -365,13 +365,13 @@ func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storag } // Look up mutable resource updates at specific periods and versions -func (self *Api) DbLookup(key storage.Key, name string, period uint32, version uint32) (io.ReadSeeker, error) { +func (self *Api) DbLookup(key storage.Key, name string, period uint32, version uint32) (storage.Key, io.ReadSeeker, int, error) { var err error if version != 0 { if period == 0 { currentblocknumber, err := self.resource.GetBlock() if err != nil { - return nil, fmt.Errorf("Could not determine latest block: %v", err) + return nil, nil, 0, fmt.Errorf("Could not determine latest block: %v", err) } period = self.resource.BlockToPeriod(name, currentblocknumber) } @@ -382,13 +382,13 @@ func (self *Api) DbLookup(key storage.Key, name string, period uint32, version u _, err = self.resource.LookupLatestByName(name, true) } if err != nil { - return nil, err + return nil, nil, 0, err } - data, err := self.resource.GetData(name) + key, data, err := self.resource.GetContent(name) if err != nil { - return nil, err + return nil, nil, 0, err } - return bytes.NewReader(data), nil + return key, bytes.NewReader(data), len(data), nil } func (self *Api) DbCreate(name string, frequency uint64) (err error) { @@ -406,3 +406,7 @@ func (self *Api) DbUpdate(name string, data []byte) (storage.Key, uint32, uint32 func (self *Api) DbHashSize() int { return self.resource.HashSize() } + +func (self *Api) DbIsValidated() bool { + return self.resource.IsValidated() +} diff --git a/swarm/api/config.go b/swarm/api/config.go index d4dba36094..31281f9bff 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -46,22 +46,23 @@ type Config struct { *network.HiveParams Swap *swap.SwapParams //*network.SyncParams - Contract common.Address - EnsRoot common.Address - EnsApi string - Path string - ListenAddr string - Port string - PublicKey string - BzzKey string - NetworkId uint64 - SwapEnabled bool - SyncEnabled bool - PssEnabled bool - SwapApi string - Cors string - BzzAccount string - BootNodes string + Contract common.Address + EnsRoot common.Address + EnsApi string + Path string + ListenAddr string + Port string + PublicKey string + BzzKey string + NetworkId uint64 + SwapEnabled bool + SyncEnabled bool + PssEnabled bool + ResourceEnabled bool + SwapApi string + Cors string + BzzAccount string + BootNodes string } //create a default config with all parameters to set to defaults @@ -72,18 +73,19 @@ func NewConfig() (self *Config) { ChunkerParams: storage.NewChunkerParams(), HiveParams: network.NewHiveParams(), //SyncParams: network.NewDefaultSyncParams(), - Swap: swap.NewDefaultSwapParams(), - ListenAddr: DefaultHTTPListenAddr, - Port: DefaultHTTPPort, - Path: node.DefaultDataDir(), - EnsApi: node.DefaultIPCEndpoint("geth"), - EnsRoot: ens.TestNetAddress, - NetworkId: network.NetworkID, - SwapEnabled: false, - SyncEnabled: true, - PssEnabled: true, - SwapApi: "", - BootNodes: "", + Swap: swap.NewDefaultSwapParams(), + ListenAddr: DefaultHTTPListenAddr, + Port: DefaultHTTPPort, + Path: node.DefaultDataDir(), + EnsApi: node.DefaultIPCEndpoint("geth"), + EnsRoot: ens.TestNetAddress, + NetworkId: network.NetworkID, + SwapEnabled: false, + SyncEnabled: true, + PssEnabled: true, + ResourceEnabled: true, + SwapApi: "", + BootNodes: "", } return diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 9e3c7711c1..26c59c313e 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -327,7 +327,7 @@ func (s *Server) HandlePostDb(w http.ResponseWriter, r *Request) { func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { w.Header().Set("Content-Type", "application/octet-stream") - key, err := s.api.Resolve(r.uri) + rootKey, err := s.api.Resolve(r.uri) if err != nil { s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) return @@ -337,12 +337,15 @@ func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { if len(r.uri.Path) > 0 { params = strings.Split(r.uri.Path, "/") } + var updateKey storage.Key var period uint64 var version uint64 var data io.ReadSeeker + var dataLength int + now := time.Now() switch len(params) { case 0: - data, err = s.api.DbLookup(key, r.uri.Addr, 0, 0) + updateKey, data, dataLength, err = s.api.DbLookup(rootKey, r.uri.Addr, 0, 0) break case 2: version, err = strconv.ParseUint(params[1], 10, 32) @@ -354,7 +357,7 @@ func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { if err != nil { break } - data, err = s.api.DbLookup(key, r.uri.Addr, uint32(period), uint32(version)) + updateKey, data, dataLength, err = s.api.DbLookup(rootKey, r.uri.Addr, uint32(period), uint32(version)) break default: w.WriteHeader(http.StatusBadRequest) @@ -365,9 +368,32 @@ func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { return } if !r.uri.DbRaw() { - + entry := api.ManifestEntry{ + Hash: rootKey.Hex(), + Path: updateKey.Hex(), + ContentType: api.DbManifestType, + Size: int64(dataLength), + ModTime: now, + Status: http.StatusOK, + } + mode := (6 << 2) | (4 << 1) | 4 + if s.api.DbIsValidated() { + mode |= 2 << 1 + } + entry.Mode = int64(mode) + manifest := api.Manifest{ + Entries: []api.ManifestEntry{ + entry, + }, + } + manifestJson, err := json.Marshal(manifest) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + data = bytes.NewReader(manifestJson) } - http.ServeContent(w, &r.Request, "", time.Now(), data) + http.ServeContent(w, &r.Request, "", now, data) } // HandleGet handles a GET request to diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 888445d88d..e2165f1d61 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -78,6 +78,16 @@ func TestBzzGetDb(t *testing.T) { } b, err = ioutil.ReadAll(resp.Body) fmt.Printf("Get: %s : %s\n", resp.Status, b) + + url = fmt.Sprintf("%s/bzz-db:/%s", srv.URL, fmt.Sprintf("%x", keybytes)) + resp, err = http.Get(url) + if err != nil { + fmt.Printf("err: %v\n", err) + return + } + b, err = ioutil.ReadAll(resp.Body) + fmt.Printf("Get: %s : %s\n", resp.Status, b) + } func TestBzzGetPath(t *testing.T) { diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index 685a300fca..46b55b3b7a 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -33,7 +33,8 @@ import ( ) const ( - ManifestType = "application/bzz-manifest+json" + ManifestType = "application/bzz-manifest+json" + DbManifestType = "application/bzz-db-manifest+json" ) // Manifest represents a swarm manifest diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index b76117b336..23e1a6336b 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -1,10 +1,10 @@ package storage import ( + "context" "encoding/binary" "fmt" "path/filepath" - "strconv" "sync" "time" @@ -12,8 +12,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rpc" ) const ( @@ -37,6 +37,7 @@ type resource struct { nameHash common.Hash startBlock uint64 lastPeriod uint32 + lastKey Key frequency uint64 version uint32 data []byte @@ -114,26 +115,30 @@ type ResourceValidator interface { // stored using a separate store, and forwarding/syncing protocols carry per-chunk // flags to tell whether the chunk can be validated or not; if not it is to be // treated as a resource update chunk. +// +// TODO: Include modtime in chunk data + signature type ResourceHandler struct { ChunkStore validator ResourceValidator - rpcClient *rpc.Client + ethClient *ethclient.Client resources map[string]*resource hashLock sync.Mutex resourceLock sync.RWMutex hasher SwarmHash nameHash nameHashFunc storeTimeout time.Duration + ctx context.Context + cancelFunc func() } // Create or open resource update chunk store // // If validator is nil, signature and access validation will be deactivated -func NewResourceHandler(datadir string, cloudStore CloudStore, rpcClient *rpc.Client, validator ResourceValidator) (*ResourceHandler, error) { +func NewResourceHandler(datadir string, cloudStore CloudStore, ethClient *ethclient.Client, validator ResourceValidator) (*ResourceHandler, error) { hashfunc := MakeHashFunc(SHA3Hash) - path := filepath.Join(datadir, dbDirName) + path := filepath.Join(datadir, DbDirName) dbStore, err := NewDbStore(datadir, hashfunc, singletonSwarmDbCapacity, 0) if err != nil { return nil, err @@ -143,6 +148,7 @@ func NewResourceHandler(datadir string, cloudStore CloudStore, rpcClient *rpc.Cl DbStore: dbStore, } + ctx, cancel := context.WithCancel(context.Background()) rh := &ResourceHandler{ ChunkStore: newResourceChunkStore(path, hashfunc, localStore, cloudStore), rpcClient: rpcClient, @@ -150,6 +156,8 @@ func NewResourceHandler(datadir string, cloudStore CloudStore, rpcClient *rpc.Cl hasher: hashfunc(), validator: validator, storeTimeout: defaultStoreTimeout, + ctx: ctx, + cancelFunc: cancel, } if rh.validator != nil { @@ -167,17 +175,22 @@ func NewResourceHandler(datadir string, cloudStore CloudStore, rpcClient *rpc.Cl return rh, nil } +func (self *ResourceHandler) IsValidated() bool { + return self.validator == nil +} + func (self *ResourceHandler) HashSize() int { return self.validator.hashSize() } // get data from current resource -func (self *ResourceHandler) GetData(name string) ([]byte, error) { + +func (self *ResourceHandler) GetContent(name string) (Key, []byte, error) { rsrc := self.getResource(name) if rsrc == nil || !rsrc.isSynced() { - return nil, fmt.Errorf("Resource does not exist or is not synced") + return nil, nil, fmt.Errorf("Resource does not exist or is not synced") } - return rsrc.data, nil + return rsrc.lastKey, rsrc.data, nil } func (self *ResourceHandler) GetLastPeriod(name string) (uint32, error) { @@ -430,6 +443,7 @@ func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk) ( if *rsrc.name != name { return nil, fmt.Errorf("Update belongs to '%s', but have '%s'", name, *rsrc.name) } + log.Trace("update", "name", *rsrc.name, "rootkey", rsrc.nameHash, "updatekey", chunk.Key, "period", period, "version", version) // only check signature if validator is present if self.validator != nil { digest := self.keyDataHash(chunk.Key, data) @@ -440,6 +454,7 @@ func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk) ( } // update our rsrcs entry map + rsrc.lastKey = chunk.Key rsrc.lastPeriod = period rsrc.version = version rsrc.updated = time.Now() @@ -582,20 +597,22 @@ func (self *ResourceHandler) Update(name string, data []byte) (Key, error) { // Closes the datastore. // Always call this at shutdown to avoid data corruption. func (self *ResourceHandler) Close() { + self.cancelFunc() self.ChunkStore.Close() } func (self *ResourceHandler) GetBlock() (uint64, error) { + return self.ethClient.BlockNumber(self.ctx) // get the block height and convert to uint64 - var currentblock string - err := self.rpcClient.Call(¤tblock, "eth_blockNumber") - if err != nil { - return 0, err - } - if currentblock == "0x0" { - return 0, nil - } - return strconv.ParseUint(currentblock, 10, 64) + // var currentblock string + // err := self.rpcClient.Call(¤tblock, "eth_blockNumber") + // if err != nil { + // return 0, err + // } + // if currentblock == "0x0" { + // return 0, nil + // } + // return strconv.ParseUint(currentblock, 10, 64) } // Calculate the period index (aka major version number) from a given block number diff --git a/swarm/storage/resource_sign.go b/swarm/storage/resource_sign.go new file mode 100644 index 0000000000..840e705ac1 --- /dev/null +++ b/swarm/storage/resource_sign.go @@ -0,0 +1,20 @@ +package storage + +import ( + "crypto/ecdsa" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" +) + +// matches the SignFunc type +func NewGenericResourceSigner(privKey *ecdsa.PrivateKey) SignFunc { + return func(data common.Hash) (signature Signature, err error) { + signaturebytes, err := crypto.Sign(data.Bytes(), privKey) + if err != nil { + return + } + copy(signature[:], signaturebytes) + return + } +} diff --git a/swarm/storage/resource_test.go b/swarm/storage/resource_test.go index d130709428..2da9b7704a 100644 --- a/swarm/storage/resource_test.go +++ b/swarm/storage/resource_test.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/contracts/ens/contract" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" ) @@ -211,8 +212,8 @@ func TestResourceHandler(t *testing.T) { // it will match on second iteration startblocknumber + (resourceFrequency * 3) fwdBlocks(int(resourceFrequency*2)-1, backend) - rh2, err := NewResourceHandler(datadir, &testCloudStore{}, rh.rpcClient, nil) - _, err = rh2.LookupLatest(safeName, true) + rh2, err := NewResourceHandler(datadir, &testCloudStore{}, rh.ethClient, nil) + _, err = rh2.LookupLatestByName(safeName, true) if err != nil { teardownTest(t, err) } @@ -230,7 +231,7 @@ func TestResourceHandler(t *testing.T) { log.Debug("Latest lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data) // specific block, latest version - rsrc, err := rh2.LookupHistorical(safeName, 3, true) + rsrc, err := rh2.LookupHistoricalByName(safeName, 3, true) if err != nil { teardownTest(t, err) } @@ -241,7 +242,7 @@ func TestResourceHandler(t *testing.T) { log.Debug("Historical lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data) // specific block, specific version - rsrc, err = rh2.LookupVersion(safeName, 3, 1, true) + rsrc, err = rh2.LookupVersionByName(safeName, 3, 1, true) if err != nil { teardownTest(t, err) } @@ -365,12 +366,14 @@ func setupTest(contractbackend bind.ContractBackend, validator ResourceValidator } // connect to fake rpc - rpcclient, err := rpc.Dial(ipcpath) + rpcClient, err := rpc.Dial(ipcpath) if err != nil { return } - rh, err = NewResourceHandler(datadir, &testCloudStore{}, rpcclient, validator) + ethClient := ethclient.NewClient(rpcClient) + + rh, err = NewResourceHandler(datadir, &testCloudStore{}, ethClient, validator) teardown = func(t *testing.T, err error) { cleanF() if err != nil { @@ -426,8 +429,9 @@ func setupENS(addr common.Address, transactOpts *bind.TransactOpts, sub string, // implementation of an external signer to pass to validator type testSigner struct { - privKey *ecdsa.PrivateKey - hasher SwarmHash + privKey *ecdsa.PrivateKey + hasher SwarmHash + signContent SignFunc } func newTestSigner() (*testSigner, error) { @@ -436,21 +440,12 @@ func newTestSigner() (*testSigner, error) { return nil, err } return &testSigner{ - privKey: privKey, - hasher: testHasher, + privKey: privKey, + hasher: testHasher, + signContent: NewGenericResourceSigner(privKey), }, nil } -// matches the SignFunc type -func (self *testSigner) signContent(data common.Hash) (signature Signature, err error) { - signaturebytes, err := crypto.Sign(data.Bytes(), self.privKey) - if err != nil { - return - } - copy(signature[:], signaturebytes) - return -} - type testCloudStore struct { } diff --git a/swarm/swarm.go b/swarm/swarm.go index 14826bb021..766c210d3d 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -22,6 +22,7 @@ import ( "crypto/ecdsa" "fmt" "net" + "path/filepath" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" @@ -82,7 +83,8 @@ func (self *Swarm) API() *SwarmAPI { // implements node.Service // If mockStore is not nil, it will be used as the storage for chunk data. // MockStore should be used only for testing. -func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *ethclient.Client, config *api.Config, swapEnabled, syncEnabled bool, cors string, pssEnabled bool, mockStore *mock.NodeStore) (self *Swarm, err error) { +func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *ethclient.Client, config *api.Config, swapEnabled, syncEnabled bool, cors string, pssEnabled bool, resourceEnabled bool, mockStore *mock.NodeStore) (self *Swarm, err error) { + if bytes.Equal(common.FromHex(config.PublicKey), storage.ZeroKey) { return nil, fmt.Errorf("empty public key") } @@ -158,7 +160,23 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e } log.Debug(fmt.Sprintf("-> Swarm Domain Name Registrar @ address %v", config.EnsRoot.Hex())) - self.api = api.NewApi(self.dpa, self.dns) + var resourceHandler *storage.ResourceHandler + // if use resource updates + if resourceEnabled { + var resourceValidator storage.ResourceValidator + if self.dns != nil { + resourceValidator, err = storage.NewENSValidator(config.EnsRoot, ensClient, transactOpts, storage.NewGenericResourceSigner(self.privateKey)) + if err != nil { + return nil, err + } + } + resourceHandler, err = storage.NewResourceHandler(filepath.Join(self.config.Path, storage.DbDirName), self.cloud, ensClient, resourceValidator) + if err != nil { + return nil, err + } + } + + self.api = api.NewApi(self.dpa, self.dns, resourceHandler) // Manifests for Smart Hosting log.Debug(fmt.Sprintf("-> Web3 virtual server API")) @@ -360,7 +378,7 @@ func NewLocalSwarm(datadir, port string) (self *Swarm, err error) { } self = &Swarm{ - api: api.NewApi(dpa, nil), + api: api.NewApi(dpa, nil, nil), config: config, } diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go index dcba90c9c5..4cec635936 100644 --- a/swarm/testutil/http.go +++ b/swarm/testutil/http.go @@ -25,6 +25,7 @@ import ( "strconv" "testing" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm/api" httpapi "github.com/ethereum/go-ethereum/swarm/api/http" @@ -55,30 +56,32 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { dpa.Start() // mutable resources test setup - resourcedir, err := ioutil.TempDir("", "swarm-resource-test") + resourceDir, err := ioutil.TempDir("", "swarm-resource-test") if err != nil { t.Fatal(err) } - ipcpath := filepath.Join(resourcedir, "test.ipc") - ipcl, err := rpc.CreateIPCListener(ipcpath) + ipcPath := filepath.Join(resourceDir, "test.ipc") + ipcl, err := rpc.CreateIPCListener(ipcPath) if err != nil { t.Fatal(err) } - rpcserver := rpc.NewServer() - rpcserver.RegisterName("eth", &FakeRPC{}) + rpcServer := rpc.NewServer() + rpcServer.RegisterName("eth", &FakeRPC{}) go func() { - rpcserver.ServeListener(ipcl) + rpcServer.ServeListener(ipcl) }() rpcClean := func() { - rpcserver.Stop() + rpcServer.Stop() } // connect to fake rpc - rpcclient, err := rpc.Dial(ipcpath) + rpcClient, err := rpc.Dial(ipcPath) if err != nil { t.Fatal(err) } - rh, err := storage.NewResourceHandler(resourcedir, &testCloudStore{}, rpcclient, nil) + ethClient := ethclient.NewClient(rpcClient) + + rh, err := storage.NewResourceHandler(resourceDir, &testCloudStore{}, ethClient, nil) if err != nil { t.Fatal(err) } @@ -94,7 +97,7 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { rh.Close() rpcClean() os.RemoveAll(dir) - os.RemoveAll(resourcedir) + os.RemoveAll(resourceDir) }, } } From a16d0af41dde358541357909d546fe23575cee5a Mon Sep 17 00:00:00 2001 From: lash Date: Sun, 21 Jan 2018 03:13:52 +0100 Subject: [PATCH 057/107] swarm/, cmd/swarm: Amend comments from @lmars PR 204 --- cmd/swarm/main.go | 2 +- ethclient/ethclient.go | 17 ++-- swarm/api/api.go | 10 +-- swarm/api/config.go | 6 +- swarm/api/config_test.go | 5 +- swarm/api/http/server.go | 39 +++++---- swarm/api/http/server_test.go | 37 +++++---- swarm/storage/resource.go | 33 ++++---- swarm/storage/resource_test.go | 142 ++++++++++++--------------------- swarm/swarm.go | 38 ++++----- swarm/testutil/http.go | 54 +++++-------- 11 files changed, 166 insertions(+), 217 deletions(-) diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 4233c7c982..3ae45d4c2f 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -560,7 +560,7 @@ func registerBzzService(bzzconfig *bzzapi.Config, ctx *cli.Context, stack *node. } // In production, mockStore must be always nil. - return swarm.NewSwarm(ctx, swapClient, ensClient, bzzconfig, bzzconfig.SwapEnabled, bzzconfig.SyncEnabled, bzzconfig.Cors, bzzconfig.PssEnabled, bzzconfig.ResourceEnabled, nil) + return swarm.NewSwarm(ctx, swapClient, ensClient, bzzconfig, nil) } //register within the ethereum node if err := stack.Register(boot); err != nil { diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 4b53e785d6..2d9553a7bd 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -23,7 +23,6 @@ import ( "errors" "fmt" "math/big" - "strconv" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" @@ -77,13 +76,19 @@ type rpcBlock struct { UncleHashes []common.Hash `json:"uncles"` } -func (ec *Client) BlockNumber(ctx context.Context) (uint64, error) { - var number string - err := ec.c.CallContext(ctx, &number, "eth_blockNumber") +func (ec *Client) BlockNumber(ctx context.Context) (big.Int, error) { + var numberstr string + number := &big.Int{} + err := ec.c.CallContext(ctx, &numberstr, "eth_blockNumber") if err != nil { - return 0, err + return *number, err } - return strconv.ParseUint(number, 10, 64) + var ok bool + number, ok = number.SetString(numberstr, 10) + if !ok { + err = errors.New("Failed to parse bigint") + } + return *number, err } func (ec *Client) getBlock(ctx context.Context, method string, args ...interface{}) (*types.Block, error) { diff --git a/swarm/api/api.go b/swarm/api/api.go index 0e11028ec4..1393fa4406 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -365,13 +365,13 @@ func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storag } // Look up mutable resource updates at specific periods and versions -func (self *Api) DbLookup(key storage.Key, name string, period uint32, version uint32) (storage.Key, io.ReadSeeker, int, error) { +func (self *Api) DbLookup(key storage.Key, name string, period uint32, version uint32) (storage.Key, []byte, error) { var err error if version != 0 { if period == 0 { currentblocknumber, err := self.resource.GetBlock() if err != nil { - return nil, nil, 0, fmt.Errorf("Could not determine latest block: %v", err) + return nil, nil, fmt.Errorf("Could not determine latest block: %v", err) } period = self.resource.BlockToPeriod(name, currentblocknumber) } @@ -382,13 +382,13 @@ func (self *Api) DbLookup(key storage.Key, name string, period uint32, version u _, err = self.resource.LookupLatestByName(name, true) } if err != nil { - return nil, nil, 0, err + return nil, nil, err } key, data, err := self.resource.GetContent(name) if err != nil { - return nil, nil, 0, err + return nil, nil, err } - return key, bytes.NewReader(data), len(data), nil + return key, data, nil } func (self *Api) DbCreate(name string, frequency uint64) (err error) { diff --git a/swarm/api/config.go b/swarm/api/config.go index 31281f9bff..93b386cc11 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -110,8 +110,8 @@ func (self *Config) Init(prvKey *ecdsa.PrivateKey) { self.PublicKey = pubkeyhex self.BzzKey = keyhex - self.Swap.Init(self.Contract, prvKey) - //self.SyncParams.Init(self.Path) - //self.HiveParams.Init(self.Path) + if self.SwapEnabled { + self.Swap.Init(self.Contract, prvKey) + } self.StoreParams.Init(self.Path) } diff --git a/swarm/api/config_test.go b/swarm/api/config_test.go index 993388686b..5a5f176c0b 100644 --- a/swarm/api/config_test.go +++ b/swarm/api/config_test.go @@ -49,12 +49,9 @@ func TestConfig(t *testing.T) { if one.PublicKey == "" { t.Fatal("Expected PublicKey to be set") } - - //the Init function should append subdirs to the given path - if one.Swap.PayProfile.Beneficiary == (common.Address{}) { + if one.Swap.PayProfile.Beneficiary == (common.Address{}) && one.SwapEnabled { t.Fatal("Failed to correctly initialize SwapParams") } - if one.StoreParams.ChunkDbPath == one.Path { t.Fatal("Failed to correctly initialize StoreParams") } diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 26c59c313e..2a8046f1c7 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -295,13 +295,12 @@ func (s *Server) HandlePostDb(w http.ResponseWriter, r *Request) { if r.ContentLength == 0 { frequency, err := strconv.ParseUint(r.uri.Path, 10, 64) if err != nil { - w.WriteHeader(http.StatusBadRequest) - http.ServeContent(w, &r.Request, "", time.Now(), bytes.NewReader([]byte(err.Error()))) + s.BadRequest(w, r, fmt.Sprintf("Cannot parse frequency parameter: %v", err)) return } err = s.api.DbCreate(r.uri.Addr, frequency) if err != nil { - w.WriteHeader(http.StatusInternalServerError) + s.Error(w, r, fmt.Errorf("Resource creation failed: %v", err)) return } } else { @@ -324,8 +323,8 @@ func (s *Server) HandlePostDb(w http.ResponseWriter, r *Request) { // bzz-db[-[immutable|-raw]]:// - get latest update // bzz-db[-[immutable|-raw]]:/// - get latest update on period n // bzz-db[-[immutable|-raw]]://// - get update version m of period n +// = ens name or hash func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { - w.Header().Set("Content-Type", "application/octet-stream") rootKey, err := s.api.Resolve(r.uri) if err != nil { @@ -340,34 +339,39 @@ func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { var updateKey storage.Key var period uint64 var version uint64 - var data io.ReadSeeker + var data []byte var dataLength int now := time.Now() switch len(params) { case 0: - updateKey, data, dataLength, err = s.api.DbLookup(rootKey, r.uri.Addr, 0, 0) - break + updateKey, data, err = s.api.DbLookup(rootKey, r.uri.Addr, 0, 0) case 2: version, err = strconv.ParseUint(params[1], 10, 32) if err != nil { break } + updateKey, data, err = s.api.DbLookup(rootKey, r.uri.Addr, uint32(period), uint32(version)) case 1: + version, err = strconv.ParseUint(params[1], 10, 32) + if err != nil { + break + } period, err = strconv.ParseUint(params[0], 10, 32) if err != nil { break } - updateKey, data, dataLength, err = s.api.DbLookup(rootKey, r.uri.Addr, uint32(period), uint32(version)) - break + updateKey, data, err = s.api.DbLookup(rootKey, r.uri.Addr, uint32(period), uint32(version)) default: - w.WriteHeader(http.StatusBadRequest) - err = fmt.Errorf("params 0-2") + s.BadRequest(w, r, fmt.Sprintf("Invalid mutable resource request")) + return } if err != nil { - w.WriteHeader(http.StatusInternalServerError) + s.Error(w, r, fmt.Errorf("Mutable resource lookup failed: %v", err)) return } if !r.uri.DbRaw() { + w.Header().Set("Content-Type", "application/octet-stream") + } else { entry := api.ManifestEntry{ Hash: rootKey.Hex(), Path: updateKey.Hex(), @@ -376,9 +380,9 @@ func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { ModTime: now, Status: http.StatusOK, } - mode := (6 << 2) | (4 << 1) | 4 + mode := 0644 if s.api.DbIsValidated() { - mode |= 2 << 1 + mode |= (2 << 3) | 2 } entry.Mode = int64(mode) manifest := api.Manifest{ @@ -388,12 +392,13 @@ func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { } manifestJson, err := json.Marshal(manifest) if err != nil { - w.WriteHeader(http.StatusInternalServerError) + s.Error(w, r, fmt.Errorf("Could not convert manifest to json: %v", err)) return } - data = bytes.NewReader(manifestJson) + w.Header().Set("Content-Type", api.DbManifestType) + data = []byte(manifestJson) } - http.ServeContent(w, &r.Request, "", now, data) + http.ServeContent(w, &r.Request, "", now, bytes.NewReader(data)) } // HandleGet handles a GET request to diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index e2165f1d61..42c9ce3f56 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -47,46 +47,53 @@ func TestBzzGetDb(t *testing.T) { keybytes := make([]byte, common.HashLength) // nearest we get to source of info _, err := rand.Read(keybytes) if err != nil { - fmt.Printf("err: %v\n", err) - return + t.Fatal(err) } url := fmt.Sprintf("%s/bzz-db:/%s/42", srv.URL, fmt.Sprintf("%x", keybytes)) resp, err := http.Post(url, "application/octet-stream", nil) if err != nil { - fmt.Printf("err: %v\n", err) - return + t.Fatal(err) } b, err := ioutil.ReadAll(resp.Body) - fmt.Printf("Create: %s : %s\n", resp.Status, b) + if err != nil { + t.Fatal(err) + } + log.Debug("Create", "status", resp.Status, "body", b) - url = fmt.Sprintf("%s/bzz-db:/%s", srv.URL, fmt.Sprintf("%x", keybytes)) + url = fmt.Sprintf("%s/bzz-db:/%x", srv.URL, keybytes) data := []byte("foo") resp, err = http.Post(url, "application/octet-stream", bytes.NewReader(data)) if err != nil { - fmt.Printf("err: %v\n", err) - return + t.Fatal(err) } b, err = ioutil.ReadAll(resp.Body) - fmt.Printf("Update: %s : %s\n", resp.Status, b) + if err != nil { + t.Fatal(err) + } + log.Debug("Update", "status", resp.Status, "body", b) url = fmt.Sprintf("%s/bzz-db-raw:/%s", srv.URL, fmt.Sprintf("%x", keybytes)) resp, err = http.Get(url) if err != nil { - fmt.Printf("err: %v\n", err) - return + t.Fatal(err) } b, err = ioutil.ReadAll(resp.Body) - fmt.Printf("Get: %s : %s\n", resp.Status, b) + if err != nil { + t.Fatal(err) + } + log.Debug("Get raw", "status", resp.Status, "body", b) url = fmt.Sprintf("%s/bzz-db:/%s", srv.URL, fmt.Sprintf("%x", keybytes)) resp, err = http.Get(url) if err != nil { - fmt.Printf("err: %v\n", err) - return + t.Fatal(err) } b, err = ioutil.ReadAll(resp.Body) - fmt.Printf("Get: %s : %s\n", resp.Status, b) + if err != nil { + t.Fatal(err) + } + log.Debug("Get manifest", "status", resp.Status, "body", b) } diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 23e1a6336b..c51a85105b 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -4,6 +4,7 @@ import ( "context" "encoding/binary" "fmt" + "math/big" "path/filepath" "sync" "time" @@ -12,7 +13,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" ) @@ -58,6 +58,10 @@ type ResourceValidator interface { sign(common.Hash) (Signature, error) // SignFunc } +type ethApi interface { + BlockNumber(context.Context) (big.Int, error) +} + // Mutable resource is an entity which allows updates to a resource // without resorting to ENS on each update. // The update scheme is built on swarm chunks with chunk keys following @@ -119,8 +123,10 @@ type ResourceValidator interface { // TODO: Include modtime in chunk data + signature type ResourceHandler struct { ChunkStore + ctx context.Context + cancelFunc func() validator ResourceValidator - ethClient *ethclient.Client + ethClient ethApi resources map[string]*resource hashLock sync.Mutex resourceLock sync.RWMutex @@ -134,7 +140,7 @@ type ResourceHandler struct { // Create or open resource update chunk store // // If validator is nil, signature and access validation will be deactivated -func NewResourceHandler(datadir string, cloudStore CloudStore, ethClient *ethclient.Client, validator ResourceValidator) (*ResourceHandler, error) { +func NewResourceHandler(datadir string, cloudStore CloudStore, ethClient ethApi, validator ResourceValidator) (*ResourceHandler, error) { hashfunc := MakeHashFunc(SHA3Hash) @@ -602,17 +608,11 @@ func (self *ResourceHandler) Close() { } func (self *ResourceHandler) GetBlock() (uint64, error) { - return self.ethClient.BlockNumber(self.ctx) - // get the block height and convert to uint64 - // var currentblock string - // err := self.rpcClient.Call(¤tblock, "eth_blockNumber") - // if err != nil { - // return 0, err - // } - // if currentblock == "0x0" { - // return 0, nil - // } - // return strconv.ParseUint(currentblock, 10, 64) + bigblocknumber, err := self.ethClient.BlockNumber(self.ctx) + if err != nil { + return 0, err + } + return bigblocknumber.Uint64(), nil } // Calculate the period index (aka major version number) from a given block number @@ -776,10 +776,7 @@ func isSafeName(name string) bool { if err != nil { return false } - if validname != name { - return false - } - return true + return validname == name } // convenience for creating signature hashes of update data diff --git a/swarm/storage/resource_test.go b/swarm/storage/resource_test.go index 2da9b7704a..dd96f0b3d7 100644 --- a/swarm/storage/resource_test.go +++ b/swarm/storage/resource_test.go @@ -2,6 +2,7 @@ package storage import ( "bytes" + "context" "crypto/ecdsa" "crypto/rand" "encoding/binary" @@ -9,8 +10,6 @@ import ( "io/ioutil" "math/big" "os" - "path/filepath" - "strconv" "strings" "testing" "time" @@ -22,9 +21,7 @@ import ( "github.com/ethereum/go-ethereum/contracts/ens/contract" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rpc" ) var ( @@ -50,7 +47,7 @@ func init() { // so we use this wrapper to fake returning the block count type fakeBackend struct { *backends.SimulatedBackend - blocknumber uint64 + blocknumber int64 } func (f *fakeBackend) Commit() { @@ -60,13 +57,10 @@ func (f *fakeBackend) Commit() { f.blocknumber++ } -// for faking the rpc service, since we don't need the whole node stack -type FakeRPC struct { - backend *fakeBackend -} - -func (r *FakeRPC) BlockNumber() (string, error) { - return strconv.FormatUint(r.backend.blocknumber, 10), nil +func (f *fakeBackend) BlockNumber(context context.Context) (big.Int, error) { + f.blocknumber++ + biggie := big.NewInt(f.blocknumber) + return *biggie, nil } // check that signature address matches update signer address @@ -84,8 +78,9 @@ func TestResourceReverse(t *testing.T) { // set up rpc and create resourcehandler rh, _, _, teardownTest, err := setupTest(nil, newTestValidator(signer.signContent)) if err != nil { - teardownTest(t, err) + t.Fatal(err) } + defer teardownTest() // generate a hash for block 4200 version 1 key := rh.resourceHash(period, version, rh.nameHash(safeName)) @@ -94,14 +89,14 @@ func TestResourceReverse(t *testing.T) { data := make([]byte, 8) _, err = rand.Read(data) if err != nil { - teardownTest(t, err) + t.Fatal(err) } testHasher.Reset() testHasher.Write(data) digest := rh.keyDataHash(key, data) sig, err := rh.validator.sign(digest) if err != nil { - teardownTest(t, err) + t.Fatal(err) } chunk := newUpdateChunk(key, &sig, period, version, safeName, data) @@ -111,31 +106,30 @@ func TestResourceReverse(t *testing.T) { checkdigest := rh.keyDataHash(chunk.Key, checkdata) recoveredaddress, err := getAddressFromDataSig(checkdigest, *checksig) if err != nil { - teardownTest(t, fmt.Errorf("Retrieve address from signature fail: %v", err)) + t.Fatalf("Retrieve address from signature fail: %v", err) } originaladdress := crypto.PubkeyToAddress(signer.privKey.PublicKey) // check that the metadata retrieved from the chunk matches what we gave it if recoveredaddress != originaladdress { - teardownTest(t, fmt.Errorf("addresses dont match: %x != %x", originaladdress, recoveredaddress)) + t.Fatalf("addresses dont match: %x != %x", originaladdress, recoveredaddress) } if !bytes.Equal(key[:], chunk.Key[:]) { - teardownTest(t, fmt.Errorf("Expected chunk key '%x', was '%x'", key, chunk.Key)) + t.Fatalf("Expected chunk key '%x', was '%x'", key, chunk.Key) } if period != checkperiod { - teardownTest(t, fmt.Errorf("Expected period '%d', was '%d'", period, checkperiod)) + t.Fatalf("Expected period '%d', was '%d'", period, checkperiod) } if version != checkversion { - teardownTest(t, fmt.Errorf("Expected version '%d', was '%d'", version, checkversion)) + t.Fatalf("Expected version '%d', was '%d'", version, checkversion) } if safeName != checkname { - teardownTest(t, fmt.Errorf("Expected name '%s', was '%s'", safeName, checkname)) + t.Fatalf("Expected name '%s', was '%s'", safeName, checkname) } if !bytes.Equal(data, checkdata) { - teardownTest(t, fmt.Errorf("Expectedn data '%x', was '%x'", data, checkdata)) + t.Fatalf("Expectedn data '%x', was '%x'", data, checkdata) } - teardownTest(t, nil) } // make updates and retrieve them based on periods and versions @@ -143,34 +137,35 @@ func TestResourceHandler(t *testing.T) { // make fake backend, set up rpc and create resourcehandler backend := &fakeBackend{ - blocknumber: startBlock, + blocknumber: int64(startBlock), } rh, datadir, _, teardownTest, err := setupTest(backend, nil) if err != nil { - teardownTest(t, err) + t.Fatal(err) } + defer teardownTest() // create a new resource _, err = rh.NewResource(safeName, resourceFrequency) if err != nil { - teardownTest(t, err) + t.Fatal(err) } // check that the new resource is stored correctly namehash := rh.nameHash(safeName) chunk, err := rh.ChunkStore.(*resourceChunkStore).localStore.(*LocalStore).memStore.Get(Key(namehash[:])) if err != nil { - teardownTest(t, err) + t.Fatal(err) } else if len(chunk.SData) < 16 { - teardownTest(t, fmt.Errorf("chunk data must be minimum 16 bytes, is %d", len(chunk.SData))) + t.Fatalf("chunk data must be minimum 16 bytes, is %d", len(chunk.SData)) } startblocknumber := binary.LittleEndian.Uint64(chunk.SData[:8]) chunkfrequency := binary.LittleEndian.Uint64(chunk.SData[8:]) - if startblocknumber != backend.blocknumber { - teardownTest(t, fmt.Errorf("stored block number %d does not match provided block number %d", startblocknumber, backend.blocknumber)) + if startblocknumber != uint64(backend.blocknumber) { + t.Fatalf("stored block number %d does not match provided block number %d", startblocknumber, backend.blocknumber) } if chunkfrequency != resourceFrequency { - teardownTest(t, fmt.Errorf("stored frequency %d does not match provided frequency %d", chunkfrequency, resourceFrequency)) + t.Fatalf("stored frequency %d does not match provided frequency %d", chunkfrequency, resourceFrequency) } // update halfway to first period @@ -179,7 +174,7 @@ func TestResourceHandler(t *testing.T) { data := []byte("blinky") resourcekey["blinky"], err = rh.Update(safeName, data) if err != nil { - teardownTest(t, err) + t.Fatal(err) } // update on first period @@ -187,7 +182,7 @@ func TestResourceHandler(t *testing.T) { data = []byte("pinky") resourcekey["pinky"], err = rh.Update(safeName, data) if err != nil { - teardownTest(t, err) + t.Fatal(err) } // update on second period @@ -195,7 +190,7 @@ func TestResourceHandler(t *testing.T) { data = []byte("inky") resourcekey["inky"], err = rh.Update(safeName, data) if err != nil { - teardownTest(t, err) + t.Fatal(err) } // update just after second period @@ -203,7 +198,7 @@ func TestResourceHandler(t *testing.T) { data = []byte("clyde") resourcekey["clyde"], err = rh.Update(safeName, data) if err != nil { - teardownTest(t, err) + t.Fatal(err) } time.Sleep(time.Second) rh.Close() @@ -215,43 +210,42 @@ func TestResourceHandler(t *testing.T) { rh2, err := NewResourceHandler(datadir, &testCloudStore{}, rh.ethClient, nil) _, err = rh2.LookupLatestByName(safeName, true) if err != nil { - teardownTest(t, err) + t.Fatal(err) } // last update should be "clyde", version two, blockheight startblocknumber + (resourcefrequency * 3) if !bytes.Equal(rh2.resources[safeName].data, []byte("clyde")) { - teardownTest(t, fmt.Errorf("resource data was %v, expected %v", rh2.resources[safeName].data, []byte("clyde"))) + t.Fatalf("resource data was %v, expected %v", rh2.resources[safeName].data, []byte("clyde")) } if rh2.resources[safeName].version != 2 { - teardownTest(t, fmt.Errorf("resource version was %d, expected 2", rh2.resources[safeName].version)) + t.Fatalf("resource version was %d, expected 2", rh2.resources[safeName].version) } if rh2.resources[safeName].lastPeriod != 3 { - teardownTest(t, fmt.Errorf("resource period was %d, expected 3", rh2.resources[safeName].lastPeriod)) + t.Fatalf("resource period was %d, expected 3", rh2.resources[safeName].lastPeriod) } log.Debug("Latest lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data) // specific block, latest version rsrc, err := rh2.LookupHistoricalByName(safeName, 3, true) if err != nil { - teardownTest(t, err) + t.Fatal(err) } // check data if !bytes.Equal(rsrc.data, []byte("clyde")) { - teardownTest(t, fmt.Errorf("resource data (historical) was %v, expected %v", rh2.resources[domainName].data, []byte("clyde"))) + t.Fatalf("resource data (historical) was %v, expected %v", rh2.resources[domainName].data, []byte("clyde")) } log.Debug("Historical lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data) // specific block, specific version rsrc, err = rh2.LookupVersionByName(safeName, 3, 1, true) if err != nil { - teardownTest(t, err) + t.Fatal(err) } // check data if !bytes.Equal(rsrc.data, []byte("inky")) { - teardownTest(t, fmt.Errorf("resource data (historical) was %v, expected %v", rh2.resources[domainName].data, []byte("inky"))) + t.Fatalf("resource data (historical) was %v, expected %v", rh2.resources[domainName].data, []byte("inky")) } log.Debug("Specific version lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data) - teardownTest(t, nil) } @@ -283,34 +277,33 @@ func TestResourceENSOwner(t *testing.T) { // set up rpc and create resourcehandler with ENS sim backend rh, _, _, teardownTest, err := setupTest(contractbackend, validator) if err != nil { - teardownTest(t, err) + t.Fatal(err) } + defer teardownTest() // create new resource when we are owner = ok _, err = rh.NewResource(safeName, resourceFrequency) if err != nil { - teardownTest(t, fmt.Errorf("Create resource fail: %v", err)) + t.Fatalf("Create resource fail: %v", err) } data := []byte("foo") // update resource when we are owner = ok _, err = rh.Update(safeName, data) if err != nil { - teardownTest(t, fmt.Errorf("Update resource fail: %v", err)) + t.Fatalf("Update resource fail: %v", err) } // update resource when we are owner = ok signertwo, err := newTestSigner() if err != nil { - teardownTest(t, err) + t.Fatal(err) } rh.validator.(*ENSValidator).signFunc = signertwo.signContent _, err = rh.Update(safeName, data) if err == nil { - teardownTest(t, fmt.Errorf("Expected resource update fail due to owner mismatch")) + t.Fatalf("Expected resource update fail due to owner mismatch") } - - teardownTest(t, nil) } // fast-forward blockheight @@ -321,7 +314,7 @@ func fwdBlocks(count int, backend *fakeBackend) { } // create rpc and resourcehandler -func setupTest(contractbackend bind.ContractBackend, validator ResourceValidator) (rh *ResourceHandler, datadir string, signer *testSigner, teardown func(*testing.T, error), err error) { +func setupTest(backend ethApi, validator ResourceValidator) (rh *ResourceHandler, datadir string, signer *testSigner, teardown func(), err error) { var fsClean func() var rpcClean func() @@ -337,55 +330,18 @@ func setupTest(contractbackend bind.ContractBackend, validator ResourceValidator // temp datadir datadir, err = ioutil.TempDir("", "rh") if err != nil { - return + return nil, "", nil, nil, err } fsClean = func() { os.RemoveAll(datadir) } - // starting the whole stack just to get blocknumbers is too cumbersome - // so we fake the rpc server to get blocknumbers for testing - ipcpath := filepath.Join(datadir, "test.ipc") - ipcl, err := rpc.CreateIPCListener(ipcpath) - if err != nil { - return - } - rpcserver := rpc.NewServer() - var fake *fakeBackend - if contractbackend != nil { - fake = contractbackend.(*fakeBackend) - } - rpcserver.RegisterName("eth", &FakeRPC{ - backend: fake, - }) - go func() { - rpcserver.ServeListener(ipcl) - }() - rpcClean = func() { - rpcserver.Stop() - } - - // connect to fake rpc - rpcClient, err := rpc.Dial(ipcpath) - if err != nil { - return - } - - ethClient := ethclient.NewClient(rpcClient) - - rh, err = NewResourceHandler(datadir, &testCloudStore{}, ethClient, validator) - teardown = func(t *testing.T, err error) { - cleanF() - if err != nil { - t.Fatal(err) - } - } - - return + rh, err = NewResourceHandler(datadir, &testCloudStore{}, backend, validator) + return rh, datadir, signer, cleanF, nil } // Set up simulated ENS backend for use with ENSResourceHandler tests -func setupENS(addr common.Address, transactOpts *bind.TransactOpts, sub string, top string) (common.Address, bind.ContractBackend, error) { +func setupENS(addr common.Address, transactOpts *bind.TransactOpts, sub string, top string) (common.Address, *fakeBackend, error) { // create the domain hash values to pass to the ENS contract methods var tophash [32]byte diff --git a/swarm/swarm.go b/swarm/swarm.go index 766c210d3d..c650567947 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -54,15 +54,13 @@ type Swarm struct { storage storage.ChunkStore // internal access to storage, common interface to cloud storage backends dpa *storage.DPA // distributed preimage archive, the local API to the storage with document level storage/retrieval support //depo network.StorageHandler // remote request handler, interface between bzz protocol and the storage - cloud storage.CloudStore // procurement, cloud storage backend (can multi-cloud) - bzz *network.Bzz // the logistic manager - backend chequebook.Backend // simple blockchain Backend - privateKey *ecdsa.PrivateKey - corsString string - swapEnabled bool - lstore *storage.LocalStore // local store, needs to store for releasing resources after node stopped - sfs *fuse.SwarmFS // need this to cleanup all the active mounts on node exit - ps *pss.Pss + cloud storage.CloudStore // procurement, cloud storage backend (can multi-cloud) + bzz *network.Bzz // the logistic manager + backend chequebook.Backend // simple blockchain Backend + privateKey *ecdsa.PrivateKey + lstore *storage.LocalStore // local store, needs to store for releasing resources after node stopped + sfs *fuse.SwarmFS // need this to cleanup all the active mounts on node exit + ps *pss.Pss } type SwarmAPI struct { @@ -83,7 +81,7 @@ func (self *Swarm) API() *SwarmAPI { // implements node.Service // If mockStore is not nil, it will be used as the storage for chunk data. // MockStore should be used only for testing. -func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *ethclient.Client, config *api.Config, swapEnabled, syncEnabled bool, cors string, pssEnabled bool, resourceEnabled bool, mockStore *mock.NodeStore) (self *Swarm, err error) { +func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *ethclient.Client, config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err error) { if bytes.Equal(common.FromHex(config.PublicKey), storage.ZeroKey) { return nil, fmt.Errorf("empty public key") @@ -93,11 +91,9 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e } self = &Swarm{ - config: config, - swapEnabled: swapEnabled, - backend: backend, - privateKey: config.Swap.PrivateKey(), - corsString: cors, + config: config, + backend: backend, + privateKey: config.Swap.PrivateKey(), } log.Debug(fmt.Sprintf("Setting up Swarm service components")) @@ -139,7 +135,7 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e log.Debug(fmt.Sprintf("-> Content Store API")) // Pss = postal service over swarm (devp2p over bzz) - if pssEnabled { + if self.config.PssEnabled { pssparams := pss.NewPssParams(self.privateKey) self.ps = pss.NewPss(to, self.dpa, pssparams) if pss.IsActiveHandshake { @@ -162,7 +158,7 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e var resourceHandler *storage.ResourceHandler // if use resource updates - if resourceEnabled { + if self.config.ResourceEnabled { var resourceValidator storage.ResourceValidator if self.dns != nil { resourceValidator, err = storage.NewENSValidator(config.EnsRoot, ensClient, transactOpts, storage.NewGenericResourceSigner(self.privateKey)) @@ -204,7 +200,7 @@ func (self *Swarm) Start(srv *p2p.Server) error { log.Warn("Updated bzz local addr", "oaddr", fmt.Sprintf("%x", newaddr.OAddr), "uaddr", fmt.Sprintf("%x", newaddr.UAddr)) // set chequebook - if self.swapEnabled { + if self.config.SwapEnabled { ctx := context.Background() // The initial setup has no deadline. err := self.SetChequebook(ctx) if err != nil { @@ -237,14 +233,14 @@ func (self *Swarm) Start(srv *p2p.Server) error { addr := net.JoinHostPort(self.config.ListenAddr, self.config.Port) go httpapi.StartHttpServer(self.api, &httpapi.ServerConfig{ Addr: addr, - CorsString: self.corsString, + CorsString: self.config.Cors, }) } log.Debug(fmt.Sprintf("Swarm http proxy started on port: %v", self.config.Port)) - if self.corsString != "" { - log.Debug(fmt.Sprintf("Swarm http proxy started with corsdomain: %v", self.corsString)) + if self.config.Cors != "" { + log.Debug(fmt.Sprintf("Swarm http proxy started with corsdomain: %v", self.config.Cors)) } return nil diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go index 4cec635936..3556101850 100644 --- a/swarm/testutil/http.go +++ b/swarm/testutil/http.go @@ -17,21 +17,29 @@ package testutil import ( - "crypto/ecdsa" + "context" "io/ioutil" + "math/big" "net/http/httptest" "os" - "path/filepath" "strconv" "testing" - "github.com/ethereum/go-ethereum/ethclient" - "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm/api" httpapi "github.com/ethereum/go-ethereum/swarm/api/http" "github.com/ethereum/go-ethereum/swarm/storage" ) +type fakeBackend struct { + blocknumber int64 +} + +func (f *fakeBackend) BlockNumber(ctx context.Context) (big.Int, error) { + f.blocknumber++ + biggie := big.NewInt(f.blocknumber) + return *biggie, nil +} + func NewTestSwarmServer(t *testing.T) *TestSwarmServer { dir, err := ioutil.TempDir("", "swarm-storage-test") if err != nil { @@ -60,28 +68,8 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { if err != nil { t.Fatal(err) } - ipcPath := filepath.Join(resourceDir, "test.ipc") - ipcl, err := rpc.CreateIPCListener(ipcPath) - if err != nil { - t.Fatal(err) - } - rpcServer := rpc.NewServer() - rpcServer.RegisterName("eth", &FakeRPC{}) - go func() { - rpcServer.ServeListener(ipcl) - }() - rpcClean := func() { - rpcServer.Stop() - } - // connect to fake rpc - rpcClient, err := rpc.Dial(ipcPath) - if err != nil { - t.Fatal(err) - } - ethClient := ethclient.NewClient(rpcClient) - - rh, err := storage.NewResourceHandler(resourceDir, &testCloudStore{}, ethClient, nil) + rh, err := storage.NewResourceHandler(resourceDir, &testCloudStore{}, &fakeBackend{}, nil) if err != nil { t.Fatal(err) } @@ -94,8 +82,9 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { dir: dir, hasher: storage.MakeHashFunc(storage.SHA3Hash)(), cleanup: func() { + srv.Close() rh.Close() - rpcClean() + dpa.Stop() os.RemoveAll(dir) os.RemoveAll(resourceDir) }, @@ -104,17 +93,14 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { type TestSwarmServer struct { *httptest.Server - hasher storage.SwarmHash - privatekey *ecdsa.PrivateKey - Dpa *storage.DPA - dir string - cleanup func() + hasher storage.SwarmHash + Dpa *storage.DPA + dir string + cleanup func() } func (t *TestSwarmServer) Close() { - t.Server.Close() - t.Dpa.Stop() - os.RemoveAll(t.dir) + t.cleanup() } type testCloudStore struct { From 11c3b4cae16c749ee9ad894dc38b196525cb9cf3 Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 22 Jan 2018 05:08:45 +0100 Subject: [PATCH 058/107] swarm/api: Add contenttype contingent resolve manifest -> rsrc --- swarm/api/api.go | 2 +- swarm/api/http/server.go | 80 ++++++++++++++++++++++++++--------- swarm/api/http/server_test.go | 39 ++++++++++++----- swarm/api/manifest.go | 4 +- 4 files changed, 91 insertions(+), 34 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 1393fa4406..2a3de5b5ca 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -365,7 +365,7 @@ func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storag } // Look up mutable resource updates at specific periods and versions -func (self *Api) DbLookup(key storage.Key, name string, period uint32, version uint32) (storage.Key, []byte, error) { +func (self *Api) DbLookup(name string, period uint32, version uint32) (storage.Key, []byte, error) { var err error if version != 0 { if period == 0 { diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 2a8046f1c7..49103f5f85 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -292,7 +292,8 @@ func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) { } func (s *Server) HandlePostDb(w http.ResponseWriter, r *Request) { - if r.ContentLength == 0 { + var outdata string + if r.uri.Path != "" { frequency, err := strconv.ParseUint(r.uri.Path, 10, 64) if err != nil { s.BadRequest(w, r, fmt.Sprintf("Cannot parse frequency parameter: %v", err)) @@ -303,20 +304,50 @@ func (s *Server) HandlePostDb(w http.ResponseWriter, r *Request) { s.Error(w, r, fmt.Errorf("Resource creation failed: %v", err)) return } - } else { - data, err := ioutil.ReadAll(r.Body) + manifestKey, err := s.api.NewManifest() if err != nil { - w.WriteHeader(http.StatusInternalServerError) + s.Error(w, r, fmt.Errorf("create manifest err: %v", err)) return } - _, _, _, err = s.api.DbUpdate(r.uri.Addr, data) + newKey, err := s.updateManifest(manifestKey, func(mw *api.ManifestWriter) error { + key, err := mw.AddEntry(bytes.NewReader([]byte(r.uri.Addr)), &api.ManifestEntry{ + Path: r.uri.Addr, + ContentType: api.ResourceContentType, + Mode: 0644, + Size: int64(len(r.uri.Addr)), + ModTime: time.Now(), + }) + if err != nil { + return err + } + s.logDebug("resource manifest for for %s stored", key.Log()) + return nil + }) if err != nil { - w.WriteHeader(http.StatusUnauthorized) - http.ServeContent(w, &r.Request, "", time.Now(), bytes.NewReader([]byte(err.Error()))) + s.Error(w, r, fmt.Errorf("update manifest err: %v", err)) return } + log.Debug("manifests", "new", newKey, "old", manifestKey) + outdata = fmt.Sprintf("%s", newKey) } + + data, err := ioutil.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + _, _, _, err = s.api.DbUpdate(r.uri.Addr, data) + if err != nil { + w.Header().Add("Status", fmt.Sprintf("%d", http.StatusUnauthorized)) + http.ServeContent(w, &r.Request, "", time.Now(), bytes.NewReader([]byte(err.Error()))) + return + } + w.WriteHeader(http.StatusOK) + if outdata != "" { + w.Header().Set("Content-type", "text/plain") + fmt.Fprintf(w, outdata) + } } // Retrieve mutable resource updates: @@ -325,13 +356,10 @@ func (s *Server) HandlePostDb(w http.ResponseWriter, r *Request) { // bzz-db[-[immutable|-raw]]://// - get update version m of period n // = ens name or hash func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { + s.handleGetDb(w, r, r.uri.Addr) +} - rootKey, err := s.api.Resolve(r.uri) - if err != nil { - s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) - return - } - +func (s *Server) handleGetDb(w http.ResponseWriter, r *Request, name string) { var params []string if len(r.uri.Path) > 0 { params = strings.Split(r.uri.Path, "/") @@ -341,16 +369,18 @@ func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { var version uint64 var data []byte var dataLength int + var err error now := time.Now() + log.Debug("handlegetdb", "name", name) switch len(params) { case 0: - updateKey, data, err = s.api.DbLookup(rootKey, r.uri.Addr, 0, 0) + updateKey, data, err = s.api.DbLookup(name, 0, 0) case 2: version, err = strconv.ParseUint(params[1], 10, 32) if err != nil { break } - updateKey, data, err = s.api.DbLookup(rootKey, r.uri.Addr, uint32(period), uint32(version)) + updateKey, data, err = s.api.DbLookup(name, uint32(period), uint32(version)) case 1: version, err = strconv.ParseUint(params[1], 10, 32) if err != nil { @@ -360,7 +390,7 @@ func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { if err != nil { break } - updateKey, data, err = s.api.DbLookup(rootKey, r.uri.Addr, uint32(period), uint32(version)) + updateKey, data, err = s.api.DbLookup(name, uint32(period), uint32(version)) default: s.BadRequest(w, r, fmt.Sprintf("Invalid mutable resource request")) return @@ -373,9 +403,9 @@ func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { w.Header().Set("Content-Type", "application/octet-stream") } else { entry := api.ManifestEntry{ - Hash: rootKey.Hex(), + Hash: name, Path: updateKey.Hex(), - ContentType: api.DbManifestType, + ContentType: api.ManifestType, Size: int64(dataLength), ModTime: now, Status: http.StatusOK, @@ -395,7 +425,7 @@ func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { s.Error(w, r, fmt.Errorf("Could not convert manifest to json: %v", err)) return } - w.Header().Set("Content-Type", api.DbManifestType) + w.Header().Set("Content-Type", api.ManifestType) data = []byte(manifestJson) } http.ServeContent(w, &r.Request, "", now, bytes.NewReader(data)) @@ -461,6 +491,17 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { switch { case r.uri.Raw(): + m := &api.Manifest{} + sz, _ := reader.Size(nil) + b := make([]byte, sz) + reader.Read(b) + err = json.Unmarshal(b, m) + if len(m.Entries) > 0 { + if m.Entries[0].ContentType == api.ResourceContentType { + s.handleGetDb(w, r, m.Entries[0].Path) + return + } + } // allow the request to overwrite the content type using a query // parameter contentType := "application/octet-stream" @@ -468,7 +509,6 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { contentType = typ } w.Header().Set("Content-Type", contentType) - http.ServeContent(w, &r.Request, "", time.Now(), reader) case r.uri.Hash(): w.Header().Set("Content-Type", "text/plain") diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 42c9ce3f56..cd7d1a0c9b 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -45,21 +45,27 @@ func TestBzzGetDb(t *testing.T) { defer srv.Close() keybytes := make([]byte, common.HashLength) // nearest we get to source of info - _, err := rand.Read(keybytes) + copy(keybytes, []byte{42}) + + databytes := make([]byte, 42) + _, err := rand.Read(databytes) if err != nil { t.Fatal(err) } - url := fmt.Sprintf("%s/bzz-db:/%s/42", srv.URL, fmt.Sprintf("%x", keybytes)) - resp, err := http.Post(url, "application/octet-stream", nil) + url := fmt.Sprintf("%s/bzz-db:/%x/42", srv.URL, keybytes) + resp, err := http.Post(url, "application/octet-stream", bytes.NewReader(databytes)) if err != nil { t.Fatal(err) } - b, err := ioutil.ReadAll(resp.Body) + manifesthash, err := ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) } - log.Debug("Create", "status", resp.Status, "body", b) + if resp.StatusCode != http.StatusOK { + t.Fatalf("err %s", resp.Status) + } + log.Debug("Create", "status", resp.Status, "body", manifesthash) url = fmt.Sprintf("%s/bzz-db:/%x", srv.URL, keybytes) data := []byte("foo") @@ -67,13 +73,13 @@ func TestBzzGetDb(t *testing.T) { if err != nil { t.Fatal(err) } - b, err = ioutil.ReadAll(resp.Body) + b, err := ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) } - log.Debug("Update", "status", resp.Status, "body", b) + log.Debug("Update", "status", resp.Status) - url = fmt.Sprintf("%s/bzz-db-raw:/%s", srv.URL, fmt.Sprintf("%x", keybytes)) + url = fmt.Sprintf("%s/bzz-raw:/%s", srv.URL, manifesthash) resp, err = http.Get(url) if err != nil { t.Fatal(err) @@ -82,9 +88,9 @@ func TestBzzGetDb(t *testing.T) { if err != nil { t.Fatal(err) } - log.Debug("Get raw", "status", resp.Status, "body", b) + log.Debug("Manifest", "status", resp.Status, "body", fmt.Sprintf("%s", b)) - url = fmt.Sprintf("%s/bzz-db:/%s", srv.URL, fmt.Sprintf("%x", keybytes)) + url = fmt.Sprintf("%s/bzz-db-raw:/%x", srv.URL, keybytes) resp, err = http.Get(url) if err != nil { t.Fatal(err) @@ -93,7 +99,18 @@ func TestBzzGetDb(t *testing.T) { if err != nil { t.Fatal(err) } - log.Debug("Get manifest", "status", resp.Status, "body", b) + log.Debug("Get raw", "status", resp.Status) + + url = fmt.Sprintf("%s/bzz-db:/%x", srv.URL, keybytes) + resp, err = http.Get(url) + if err != nil { + t.Fatal(err) + } + b, err = ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + log.Debug("Get manifest", "status", resp.Status) } diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index 46b55b3b7a..fde086b7ac 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -33,8 +33,8 @@ import ( ) const ( - ManifestType = "application/bzz-manifest+json" - DbManifestType = "application/bzz-db-manifest+json" + ManifestType = "application/bzz-manifest+json" + ResourceContentType = "application/bzz-resource" ) // Manifest represents a swarm manifest From 80f539c21715a15c943edb2079fc9f13ab560db4 Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 22 Jan 2018 05:13:28 +0100 Subject: [PATCH 059/107] swarm/api: Rename Db -> Resource --- swarm/api/api.go | 10 ++--- swarm/api/http/server.go | 71 +++++++++++------------------------ swarm/api/http/server_test.go | 21 +++-------- swarm/api/uri.go | 10 ++--- 4 files changed, 34 insertions(+), 78 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 2a3de5b5ca..0c8d9d1ea7 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -365,7 +365,7 @@ func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storag } // Look up mutable resource updates at specific periods and versions -func (self *Api) DbLookup(name string, period uint32, version uint32) (storage.Key, []byte, error) { +func (self *Api) ResourceLookup(name string, period uint32, version uint32) (storage.Key, []byte, error) { var err error if version != 0 { if period == 0 { @@ -391,22 +391,22 @@ func (self *Api) DbLookup(name string, period uint32, version uint32) (storage.K return key, data, nil } -func (self *Api) DbCreate(name string, frequency uint64) (err error) { +func (self *Api) ResourceCreate(name string, frequency uint64) (err error) { _, err = self.resource.NewResource(name, frequency) return err } -func (self *Api) DbUpdate(name string, data []byte) (storage.Key, uint32, uint32, error) { +func (self *Api) ResourceUpdate(name string, data []byte) (storage.Key, uint32, uint32, error) { key, err := self.resource.Update(name, data) period, _ := self.resource.GetLastPeriod(name) version, _ := self.resource.GetVersion(name) return key, period, version, err } -func (self *Api) DbHashSize() int { +func (self *Api) ResourceHashSize() int { return self.resource.HashSize() } -func (self *Api) DbIsValidated() bool { +func (self *Api) ResourceIsValidated() bool { return self.resource.IsValidated() } diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 49103f5f85..dc4025896c 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -291,7 +291,7 @@ func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) { fmt.Fprint(w, newKey) } -func (s *Server) HandlePostDb(w http.ResponseWriter, r *Request) { +func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { var outdata string if r.uri.Path != "" { frequency, err := strconv.ParseUint(r.uri.Path, 10, 64) @@ -299,7 +299,7 @@ func (s *Server) HandlePostDb(w http.ResponseWriter, r *Request) { s.BadRequest(w, r, fmt.Sprintf("Cannot parse frequency parameter: %v", err)) return } - err = s.api.DbCreate(r.uri.Addr, frequency) + err = s.api.ResourceCreate(r.uri.Addr, frequency) if err != nil { s.Error(w, r, fmt.Errorf("Resource creation failed: %v", err)) return @@ -333,13 +333,12 @@ func (s *Server) HandlePostDb(w http.ResponseWriter, r *Request) { data, err := ioutil.ReadAll(r.Body) if err != nil { - w.WriteHeader(http.StatusInternalServerError) + s.Error(w, r, err) return } - _, _, _, err = s.api.DbUpdate(r.uri.Addr, data) + _, _, _, err = s.api.ResourceUpdate(r.uri.Addr, data) if err != nil { - w.Header().Add("Status", fmt.Sprintf("%d", http.StatusUnauthorized)) - http.ServeContent(w, &r.Request, "", time.Now(), bytes.NewReader([]byte(err.Error()))) + w.WriteHeader(http.StatusUnauthorized) return } @@ -351,15 +350,15 @@ func (s *Server) HandlePostDb(w http.ResponseWriter, r *Request) { } // Retrieve mutable resource updates: -// bzz-db[-[immutable|-raw]]:// - get latest update -// bzz-db[-[immutable|-raw]]:/// - get latest update on period n -// bzz-db[-[immutable|-raw]]://// - get update version m of period n +// bzz-resource:// - get latest update +// bzz-resource:/// - get latest update on period n +// bzz-resource://// - get update version m of period n // = ens name or hash -func (s *Server) HandleGetDb(w http.ResponseWriter, r *Request) { - s.handleGetDb(w, r, r.uri.Addr) +func (s *Server) HandleGetResource(w http.ResponseWriter, r *Request) { + s.handleGetResource(w, r, r.uri.Addr) } -func (s *Server) handleGetDb(w http.ResponseWriter, r *Request, name string) { +func (s *Server) handleGetResource(w http.ResponseWriter, r *Request, name string) { var params []string if len(r.uri.Path) > 0 { params = strings.Split(r.uri.Path, "/") @@ -368,19 +367,18 @@ func (s *Server) handleGetDb(w http.ResponseWriter, r *Request, name string) { var period uint64 var version uint64 var data []byte - var dataLength int var err error now := time.Now() log.Debug("handlegetdb", "name", name) switch len(params) { case 0: - updateKey, data, err = s.api.DbLookup(name, 0, 0) + updateKey, data, err = s.api.ResourceLookup(name, 0, 0) case 2: version, err = strconv.ParseUint(params[1], 10, 32) if err != nil { break } - updateKey, data, err = s.api.DbLookup(name, uint32(period), uint32(version)) + updateKey, data, err = s.api.ResourceLookup(name, uint32(period), uint32(version)) case 1: version, err = strconv.ParseUint(params[1], 10, 32) if err != nil { @@ -390,7 +388,7 @@ func (s *Server) handleGetDb(w http.ResponseWriter, r *Request, name string) { if err != nil { break } - updateKey, data, err = s.api.DbLookup(name, uint32(period), uint32(version)) + updateKey, data, err = s.api.ResourceLookup(name, uint32(period), uint32(version)) default: s.BadRequest(w, r, fmt.Sprintf("Invalid mutable resource request")) return @@ -399,35 +397,8 @@ func (s *Server) handleGetDb(w http.ResponseWriter, r *Request, name string) { s.Error(w, r, fmt.Errorf("Mutable resource lookup failed: %v", err)) return } - if !r.uri.DbRaw() { - w.Header().Set("Content-Type", "application/octet-stream") - } else { - entry := api.ManifestEntry{ - Hash: name, - Path: updateKey.Hex(), - ContentType: api.ManifestType, - Size: int64(dataLength), - ModTime: now, - Status: http.StatusOK, - } - mode := 0644 - if s.api.DbIsValidated() { - mode |= (2 << 3) | 2 - } - entry.Mode = int64(mode) - manifest := api.Manifest{ - Entries: []api.ManifestEntry{ - entry, - }, - } - manifestJson, err := json.Marshal(manifest) - if err != nil { - s.Error(w, r, fmt.Errorf("Could not convert manifest to json: %v", err)) - return - } - w.Header().Set("Content-Type", api.ManifestType) - data = []byte(manifestJson) - } + log.Debug("Found update", "key", updateKey) + w.Header().Set("Content-Type", "application/octet-stream") http.ServeContent(w, &r.Request, "", now, bytes.NewReader(data)) } @@ -498,7 +469,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { err = json.Unmarshal(b, m) if len(m.Entries) > 0 { if m.Entries[0].ContentType == api.ResourceContentType { - s.handleGetDb(w, r, m.Entries[0].Path) + s.handleGetResource(w, r, m.Entries[0].Path) return } } @@ -755,8 +726,8 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { case "POST": if uri.Raw() || uri.DeprecatedRaw() { s.HandlePostRaw(w, req) - } else if uri.Db() { - s.HandlePostDb(w, req) + } else if uri.Resource() { + s.HandlePostResource(w, req) } else { s.HandlePostFiles(w, req) } @@ -783,8 +754,8 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { case "GET": - if uri.Db() || uri.DbRaw() { - s.HandleGetDb(w, req) + if uri.Resource() { + s.HandleGetResource(w, req) return } diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index cd7d1a0c9b..c79922a7f2 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -53,7 +53,7 @@ func TestBzzGetDb(t *testing.T) { t.Fatal(err) } - url := fmt.Sprintf("%s/bzz-db:/%x/42", srv.URL, keybytes) + url := fmt.Sprintf("%s/bzz-resource:/%x/42", srv.URL, keybytes) resp, err := http.Post(url, "application/octet-stream", bytes.NewReader(databytes)) if err != nil { t.Fatal(err) @@ -67,7 +67,7 @@ func TestBzzGetDb(t *testing.T) { } log.Debug("Create", "status", resp.Status, "body", manifesthash) - url = fmt.Sprintf("%s/bzz-db:/%x", srv.URL, keybytes) + url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) data := []byte("foo") resp, err = http.Post(url, "application/octet-stream", bytes.NewReader(data)) if err != nil { @@ -88,9 +88,9 @@ func TestBzzGetDb(t *testing.T) { if err != nil { t.Fatal(err) } - log.Debug("Manifest", "status", resp.Status, "body", fmt.Sprintf("%s", b)) + log.Debug("Get raw", "status", resp.Status, "body", fmt.Sprintf("%s", b)) - url = fmt.Sprintf("%s/bzz-db-raw:/%x", srv.URL, keybytes) + url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) resp, err = http.Get(url) if err != nil { t.Fatal(err) @@ -99,18 +99,7 @@ func TestBzzGetDb(t *testing.T) { if err != nil { t.Fatal(err) } - log.Debug("Get raw", "status", resp.Status) - - url = fmt.Sprintf("%s/bzz-db:/%x", srv.URL, keybytes) - resp, err = http.Get(url) - if err != nil { - t.Fatal(err) - } - b, err = ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatal(err) - } - log.Debug("Get manifest", "status", resp.Status) + log.Debug("Get resource", "status", resp.Status, "data", b) } diff --git a/swarm/api/uri.go b/swarm/api/uri.go index c7a929f0f2..009bc016fd 100644 --- a/swarm/api/uri.go +++ b/swarm/api/uri.go @@ -69,7 +69,7 @@ func Parse(rawuri string) (*URI, error) { // check the scheme is valid switch uri.Scheme { - case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzz-hash", "bzzr", "bzzi", "bzz-db", "bzz-db-raw": + case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzz-hash", "bzzr", "bzzi", "bzz-resource": default: return nil, fmt.Errorf("unknown scheme %q", u.Scheme) } @@ -92,12 +92,8 @@ func Parse(rawuri string) (*URI, error) { return uri, nil } -func (u *URI) Db() bool { - return u.Scheme == "bzz-db" -} - -func (u *URI) DbRaw() bool { - return u.Scheme == "bzz-db-raw" +func (u *URI) Resource() bool { + return u.Scheme == "bzz-resource" } func (u *URI) Raw() bool { From aff40c4e8c8661456711ea34ba2e7359ff71a71c Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 22 Jan 2018 18:19:51 +0100 Subject: [PATCH 060/107] swarm/storage, ethclient: Move signature to end of chunk data ethclient: Omit string conversion detour --- ethclient/ethclient.go | 11 +----- swarm/storage/resource.go | 73 ++++++++++++++++++++++----------------- 2 files changed, 43 insertions(+), 41 deletions(-) diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 2d9553a7bd..f41de1b42c 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -77,17 +77,8 @@ type rpcBlock struct { } func (ec *Client) BlockNumber(ctx context.Context) (big.Int, error) { - var numberstr string number := &big.Int{} - err := ec.c.CallContext(ctx, &numberstr, "eth_blockNumber") - if err != nil { - return *number, err - } - var ok bool - number, ok = number.SetString(numberstr, 10) - if !ok { - err = errors.New("Failed to parse bigint") - } + err := ec.c.CallContext(ctx, &number, "eth_blockNumber") return *number, err } diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index c51a85105b..6a90f2bdfa 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -476,20 +476,12 @@ func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk) ( func (self *ResourceHandler) parseUpdate(chunkdata []byte) (*Signature, uint32, uint32, string, []byte, error) { var err error cursor := 0 - var signature *Signature - // omit signatures if we have no validator - var sigoffset int - if self.validator != nil { - signature = &Signature{} - copy(signature[:], chunkdata[:signatureLength]) - sigoffset = signatureLength - cursor = sigoffset - } - headerlength := binary.LittleEndian.Uint16(chunkdata[cursor : cursor+2]) - if int(headerlength+2) > len(chunkdata) { - err = fmt.Errorf("Reported header length %d longer than actual data length %d", headerlength, len(chunkdata)) - return nil, 0, 0, "", nil, err + cursor += 2 + datalength := binary.LittleEndian.Uint16(chunkdata[cursor : cursor+2]) + if int(headerlength+datalength+4) > len(chunkdata) { + err = fmt.Errorf("Reported headerlength %d + datalength %d longer than actual chunk data length %d", headerlength, datalength, len(chunkdata)) + return } var period uint32 @@ -501,12 +493,21 @@ func (self *ResourceHandler) parseUpdate(chunkdata []byte) (*Signature, uint32, cursor += 4 version = binary.LittleEndian.Uint32(chunkdata[cursor : cursor+4]) cursor += 4 - namelength := int(headerlength) - cursor + sigoffset + 2 + namelength := int(headerlength) - cursor + 4 name = string(chunkdata[cursor : cursor+namelength]) cursor += namelength - data = make([]byte, len(chunkdata)-cursor) - copy(data, chunkdata[cursor:]) - return signature, period, version, name, data, err + intdatalength := int(datalength) + data = make([]byte, intdatalength) + copy(data, chunkdata[cursor:cursor+intdatalength]) + + // omit signatures if we have no validator + if self.validator != nil { + cursor += intdatalength + signature = &Signature{} + copy(signature[:], chunkdata[cursor:cursor+signatureLength]) + } + + return } // Adds an actual data update @@ -517,9 +518,9 @@ func (self *ResourceHandler) parseUpdate(chunkdata []byte) (*Signature, uint32, // A resource update cannot span chunks, and thus has max length 4096 func (self *ResourceHandler) Update(name string, data []byte) (Key, error) { - var sigoffset int + var signaturelength int if self.validator != nil { - sigoffset = signatureLength + signaturelength = signatureLength } // get the cached information @@ -532,7 +533,7 @@ func (self *ResourceHandler) Update(name string, data []byte) (Key, error) { } // an update can be only one chunk long - datalimit := self.chunkSize() - int64(sigoffset-len(name)-8) + datalimit := self.chunkSize() - int64(signaturelength-len(name)-4-4-2-2) if int64(len(data)) > datalimit { return nil, fmt.Errorf("Data overflow: %d / %d bytes", len(data), datalimit) } @@ -672,27 +673,30 @@ func getAddressFromDataSig(datahash common.Hash, signature Signature) (common.Ad func newUpdateChunk(key Key, signature *Signature, period uint32, version uint32, name string, data []byte) *Chunk { // no signatures if no validator - var sigoffset int + var signaturelength int if signature != nil { - sigoffset = signatureLength + signaturelength = signatureLength } // prepend version and period to allow reverse lookups - headerlength := uint16(len(name) + 4 + 4) + headerlength := len(name) + 4 + 4 + + // also prepend datalength + datalength := len(data) chunk := NewChunk(key, nil) - chunk.SData = make([]byte, sigoffset+int(headerlength)+2+len(data)) - - cursor := 0 - if signature != nil { - copy(chunk.SData, (*signature)[:]) - cursor += signatureLength - } + chunk.SData = make([]byte, 4+signaturelength+headerlength+datalength) // data header length does NOT include the header length prefix bytes themselves - binary.LittleEndian.PutUint16(chunk.SData[cursor:], headerlength) + cursor := 0 + binary.LittleEndian.PutUint16(chunk.SData[cursor:], uint16(headerlength)) cursor += 2 + // data length + binary.LittleEndian.PutUint16(chunk.SData[cursor:], uint16(datalength)) + cursor += 2 + + // header = period + version + name binary.LittleEndian.PutUint32(chunk.SData[cursor:], period) cursor += 4 @@ -703,8 +707,15 @@ func newUpdateChunk(key Key, signature *Signature, period uint32, version uint32 copy(chunk.SData[cursor:], namebytes) cursor += len(namebytes) + // add the data copy(chunk.SData[cursor:], data) + // if signature is present it's the last item in the chunk data + if signature != nil { + cursor += datalength + copy(chunk.SData[cursor:], signature[:]) + } + chunk.Size = int64(len(chunk.SData)) return chunk } From 703051b1aa398a4d5aade97fab780e95c1894556 Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 22 Jan 2018 19:10:32 +0100 Subject: [PATCH 061/107] swarm: Cleanup after rebase on swarm-mutableresources-extsign --- swarm/api/http/server_test.go | 2 +- swarm/storage/resource.go | 11 +++++------ swarm/testutil/http.go | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index c79922a7f2..2837decfde 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -40,7 +40,7 @@ func init() { log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) } -func TestBzzGetDb(t *testing.T) { +func TestBzzResource(t *testing.T) { srv := testutil.NewTestSwarmServer(t) defer srv.Close() diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 6a90f2bdfa..bf4404634c 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -19,7 +19,7 @@ import ( const ( signatureLength = 65 indexSize = 16 - dbDirName = "resource" + DbDirName = "resource" chunkSize = 4096 // temporary until we implement DPA in the resourcehandler defaultStoreTimeout = 4000 * time.Millisecond ) @@ -133,8 +133,6 @@ type ResourceHandler struct { hasher SwarmHash nameHash nameHashFunc storeTimeout time.Duration - ctx context.Context - cancelFunc func() } // Create or open resource update chunk store @@ -157,7 +155,7 @@ func NewResourceHandler(datadir string, cloudStore CloudStore, ethClient ethApi, ctx, cancel := context.WithCancel(context.Background()) rh := &ResourceHandler{ ChunkStore: newResourceChunkStore(path, hashfunc, localStore, cloudStore), - rpcClient: rpcClient, + ethClient: ethClient, resources: make(map[string]*resource), hasher: hashfunc(), validator: validator, @@ -481,7 +479,7 @@ func (self *ResourceHandler) parseUpdate(chunkdata []byte) (*Signature, uint32, datalength := binary.LittleEndian.Uint16(chunkdata[cursor : cursor+2]) if int(headerlength+datalength+4) > len(chunkdata) { err = fmt.Errorf("Reported headerlength %d + datalength %d longer than actual chunk data length %d", headerlength, datalength, len(chunkdata)) - return + return nil, 0, 0, "", nil, err } var period uint32 @@ -501,13 +499,14 @@ func (self *ResourceHandler) parseUpdate(chunkdata []byte) (*Signature, uint32, copy(data, chunkdata[cursor:cursor+intdatalength]) // omit signatures if we have no validator + var signature *Signature if self.validator != nil { cursor += intdatalength signature = &Signature{} copy(signature[:], chunkdata[cursor:cursor+signatureLength]) } - return + return signature, period, version, name, data, nil } // Adds an actual data update diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go index 3556101850..4a6a68e43b 100644 --- a/swarm/testutil/http.go +++ b/swarm/testutil/http.go @@ -51,7 +51,7 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { CacheCapacity: 5000, Radius: 0, } - localStore, err := storage.NewLocalStore(storage.MakeHashFunc(storage.SHA3Hash), storeparams) + localStore, err := storage.NewLocalStore(storage.MakeHashFunc(storage.SHA3Hash), storeparams, nil) if err != nil { os.RemoveAll(dir) t.Fatal(err) From c2e0be67c899df018dbbab826de93b33c88947e6 Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 22 Jan 2018 19:27:24 +0100 Subject: [PATCH 062/107] swarm/storage, swarm/api: Change noparam fmt.Errorf -> errors.New --- swarm/api/http/server.go | 2 +- swarm/storage/resource.go | 19 ++++++++++--------- swarm/storage/resource_ens.go | 4 ++-- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index dc4025896c..0b22b9d8da 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -447,7 +447,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { return api.SkipManifest }) if entry == nil { - s.NotFound(w, r, fmt.Errorf("Manifest entry could not be loaded")) + s.NotFound(w, r, errors.New("Manifest entry could not be loaded")) return } key = storage.Key(common.Hex2Bytes(entry.Hash)) diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index bf4404634c..80251c841e 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -3,6 +3,7 @@ package storage import ( "context" "encoding/binary" + "errors" "fmt" "math/big" "path/filepath" @@ -192,7 +193,7 @@ func (self *ResourceHandler) HashSize() int { func (self *ResourceHandler) GetContent(name string) (Key, []byte, error) { rsrc := self.getResource(name) if rsrc == nil || !rsrc.isSynced() { - return nil, nil, fmt.Errorf("Resource does not exist or is not synced") + return nil, nil, errors.New("Resource does not exist or is not synced") } return rsrc.lastKey, rsrc.data, nil } @@ -201,7 +202,7 @@ func (self *ResourceHandler) GetLastPeriod(name string) (uint32, error) { rsrc := self.getResource(name) if rsrc == nil || !rsrc.isSynced() { - return 0, fmt.Errorf("Resource does not exist or is not synced") + return 0, errors.New("Resource does not exist or is not synced") } return rsrc.lastPeriod, nil } @@ -209,7 +210,7 @@ func (self *ResourceHandler) GetLastPeriod(name string) (uint32, error) { func (self *ResourceHandler) GetVersion(name string) (uint32, error) { rsrc := self.getResource(name) if rsrc == nil || !rsrc.isSynced() { - return 0, fmt.Errorf("Resource does not exist or is not synced") + return 0, errors.New("Resource does not exist or is not synced") } return rsrc.version, nil } @@ -228,7 +229,7 @@ func (self *ResourceHandler) NewResource(name string, frequency uint64) (*resour // frequency 0 is invalid if frequency == 0 { - return nil, fmt.Errorf("Frequency cannot be 0") + return nil, errors.New("Frequency cannot be 0") } if !isSafeName(name) { @@ -360,7 +361,7 @@ func (self *ResourceHandler) LookupLatest(nameHash common.Hash, name string, ref func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint32, refresh bool) (*resource, error) { if period == 0 { - return nil, fmt.Errorf("period must be >0") + return nil, errors.New("period must be >0") } // start from the last possible block period, and iterate previous ones until we find a match @@ -396,7 +397,7 @@ func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint3 log.Trace("rsrc update not found, checking previous period", "period", period, "key", key) period-- } - return nil, fmt.Errorf("no updates found") + return nil, errors.New("no updates found") } // load existing mutable resource into resource struct @@ -525,10 +526,10 @@ func (self *ResourceHandler) Update(name string, data []byte) (Key, error) { // get the cached information rsrc := self.getResource(name) if rsrc == nil { - return nil, fmt.Errorf("Resource object not in index") + return nil, errors.New("Resource object not in index") } if !rsrc.isSynced() { - return nil, fmt.Errorf("Resource object not in sync") + return nil, errors.New("Resource object not in sync") } // an update can be only one chunk long @@ -747,7 +748,7 @@ func (r *resourceChunkStore) Get(key Key) (*Chunk, error) { t := time.NewTimer(time.Second * 1) select { case <-t.C: - return nil, fmt.Errorf("timeout") + return nil, errors.New("timeout") case <-chunk.C: log.Trace("Received resource update chunk", "peer", chunk.Req.Source) } diff --git a/swarm/storage/resource_ens.go b/swarm/storage/resource_ens.go index df008efa17..33163bc97a 100644 --- a/swarm/storage/resource_ens.go +++ b/swarm/storage/resource_ens.go @@ -1,7 +1,7 @@ package storage import ( - "fmt" + "errors" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" @@ -15,7 +15,7 @@ type baseValidator struct { func (b *baseValidator) sign(datahash common.Hash) (signature Signature, err error) { if b.signFunc == nil { - return signature, fmt.Errorf("No signature function") + return signature, errors.New("No signature function") } return b.signFunc(datahash) } From 400b7d6d1c44047d2a18065cc12680478851f618 Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 22 Jan 2018 19:54:22 +0100 Subject: [PATCH 063/107] swarm/api/http: Test result data --- swarm/api/http/server_test.go | 45 +++++++++++++++++------------------ 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 2837decfde..e75bd04bcf 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -23,37 +23,35 @@ import ( "fmt" "io/ioutil" "net/http" - "os" "strings" "sync" "testing" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" swarm "github.com/ethereum/go-ethereum/swarm/api/client" "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/testutil" ) -func init() { - log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) -} - +// \TODO if create -> get -> update -> get, the last get with return 1.1 because 1.2 retrieve is still pending func TestBzzResource(t *testing.T) { srv := testutil.NewTestSwarmServer(t) defer srv.Close() - keybytes := make([]byte, common.HashLength) // nearest we get to source of info + // our mutable resource "name" + keybytes := make([]byte, common.HashLength) copy(keybytes, []byte{42}) - databytes := make([]byte, 42) + // data of update 1 + databytes := make([]byte, 666) _, err := rand.Read(databytes) if err != nil { t.Fatal(err) } - url := fmt.Sprintf("%s/bzz-resource:/%x/42", srv.URL, keybytes) + // creates resource and sets update 1 + url := fmt.Sprintf("%s/bzz-resource:/%x/13", srv.URL, keybytes) resp, err := http.Post(url, "application/octet-stream", bytes.NewReader(databytes)) if err != nil { t.Fatal(err) @@ -65,31 +63,31 @@ func TestBzzResource(t *testing.T) { if resp.StatusCode != http.StatusOK { t.Fatalf("err %s", resp.Status) } - log.Debug("Create", "status", resp.Status, "body", manifesthash) + // update 2 url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) data := []byte("foo") resp, err = http.Post(url, "application/octet-stream", bytes.NewReader(data)) + if err != nil { + t.Fatal(err) + } else if resp.StatusCode != http.StatusOK { + t.Fatalf("Update returned %d", resp.Status) + } + + // get latest update (1.2) through swarm manifest + url = fmt.Sprintf("%s/bzz-raw:/%s", srv.URL, manifesthash) + resp, err = http.Get(url) if err != nil { t.Fatal(err) } b, err := ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) + } else if !bytes.Equal(data, b) { + t.Fatalf("Expected body '%x', got '%x'", data, b) } - log.Debug("Update", "status", resp.Status) - - url = fmt.Sprintf("%s/bzz-raw:/%s", srv.URL, manifesthash) - resp, err = http.Get(url) - if err != nil { - t.Fatal(err) - } - b, err = ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatal(err) - } - log.Debug("Get raw", "status", resp.Status, "body", fmt.Sprintf("%s", b)) + // get latest update (1.2) through resource directly url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) resp, err = http.Get(url) if err != nil { @@ -98,8 +96,9 @@ func TestBzzResource(t *testing.T) { b, err = ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) + } else if !bytes.Equal(data, b) { + t.Fatalf("Expected body '%x', got '%x'", data, b) } - log.Debug("Get resource", "status", resp.Status, "data", b) } From 0455925ebf1b902aaba51f83b91b1c1653edc623 Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 22 Jan 2018 23:48:21 +0100 Subject: [PATCH 064/107] swarm: Amend comments from @lmars PR 204 second review, part I --- swarm/api/api.go | 6 +----- swarm/api/http/server.go | 28 ++++++++++++++++------------ swarm/api/http/server_test.go | 13 +++++++++---- swarm/storage/resource.go | 17 ++++++++--------- swarm/storage/resource_test.go | 7 +++++-- swarm/testutil/http.go | 7 +++++-- 6 files changed, 44 insertions(+), 34 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 0c8d9d1ea7..5197d7a4bd 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -384,11 +384,7 @@ func (self *Api) ResourceLookup(name string, period uint32, version uint32) (sto if err != nil { return nil, nil, err } - key, data, err := self.resource.GetContent(name) - if err != nil { - return nil, nil, err - } - return key, data, nil + return self.resource.GetContent(name) } func (self *Api) ResourceCreate(name string, frequency uint64) (err error) { diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 0b22b9d8da..c84d47dac4 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -378,19 +378,19 @@ func (s *Server) handleGetResource(w http.ResponseWriter, r *Request, name strin if err != nil { break } - updateKey, data, err = s.api.ResourceLookup(name, uint32(period), uint32(version)) - case 1: - version, err = strconv.ParseUint(params[1], 10, 32) + period, err = strconv.ParseUint(params[0], 10, 32) if err != nil { break } + updateKey, data, err = s.api.ResourceLookup(name, uint32(period), uint32(version)) + case 1: period, err = strconv.ParseUint(params[0], 10, 32) if err != nil { break } updateKey, data, err = s.api.ResourceLookup(name, uint32(period), uint32(version)) default: - s.BadRequest(w, r, fmt.Sprintf("Invalid mutable resource request")) + s.BadRequest(w, r, "Invalid mutable resource request") return } if err != nil { @@ -463,14 +463,18 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { switch { case r.uri.Raw(): m := &api.Manifest{} - sz, _ := reader.Size(nil) - b := make([]byte, sz) - reader.Read(b) - err = json.Unmarshal(b, m) - if len(m.Entries) > 0 { - if m.Entries[0].ContentType == api.ResourceContentType { - s.handleGetResource(w, r, m.Entries[0].Path) - return + sz, err := reader.Size(nil) + if err == nil { + b := make([]byte, sz) + reader.Read(b) + err = json.Unmarshal(b, m) + if err == nil { + if len(m.Entries) > 0 { + if m.Entries[0].ContentType == api.ResourceContentType { + s.handleGetResource(w, r, m.Entries[0].Path) + return + } + } } } // allow the request to overwrite the content type using a query diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index e75bd04bcf..73036de203 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -55,14 +55,14 @@ func TestBzzResource(t *testing.T) { resp, err := http.Post(url, "application/octet-stream", bytes.NewReader(databytes)) if err != nil { t.Fatal(err) + } else if resp.StatusCode != http.StatusOK { + t.Fatalf("err %s", resp.Status) } manifesthash, err := ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) } - if resp.StatusCode != http.StatusOK { - t.Fatalf("err %s", resp.Status) - } + resp.Body.Close() // update 2 url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) @@ -79,6 +79,8 @@ func TestBzzResource(t *testing.T) { resp, err = http.Get(url) if err != nil { t.Fatal(err) + } else if resp.StatusCode != http.StatusOK { + t.Fatalf("err %s", resp.Status) } b, err := ioutil.ReadAll(resp.Body) if err != nil { @@ -86,12 +88,15 @@ func TestBzzResource(t *testing.T) { } else if !bytes.Equal(data, b) { t.Fatalf("Expected body '%x', got '%x'", data, b) } + resp.Body.Close() // get latest update (1.2) through resource directly url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) resp, err = http.Get(url) if err != nil { t.Fatal(err) + } else if resp.StatusCode != http.StatusOK { + t.Fatalf("err %s", resp.Status) } b, err = ioutil.ReadAll(resp.Body) if err != nil { @@ -99,7 +104,7 @@ func TestBzzResource(t *testing.T) { } else if !bytes.Equal(data, b) { t.Fatalf("Expected body '%x', got '%x'", data, b) } - + resp.Body.Close() } func TestBzzGetPath(t *testing.T) { diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 80251c841e..62e0dc5456 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -13,6 +13,7 @@ import ( "golang.org/x/net/idna" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" ) @@ -60,7 +61,7 @@ type ResourceValidator interface { } type ethApi interface { - BlockNumber(context.Context) (big.Int, error) + HeaderByNumber(context.Context, *big.Int) (*types.Header, error) } // Mutable resource is an entity which allows updates to a resource @@ -124,7 +125,7 @@ type ethApi interface { // TODO: Include modtime in chunk data + signature type ResourceHandler struct { ChunkStore - ctx context.Context + ctx context.Context // base for new contexts passed to storage layer and ethapi, to ensure teardown when Close() is called cancelFunc func() validator ResourceValidator ethClient ethApi @@ -609,11 +610,13 @@ func (self *ResourceHandler) Close() { } func (self *ResourceHandler) GetBlock() (uint64, error) { - bigblocknumber, err := self.ethClient.BlockNumber(self.ctx) + ctx, cancel := context.WithCancel(self.ctx) + defer cancel() + blockheader, err := self.ethClient.HeaderByNumber(ctx, nil) if err != nil { return 0, err } - return bigblocknumber.Uint64(), nil + return blockheader.Number.Uint64(), nil } // Calculate the period index (aka major version number) from a given block number @@ -771,11 +774,7 @@ func getNextPeriod(start uint64, current uint64, frequency uint64) uint32 { } func ToSafeName(name string) (string, error) { - validname, err := idna.ToASCII(name) - if err != nil { - return "", err - } - return validname, nil + return idna.ToASCII(name) } // check that name identifiers contain valid bytes diff --git a/swarm/storage/resource_test.go b/swarm/storage/resource_test.go index dd96f0b3d7..e6a00d88f4 100644 --- a/swarm/storage/resource_test.go +++ b/swarm/storage/resource_test.go @@ -20,6 +20,7 @@ import ( "github.com/ethereum/go-ethereum/contracts/ens" "github.com/ethereum/go-ethereum/contracts/ens/contract" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" ) @@ -57,10 +58,12 @@ func (f *fakeBackend) Commit() { f.blocknumber++ } -func (f *fakeBackend) BlockNumber(context context.Context) (big.Int, error) { +func (f *fakeBackend) HeaderByNumber(context context.Context, bigblock *big.Int) (*types.Header, error) { f.blocknumber++ biggie := big.NewInt(f.blocknumber) - return *biggie, nil + return &types.Header{ + Number: biggie, + }, nil } // check that signature address matches update signer address diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go index 4a6a68e43b..cf4b045c6d 100644 --- a/swarm/testutil/http.go +++ b/swarm/testutil/http.go @@ -25,6 +25,7 @@ import ( "strconv" "testing" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/swarm/api" httpapi "github.com/ethereum/go-ethereum/swarm/api/http" "github.com/ethereum/go-ethereum/swarm/storage" @@ -34,10 +35,12 @@ type fakeBackend struct { blocknumber int64 } -func (f *fakeBackend) BlockNumber(ctx context.Context) (big.Int, error) { +func (f *fakeBackend) HeaderByNumber(context context.Context, bigblock *big.Int) (*types.Header, error) { f.blocknumber++ biggie := big.NewInt(f.blocknumber) - return *biggie, nil + return &types.Header{ + Number: biggie, + }, nil } func NewTestSwarmServer(t *testing.T) *TestSwarmServer { From a32681cba4868d65af78484001f69095dbdbae6e Mon Sep 17 00:00:00 2001 From: lash Date: Tue, 23 Jan 2018 02:04:57 +0100 Subject: [PATCH 065/107] swarm/storage: Correct channel for waiting on chunk put --- swarm/api/http/server_test.go | 24 +++++++++++++++++++++++- swarm/storage/resource.go | 4 +++- swarm/testutil/http.go | 10 ---------- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 73036de203..664e56ca42 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -23,17 +23,23 @@ import ( "fmt" "io/ioutil" "net/http" + "os" "strings" "sync" "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" swarm "github.com/ethereum/go-ethereum/swarm/api/client" "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/testutil" ) +func init() { + log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) +} + // \TODO if create -> get -> update -> get, the last get with return 1.1 because 1.2 retrieve is still pending func TestBzzResource(t *testing.T) { srv := testutil.NewTestSwarmServer(t) @@ -64,6 +70,22 @@ func TestBzzResource(t *testing.T) { } resp.Body.Close() + // get latest update (1.1) through resource directly + url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) + resp, err = http.Get(url) + if err != nil { + t.Fatal(err) + } else if resp.StatusCode != http.StatusOK { + t.Fatalf("err %s", resp.Status) + } + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } else if !bytes.Equal(databytes, b) { + t.Fatalf("Expected body '%x', got '%x'", databytes, b) + } + resp.Body.Close() + // update 2 url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) data := []byte("foo") @@ -82,7 +104,7 @@ func TestBzzResource(t *testing.T) { } else if resp.StatusCode != http.StatusOK { t.Fatalf("err %s", resp.Status) } - b, err := ioutil.ReadAll(resp.Body) + b, err = ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) } else if !bytes.Equal(data, b) { diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 62e0dc5456..c7b675f10a 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -752,14 +752,16 @@ func (r *resourceChunkStore) Get(key Key) (*Chunk, error) { select { case <-t.C: return nil, errors.New("timeout") - case <-chunk.C: + case <-chunk.Req.C: log.Trace("Received resource update chunk", "peer", chunk.Req.Source) } return chunk, nil } func (r *resourceChunkStore) Put(chunk *Chunk) { + chunk.wg = &sync.WaitGroup{} r.netStore.Put(chunk) + chunk.wg.Wait() } func (r *resourceChunkStore) Close() { diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go index cf4b045c6d..afbd5d6103 100644 --- a/swarm/testutil/http.go +++ b/swarm/testutil/http.go @@ -22,7 +22,6 @@ import ( "math/big" "net/http/httptest" "os" - "strconv" "testing" "github.com/ethereum/go-ethereum/core/types" @@ -117,12 +116,3 @@ func (c *testCloudStore) Deliver(*storage.Chunk) { func (c *testCloudStore) Retrieve(*storage.Chunk) { } - -// for faking the rpc service, since we don't need the whole node stack -type FakeRPC struct { - blocknumber uint64 -} - -func (r *FakeRPC) BlockNumber() (string, error) { - return strconv.FormatUint(r.blocknumber, 10), nil -} From 94303aa1be77f34d4ecb7b8acc899df34d187026 Mon Sep 17 00:00:00 2001 From: lash Date: Tue, 23 Jan 2018 02:50:10 +0100 Subject: [PATCH 066/107] swarm/storage: Implement resourcehandler hashers as sync.Pool --- swarm/storage/resource.go | 51 ++++++++++++++++++++-------------- swarm/storage/resource_test.go | 3 ++ 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index c7b675f10a..2ec3e5327c 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -24,6 +24,7 @@ const ( DbDirName = "resource" chunkSize = 4096 // temporary until we implement DPA in the resourcehandler defaultStoreTimeout = 4000 * time.Millisecond + hasherCount = 8 ) type Signature [signatureLength]byte @@ -130,9 +131,8 @@ type ResourceHandler struct { validator ResourceValidator ethClient ethApi resources map[string]*resource - hashLock sync.Mutex + hashPool sync.Pool resourceLock sync.RWMutex - hasher SwarmHash nameHash nameHashFunc storeTimeout time.Duration } @@ -159,25 +159,34 @@ func NewResourceHandler(datadir string, cloudStore CloudStore, ethClient ethApi, ChunkStore: newResourceChunkStore(path, hashfunc, localStore, cloudStore), ethClient: ethClient, resources: make(map[string]*resource), - hasher: hashfunc(), validator: validator, storeTimeout: defaultStoreTimeout, ctx: ctx, cancelFunc: cancel, + hashPool: sync.Pool{ + New: func() interface{} { + return MakeHashFunc(SHA3Hash)() + }, + }, } if rh.validator != nil { rh.nameHash = rh.validator.nameHash } else { rh.nameHash = func(name string) common.Hash { - rh.hashLock.Lock() - defer rh.hashLock.Unlock() - rh.hasher.Reset() - rh.hasher.Write([]byte(name)) - return common.BytesToHash(rh.hasher.Sum(nil)) + hasher := rh.hashPool.Get().(SwarmHash) + defer rh.hashPool.Put(hasher) + hasher.Reset() + hasher.Write([]byte(name)) + return common.BytesToHash(hasher.Sum(nil)) } } + for i := 0; i < hasherCount; i++ { + hashfunc := MakeHashFunc(SHA3Hash)() + rh.hashPool.Put(hashfunc) + } + return rh, nil } @@ -645,16 +654,16 @@ func (self *ResourceHandler) setResource(name string, rsrc *resource) { // used for chunk keys func (self *ResourceHandler) resourceHash(period uint32, version uint32, namehash common.Hash) Key { // format is: hash(period|version|namehash) - self.hashLock.Lock() - defer self.hashLock.Unlock() - self.hasher.Reset() + hasher := self.hashPool.Get().(SwarmHash) + defer self.hashPool.Put(hasher) + hasher.Reset() b := make([]byte, 4) binary.LittleEndian.PutUint32(b, period) - self.hasher.Write(b) + hasher.Write(b) binary.LittleEndian.PutUint32(b, version) - self.hasher.Write(b) - self.hasher.Write(namehash[:]) - return self.hasher.Sum(nil) + hasher.Write(b) + hasher.Write(namehash[:]) + return hasher.Sum(nil) } func (self *ResourceHandler) hasUpdate(name string, period uint32) bool { @@ -793,10 +802,10 @@ func isSafeName(name string) bool { // convenience for creating signature hashes of update data func (self *ResourceHandler) keyDataHash(key Key, data []byte) common.Hash { - self.hashLock.Lock() - defer self.hashLock.Unlock() - self.hasher.Reset() - self.hasher.Write(key[:]) - self.hasher.Write(data) - return common.BytesToHash(self.hasher.Sum(nil)) + hasher := self.hashPool.Get().(SwarmHash) + defer self.hashPool.Put(hasher) + hasher.Reset() + hasher.Write(key[:]) + hasher.Write(data) + return common.BytesToHash(hasher.Sum(nil)) } diff --git a/swarm/storage/resource_test.go b/swarm/storage/resource_test.go index e6a00d88f4..537eb35d09 100644 --- a/swarm/storage/resource_test.go +++ b/swarm/storage/resource_test.go @@ -106,6 +106,9 @@ func TestResourceReverse(t *testing.T) { // check that we can recover the owner account from the update chunk's signature checksig, checkperiod, checkversion, checkname, checkdata, err := rh.parseUpdate(chunk.SData) + if err != nil { + t.Fatal(err) + } checkdigest := rh.keyDataHash(chunk.Key, checkdata) recoveredaddress, err := getAddressFromDataSig(checkdigest, *checksig) if err != nil { From edacd4b7c3600ed7b1888532268d701653132909 Mon Sep 17 00:00:00 2001 From: lash Date: Tue, 23 Jan 2018 03:19:00 +0100 Subject: [PATCH 067/107] swarm/api: Remove faulty manifest handling + add rsrc create keycheck --- swarm/api/api.go | 7 +++-- swarm/api/http/server.go | 53 ++++++----------------------------- swarm/api/http/server_test.go | 25 +++++------------ swarm/storage/resource.go | 8 +++++- swarm/testutil/http.go | 4 +-- 5 files changed, 28 insertions(+), 69 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 5197d7a4bd..7abe5295c1 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -387,9 +387,10 @@ func (self *Api) ResourceLookup(name string, period uint32, version uint32) (sto return self.resource.GetContent(name) } -func (self *Api) ResourceCreate(name string, frequency uint64) (err error) { - _, err = self.resource.NewResource(name, frequency) - return err +func (self *Api) ResourceCreate(name string, frequency uint64) (storage.Key, error) { + rsrc, err := self.resource.NewResource(name, frequency) + h := rsrc.NameHash() + return storage.Key(h[:]), err } func (self *Api) ResourceUpdate(name string, data []byte) (storage.Key, uint32, uint32, error) { diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index c84d47dac4..db81d89c59 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -299,36 +299,12 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { s.BadRequest(w, r, fmt.Sprintf("Cannot parse frequency parameter: %v", err)) return } - err = s.api.ResourceCreate(r.uri.Addr, frequency) + key, err := s.api.ResourceCreate(r.uri.Addr, frequency) if err != nil { s.Error(w, r, fmt.Errorf("Resource creation failed: %v", err)) return } - manifestKey, err := s.api.NewManifest() - if err != nil { - s.Error(w, r, fmt.Errorf("create manifest err: %v", err)) - return - } - newKey, err := s.updateManifest(manifestKey, func(mw *api.ManifestWriter) error { - key, err := mw.AddEntry(bytes.NewReader([]byte(r.uri.Addr)), &api.ManifestEntry{ - Path: r.uri.Addr, - ContentType: api.ResourceContentType, - Mode: 0644, - Size: int64(len(r.uri.Addr)), - ModTime: time.Now(), - }) - if err != nil { - return err - } - s.logDebug("resource manifest for for %s stored", key.Log()) - return nil - }) - if err != nil { - s.Error(w, r, fmt.Errorf("update manifest err: %v", err)) - return - } - log.Debug("manifests", "new", newKey, "old", manifestKey) - outdata = fmt.Sprintf("%s", newKey) + outdata = key.Hex() } data, err := ioutil.ReadAll(r.Body) @@ -338,15 +314,17 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { } _, _, _, err = s.api.ResourceUpdate(r.uri.Addr, data) if err != nil { - w.WriteHeader(http.StatusUnauthorized) + s.Error(w, r, fmt.Errorf("Update resource failed: %v", err)) return } - w.WriteHeader(http.StatusOK) if outdata != "" { - w.Header().Set("Content-type", "text/plain") - fmt.Fprintf(w, outdata) + w.Header().Add("Content-type", "text/plain") + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, outdata) + return } + w.WriteHeader(http.StatusOK) } // Retrieve mutable resource updates: @@ -462,21 +440,6 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { switch { case r.uri.Raw(): - m := &api.Manifest{} - sz, err := reader.Size(nil) - if err == nil { - b := make([]byte, sz) - reader.Read(b) - err = json.Unmarshal(b, m) - if err == nil { - if len(m.Entries) > 0 { - if m.Entries[0].ContentType == api.ResourceContentType { - s.handleGetResource(w, r, m.Entries[0].Path) - return - } - } - } - } // allow the request to overwrite the content type using a query // parameter contentType := "application/octet-stream" diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 664e56ca42..1a01108977 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -48,6 +48,9 @@ func TestBzzResource(t *testing.T) { // our mutable resource "name" keybytes := make([]byte, common.HashLength) copy(keybytes, []byte{42}) + srv.Hasher.Reset() + srv.Hasher.Write([]byte(fmt.Sprintf("%x", keybytes))) + keybyteshash := fmt.Sprintf("%x", srv.Hasher.Sum(nil)) // data of update 1 databytes := make([]byte, 666) @@ -64,9 +67,11 @@ func TestBzzResource(t *testing.T) { } else if resp.StatusCode != http.StatusOK { t.Fatalf("err %s", resp.Status) } - manifesthash, err := ioutil.ReadAll(resp.Body) + b, err := ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) + } else if !bytes.Equal(b, []byte(keybyteshash)) { + t.Fatalf("resource update hash mismatch, expected '%s' got '%s'", keybyteshash, b) } resp.Body.Close() @@ -78,7 +83,7 @@ func TestBzzResource(t *testing.T) { } else if resp.StatusCode != http.StatusOK { t.Fatalf("err %s", resp.Status) } - b, err := ioutil.ReadAll(resp.Body) + b, err = ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) } else if !bytes.Equal(databytes, b) { @@ -96,22 +101,6 @@ func TestBzzResource(t *testing.T) { t.Fatalf("Update returned %d", resp.Status) } - // get latest update (1.2) through swarm manifest - url = fmt.Sprintf("%s/bzz-raw:/%s", srv.URL, manifesthash) - resp, err = http.Get(url) - if err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusOK { - t.Fatalf("err %s", resp.Status) - } - b, err = ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatal(err) - } else if !bytes.Equal(data, b) { - t.Fatalf("Expected body '%x', got '%x'", data, b) - } - resp.Body.Close() - // get latest update (1.2) through resource directly url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) resp, err = http.Get(url) diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 2ec3e5327c..39b4fafa6f 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -52,6 +52,10 @@ func (self *resource) isSynced() bool { return !self.updated.IsZero() } +func (self *resource) NameHash() common.Hash { + return self.nameHash +} + // Implement to activate validation of resource updates // Specifically signing data and verification of signatures type ResourceValidator interface { @@ -178,7 +182,9 @@ func NewResourceHandler(datadir string, cloudStore CloudStore, ethClient ethApi, defer rh.hashPool.Put(hasher) hasher.Reset() hasher.Write([]byte(name)) - return common.BytesToHash(hasher.Sum(nil)) + hashval := common.BytesToHash(hasher.Sum(nil)) + log.Debug("generic namehasher", "name", name, "hash", hashval) + return hashval } } diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go index afbd5d6103..77e3386fd2 100644 --- a/swarm/testutil/http.go +++ b/swarm/testutil/http.go @@ -82,7 +82,7 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { Server: srv, Dpa: dpa, dir: dir, - hasher: storage.MakeHashFunc(storage.SHA3Hash)(), + Hasher: storage.MakeHashFunc(storage.SHA3Hash)(), cleanup: func() { srv.Close() rh.Close() @@ -95,7 +95,7 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { type TestSwarmServer struct { *httptest.Server - hasher storage.SwarmHash + Hasher storage.SwarmHash Dpa *storage.DPA dir string cleanup func() From 690522b09d613eaa507ec2f5ca076bd2515bb0b0 Mon Sep 17 00:00:00 2001 From: lash Date: Tue, 23 Jan 2018 14:57:04 +0100 Subject: [PATCH 068/107] swarm/api: Amend @gbalint comments PR 204 + args dep test loglvl --- swarm/api/api.go | 5 ++++- swarm/api/http/server_test.go | 21 ++++++++++++++------- swarm/storage/resource.go | 2 +- swarm/storage/resource_test.go | 7 ++++++- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 7abe5295c1..5a222dddc9 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -389,8 +389,11 @@ func (self *Api) ResourceLookup(name string, period uint32, version uint32) (sto func (self *Api) ResourceCreate(name string, frequency uint64) (storage.Key, error) { rsrc, err := self.resource.NewResource(name, frequency) + if err != nil { + return nil, err + } h := rsrc.NameHash() - return storage.Key(h[:]), err + return storage.Key(h[:]), nil } func (self *Api) ResourceUpdate(name string, data []byte) (storage.Key, uint32, uint32, error) { diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 1a01108977..ea6f9155d1 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -64,13 +64,15 @@ func TestBzzResource(t *testing.T) { resp, err := http.Post(url, "application/octet-stream", bytes.NewReader(databytes)) if err != nil { t.Fatal(err) - } else if resp.StatusCode != http.StatusOK { + } + if resp.StatusCode != http.StatusOK { t.Fatalf("err %s", resp.Status) } b, err := ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) - } else if !bytes.Equal(b, []byte(keybyteshash)) { + } + if !bytes.Equal(b, []byte(keybyteshash)) { t.Fatalf("resource update hash mismatch, expected '%s' got '%s'", keybyteshash, b) } resp.Body.Close() @@ -80,13 +82,15 @@ func TestBzzResource(t *testing.T) { resp, err = http.Get(url) if err != nil { t.Fatal(err) - } else if resp.StatusCode != http.StatusOK { + } + if resp.StatusCode != http.StatusOK { t.Fatalf("err %s", resp.Status) } b, err = ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) - } else if !bytes.Equal(databytes, b) { + } + if !bytes.Equal(databytes, b) { t.Fatalf("Expected body '%x', got '%x'", databytes, b) } resp.Body.Close() @@ -97,7 +101,8 @@ func TestBzzResource(t *testing.T) { resp, err = http.Post(url, "application/octet-stream", bytes.NewReader(data)) if err != nil { t.Fatal(err) - } else if resp.StatusCode != http.StatusOK { + } + if resp.StatusCode != http.StatusOK { t.Fatalf("Update returned %d", resp.Status) } @@ -106,13 +111,15 @@ func TestBzzResource(t *testing.T) { resp, err = http.Get(url) if err != nil { t.Fatal(err) - } else if resp.StatusCode != http.StatusOK { + } + if resp.StatusCode != http.StatusOK { t.Fatalf("err %s", resp.Status) } b, err = ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) - } else if !bytes.Equal(data, b) { + } + if !bytes.Equal(data, b) { t.Fatalf("Expected body '%x', got '%x'", data, b) } resp.Body.Close() diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 39b4fafa6f..7bd437e924 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -703,7 +703,7 @@ func newUpdateChunk(key Key, signature *Signature, period uint32, version uint32 datalength := len(data) chunk := NewChunk(key, nil) - chunk.SData = make([]byte, 4+signaturelength+headerlength+datalength) + chunk.SData = make([]byte, 4+signaturelength+headerlength+datalength) // initial 4 are uint16 length descriptors for headerlength and datalength // data header length does NOT include the header length prefix bytes themselves cursor := 0 diff --git a/swarm/storage/resource_test.go b/swarm/storage/resource_test.go index 537eb35d09..e84cdb7b93 100644 --- a/swarm/storage/resource_test.go +++ b/swarm/storage/resource_test.go @@ -6,6 +6,7 @@ import ( "crypto/ecdsa" "crypto/rand" "encoding/binary" + "flag" "fmt" "io/ioutil" "math/big" @@ -37,7 +38,11 @@ var ( func init() { var err error - log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) + verbose := flag.Bool("v", false, "verbose") + flag.Parse() + if *verbose { + log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) + } safeName, err = ToSafeName(domainName) if err != nil { panic(err) From 88ed2d4b2d13550d01016cdc6803bfa80083a43b Mon Sep 17 00:00:00 2001 From: lash Date: Tue, 23 Jan 2018 17:18:46 +0100 Subject: [PATCH 069/107] swarm, ethclient: Add version test for http api Clear redundant addition of BlockNumber in ethclient --- ethclient/ethclient.go | 6 ----- swarm/api/http/server_test.go | 50 ++++++++++++++++++++++++++++------- swarm/storage/resource.go | 3 +-- 3 files changed, 41 insertions(+), 18 deletions(-) diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index f41de1b42c..87a912901a 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -76,12 +76,6 @@ type rpcBlock struct { UncleHashes []common.Hash `json:"uncles"` } -func (ec *Client) BlockNumber(ctx context.Context) (big.Int, error) { - number := &big.Int{} - err := ec.c.CallContext(ctx, &number, "eth_blockNumber") - return *number, err -} - func (ec *Client) getBlock(ctx context.Context, method string, args ...interface{}) (*types.Block, error) { var raw json.RawMessage err := ec.c.CallContext(ctx, &raw, method, args...) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index ea6f9155d1..57d139ac98 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -23,24 +23,17 @@ import ( "fmt" "io/ioutil" "net/http" - "os" "strings" "sync" "testing" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" swarm "github.com/ethereum/go-ethereum/swarm/api/client" "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/testutil" ) -func init() { - log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) -} - -// \TODO if create -> get -> update -> get, the last get with return 1.1 because 1.2 retrieve is still pending func TestBzzResource(t *testing.T) { srv := testutil.NewTestSwarmServer(t) defer srv.Close() @@ -65,6 +58,7 @@ func TestBzzResource(t *testing.T) { if err != nil { t.Fatal(err) } + defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Fatalf("err %s", resp.Status) } @@ -75,7 +69,6 @@ func TestBzzResource(t *testing.T) { if !bytes.Equal(b, []byte(keybyteshash)) { t.Fatalf("resource update hash mismatch, expected '%s' got '%s'", keybyteshash, b) } - resp.Body.Close() // get latest update (1.1) through resource directly url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) @@ -83,6 +76,7 @@ func TestBzzResource(t *testing.T) { if err != nil { t.Fatal(err) } + defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Fatalf("err %s", resp.Status) } @@ -93,7 +87,6 @@ func TestBzzResource(t *testing.T) { if !bytes.Equal(databytes, b) { t.Fatalf("Expected body '%x', got '%x'", databytes, b) } - resp.Body.Close() // update 2 url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) @@ -102,6 +95,7 @@ func TestBzzResource(t *testing.T) { if err != nil { t.Fatal(err) } + defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Fatalf("Update returned %d", resp.Status) } @@ -112,6 +106,7 @@ func TestBzzResource(t *testing.T) { if err != nil { t.Fatal(err) } + defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Fatalf("err %s", resp.Status) } @@ -122,7 +117,42 @@ func TestBzzResource(t *testing.T) { if !bytes.Equal(data, b) { t.Fatalf("Expected body '%x', got '%x'", data, b) } - resp.Body.Close() + + // get latest update (1.2) with specified period + url = fmt.Sprintf("%s/bzz-resource:/%x/1", srv.URL, keybytes) + resp, err = http.Get(url) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("err %s", resp.Status) + } + b, err = ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(data, b) { + t.Fatalf("Expected body '%x', got '%x'", data, b) + } + + // get first update (1.1) with specified period and version + url = fmt.Sprintf("%s/bzz-resource:/%x/1/1", srv.URL, keybytes) + resp, err = http.Get(url) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("err %s", resp.Status) + } + b, err = ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(databytes, b) { + t.Fatalf("Expected body '%x', got '%x'", databytes, b) + } } func TestBzzGetPath(t *testing.T) { diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 7bd437e924..21e8466743 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -625,8 +625,7 @@ func (self *ResourceHandler) Close() { } func (self *ResourceHandler) GetBlock() (uint64, error) { - ctx, cancel := context.WithCancel(self.ctx) - defer cancel() + ctx, _ := context.WithCancel(self.ctx) blockheader, err := self.ethClient.HeaderByNumber(ctx, nil) if err != nil { return 0, err From 05ade19302357eba6a24348f31df140ce0eca326 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kurk=C3=B3=20Mih=C3=A1ly?= Date: Tue, 23 Jan 2018 22:51:04 +0200 Subject: [PATCH 070/107] dashboard: CPU, memory, diskIO and traffic on the footer (#15950) * dashboard: footer, deep state update * dashboard: resolve asset path * dashboard: prevent state update on every reconnection * dashboard: fix linter issue * dashboard, cmd: minor UI fix, include commit hash * dashboard: gitCommit renamed to commit * dashboard: move the geth version to the right, make commit optional * dashboard: memory, traffic and CPU on footer * dashboard: fix merge * dashboard: CPU, diskIO on footer * dashboard: rename variables, use group declaration * dashboard: docs --- dashboard/assets.go | 5392 +++++++++-------- dashboard/assets/.eslintrc | 5 +- .../{components/Common.jsx => common.jsx} | 6 + dashboard/assets/components/Body.jsx | 19 +- .../{ChartGrid.jsx => ChartRow.jsx} | 46 +- dashboard/assets/components/CustomTooltip.jsx | 95 + dashboard/assets/components/Dashboard.jsx | 109 +- dashboard/assets/components/Footer.jsx | 165 +- dashboard/assets/components/Header.jsx | 62 +- dashboard/assets/components/Home.jsx | 77 - dashboard/assets/components/Main.jsx | 39 +- dashboard/assets/components/SideBar.jsx | 84 +- dashboard/assets/index.jsx | 22 +- dashboard/assets/package-lock.json | 566 +- dashboard/assets/package.json | 11 +- dashboard/assets/types/content.jsx | 10 +- dashboard/config.go | 2 +- dashboard/cpu.go | 35 + dashboard/cpu_windows.go | 23 + dashboard/dashboard.go | 146 +- dashboard/message.go | 10 +- vendor/github.com/StackExchange/wmi/LICENSE | 20 + vendor/github.com/StackExchange/wmi/README.md | 6 + .../StackExchange/wmi/swbemservices.go | 260 + vendor/github.com/StackExchange/wmi/wmi.go | 486 ++ .../github.com/elastic/gosigar/CHANGELOG.md | 102 + vendor/github.com/elastic/gosigar/LICENSE | 201 + vendor/github.com/elastic/gosigar/NOTICE | 9 + vendor/github.com/elastic/gosigar/README.md | 57 + vendor/github.com/elastic/gosigar/Vagrantfile | 25 + vendor/github.com/elastic/gosigar/codecov.yml | 21 + .../elastic/gosigar/concrete_sigar.go | 83 + .../elastic/gosigar/sigar_darwin.go | 494 ++ .../elastic/gosigar/sigar_format.go | 126 + .../elastic/gosigar/sigar_freebsd.go | 108 + .../elastic/gosigar/sigar_interface.go | 197 + .../github.com/elastic/gosigar/sigar_linux.go | 84 + .../elastic/gosigar/sigar_linux_common.go | 468 ++ .../elastic/gosigar/sigar_openbsd.go | 418 ++ .../github.com/elastic/gosigar/sigar_stub.go | 71 + .../github.com/elastic/gosigar/sigar_unix.go | 69 + .../github.com/elastic/gosigar/sigar_util.go | 22 + .../elastic/gosigar/sigar_windows.go | 437 ++ .../elastic/gosigar/sys/windows/doc.go | 2 + .../elastic/gosigar/sys/windows/ntquery.go | 132 + .../elastic/gosigar/sys/windows/privileges.go | 272 + .../gosigar/sys/windows/syscall_windows.go | 385 ++ .../elastic/gosigar/sys/windows/version.go | 43 + .../gosigar/sys/windows/zsyscall_windows.go | 260 + vendor/github.com/go-ole/go-ole/ChangeLog.md | 49 + vendor/github.com/go-ole/go-ole/LICENSE | 21 + vendor/github.com/go-ole/go-ole/README.md | 46 + vendor/github.com/go-ole/go-ole/appveyor.yml | 54 + vendor/github.com/go-ole/go-ole/com.go | 329 + vendor/github.com/go-ole/go-ole/com_func.go | 174 + vendor/github.com/go-ole/go-ole/connect.go | 192 + vendor/github.com/go-ole/go-ole/constants.go | 153 + vendor/github.com/go-ole/go-ole/error.go | 51 + vendor/github.com/go-ole/go-ole/error_func.go | 8 + .../github.com/go-ole/go-ole/error_windows.go | 24 + vendor/github.com/go-ole/go-ole/guid.go | 284 + .../go-ole/go-ole/iconnectionpoint.go | 20 + .../go-ole/go-ole/iconnectionpoint_func.go | 21 + .../go-ole/go-ole/iconnectionpoint_windows.go | 43 + .../go-ole/iconnectionpointcontainer.go | 17 + .../go-ole/iconnectionpointcontainer_func.go | 11 + .../iconnectionpointcontainer_windows.go | 25 + vendor/github.com/go-ole/go-ole/idispatch.go | 94 + .../go-ole/go-ole/idispatch_func.go | 19 + .../go-ole/go-ole/idispatch_windows.go | 197 + .../github.com/go-ole/go-ole/ienumvariant.go | 19 + .../go-ole/go-ole/ienumvariant_func.go | 19 + .../go-ole/go-ole/ienumvariant_windows.go | 63 + .../github.com/go-ole/go-ole/iinspectable.go | 18 + .../go-ole/go-ole/iinspectable_func.go | 15 + .../go-ole/go-ole/iinspectable_windows.go | 72 + .../go-ole/go-ole/iprovideclassinfo.go | 21 + .../go-ole/go-ole/iprovideclassinfo_func.go | 7 + .../go-ole/iprovideclassinfo_windows.go | 21 + vendor/github.com/go-ole/go-ole/itypeinfo.go | 34 + .../go-ole/go-ole/itypeinfo_func.go | 7 + .../go-ole/go-ole/itypeinfo_windows.go | 21 + vendor/github.com/go-ole/go-ole/iunknown.go | 57 + .../github.com/go-ole/go-ole/iunknown_func.go | 19 + .../go-ole/go-ole/iunknown_windows.go | 58 + vendor/github.com/go-ole/go-ole/ole.go | 157 + .../go-ole/go-ole/oleutil/connection.go | 100 + .../go-ole/go-ole/oleutil/connection_func.go | 10 + .../go-ole/oleutil/connection_windows.go | 58 + .../go-ole/go-ole/oleutil/go-get.go | 6 + .../go-ole/go-ole/oleutil/oleutil.go | 127 + vendor/github.com/go-ole/go-ole/safearray.go | 27 + .../go-ole/go-ole/safearray_func.go | 211 + .../go-ole/go-ole/safearray_windows.go | 337 ++ .../go-ole/go-ole/safearrayconversion.go | 140 + .../go-ole/go-ole/safearrayslices.go | 33 + vendor/github.com/go-ole/go-ole/utility.go | 101 + vendor/github.com/go-ole/go-ole/variables.go | 16 + vendor/github.com/go-ole/go-ole/variant.go | 105 + .../github.com/go-ole/go-ole/variant_386.go | 11 + .../github.com/go-ole/go-ole/variant_amd64.go | 12 + .../github.com/go-ole/go-ole/variant_s390x.go | 12 + vendor/github.com/go-ole/go-ole/vt_string.go | 58 + vendor/github.com/go-ole/go-ole/winrt.go | 99 + vendor/github.com/go-ole/go-ole/winrt_doc.go | 36 + vendor/github.com/pkg/errors/LICENSE | 23 + vendor/github.com/pkg/errors/README.md | 52 + vendor/github.com/pkg/errors/appveyor.yml | 32 + vendor/github.com/pkg/errors/errors.go | 269 + vendor/github.com/pkg/errors/stack.go | 187 + vendor/vendor.json | 36 + 111 files changed, 13162 insertions(+), 3158 deletions(-) rename dashboard/assets/{components/Common.jsx => common.jsx} (95%) rename dashboard/assets/components/{ChartGrid.jsx => ChartRow.jsx} (55%) create mode 100644 dashboard/assets/components/CustomTooltip.jsx delete mode 100644 dashboard/assets/components/Home.jsx create mode 100644 dashboard/cpu.go create mode 100644 dashboard/cpu_windows.go create mode 100644 vendor/github.com/StackExchange/wmi/LICENSE create mode 100644 vendor/github.com/StackExchange/wmi/README.md create mode 100644 vendor/github.com/StackExchange/wmi/swbemservices.go create mode 100644 vendor/github.com/StackExchange/wmi/wmi.go create mode 100644 vendor/github.com/elastic/gosigar/CHANGELOG.md create mode 100644 vendor/github.com/elastic/gosigar/LICENSE create mode 100644 vendor/github.com/elastic/gosigar/NOTICE create mode 100644 vendor/github.com/elastic/gosigar/README.md create mode 100644 vendor/github.com/elastic/gosigar/Vagrantfile create mode 100644 vendor/github.com/elastic/gosigar/codecov.yml create mode 100644 vendor/github.com/elastic/gosigar/concrete_sigar.go create mode 100644 vendor/github.com/elastic/gosigar/sigar_darwin.go create mode 100644 vendor/github.com/elastic/gosigar/sigar_format.go create mode 100644 vendor/github.com/elastic/gosigar/sigar_freebsd.go create mode 100644 vendor/github.com/elastic/gosigar/sigar_interface.go create mode 100644 vendor/github.com/elastic/gosigar/sigar_linux.go create mode 100644 vendor/github.com/elastic/gosigar/sigar_linux_common.go create mode 100644 vendor/github.com/elastic/gosigar/sigar_openbsd.go create mode 100644 vendor/github.com/elastic/gosigar/sigar_stub.go create mode 100644 vendor/github.com/elastic/gosigar/sigar_unix.go create mode 100644 vendor/github.com/elastic/gosigar/sigar_util.go create mode 100644 vendor/github.com/elastic/gosigar/sigar_windows.go create mode 100644 vendor/github.com/elastic/gosigar/sys/windows/doc.go create mode 100644 vendor/github.com/elastic/gosigar/sys/windows/ntquery.go create mode 100644 vendor/github.com/elastic/gosigar/sys/windows/privileges.go create mode 100644 vendor/github.com/elastic/gosigar/sys/windows/syscall_windows.go create mode 100644 vendor/github.com/elastic/gosigar/sys/windows/version.go create mode 100644 vendor/github.com/elastic/gosigar/sys/windows/zsyscall_windows.go create mode 100644 vendor/github.com/go-ole/go-ole/ChangeLog.md create mode 100644 vendor/github.com/go-ole/go-ole/LICENSE create mode 100644 vendor/github.com/go-ole/go-ole/README.md create mode 100644 vendor/github.com/go-ole/go-ole/appveyor.yml create mode 100644 vendor/github.com/go-ole/go-ole/com.go create mode 100644 vendor/github.com/go-ole/go-ole/com_func.go create mode 100644 vendor/github.com/go-ole/go-ole/connect.go create mode 100644 vendor/github.com/go-ole/go-ole/constants.go create mode 100644 vendor/github.com/go-ole/go-ole/error.go create mode 100644 vendor/github.com/go-ole/go-ole/error_func.go create mode 100644 vendor/github.com/go-ole/go-ole/error_windows.go create mode 100644 vendor/github.com/go-ole/go-ole/guid.go create mode 100644 vendor/github.com/go-ole/go-ole/iconnectionpoint.go create mode 100644 vendor/github.com/go-ole/go-ole/iconnectionpoint_func.go create mode 100644 vendor/github.com/go-ole/go-ole/iconnectionpoint_windows.go create mode 100644 vendor/github.com/go-ole/go-ole/iconnectionpointcontainer.go create mode 100644 vendor/github.com/go-ole/go-ole/iconnectionpointcontainer_func.go create mode 100644 vendor/github.com/go-ole/go-ole/iconnectionpointcontainer_windows.go create mode 100644 vendor/github.com/go-ole/go-ole/idispatch.go create mode 100644 vendor/github.com/go-ole/go-ole/idispatch_func.go create mode 100644 vendor/github.com/go-ole/go-ole/idispatch_windows.go create mode 100644 vendor/github.com/go-ole/go-ole/ienumvariant.go create mode 100644 vendor/github.com/go-ole/go-ole/ienumvariant_func.go create mode 100644 vendor/github.com/go-ole/go-ole/ienumvariant_windows.go create mode 100644 vendor/github.com/go-ole/go-ole/iinspectable.go create mode 100644 vendor/github.com/go-ole/go-ole/iinspectable_func.go create mode 100644 vendor/github.com/go-ole/go-ole/iinspectable_windows.go create mode 100644 vendor/github.com/go-ole/go-ole/iprovideclassinfo.go create mode 100644 vendor/github.com/go-ole/go-ole/iprovideclassinfo_func.go create mode 100644 vendor/github.com/go-ole/go-ole/iprovideclassinfo_windows.go create mode 100644 vendor/github.com/go-ole/go-ole/itypeinfo.go create mode 100644 vendor/github.com/go-ole/go-ole/itypeinfo_func.go create mode 100644 vendor/github.com/go-ole/go-ole/itypeinfo_windows.go create mode 100644 vendor/github.com/go-ole/go-ole/iunknown.go create mode 100644 vendor/github.com/go-ole/go-ole/iunknown_func.go create mode 100644 vendor/github.com/go-ole/go-ole/iunknown_windows.go create mode 100644 vendor/github.com/go-ole/go-ole/ole.go create mode 100644 vendor/github.com/go-ole/go-ole/oleutil/connection.go create mode 100644 vendor/github.com/go-ole/go-ole/oleutil/connection_func.go create mode 100644 vendor/github.com/go-ole/go-ole/oleutil/connection_windows.go create mode 100644 vendor/github.com/go-ole/go-ole/oleutil/go-get.go create mode 100644 vendor/github.com/go-ole/go-ole/oleutil/oleutil.go create mode 100644 vendor/github.com/go-ole/go-ole/safearray.go create mode 100644 vendor/github.com/go-ole/go-ole/safearray_func.go create mode 100644 vendor/github.com/go-ole/go-ole/safearray_windows.go create mode 100644 vendor/github.com/go-ole/go-ole/safearrayconversion.go create mode 100644 vendor/github.com/go-ole/go-ole/safearrayslices.go create mode 100644 vendor/github.com/go-ole/go-ole/utility.go create mode 100644 vendor/github.com/go-ole/go-ole/variables.go create mode 100644 vendor/github.com/go-ole/go-ole/variant.go create mode 100644 vendor/github.com/go-ole/go-ole/variant_386.go create mode 100644 vendor/github.com/go-ole/go-ole/variant_amd64.go create mode 100644 vendor/github.com/go-ole/go-ole/variant_s390x.go create mode 100644 vendor/github.com/go-ole/go-ole/vt_string.go create mode 100644 vendor/github.com/go-ole/go-ole/winrt.go create mode 100644 vendor/github.com/go-ole/go-ole/winrt_doc.go create mode 100644 vendor/github.com/pkg/errors/LICENSE create mode 100644 vendor/github.com/pkg/errors/README.md create mode 100644 vendor/github.com/pkg/errors/appveyor.yml create mode 100644 vendor/github.com/pkg/errors/errors.go create mode 100644 vendor/github.com/pkg/errors/stack.go diff --git a/dashboard/assets.go b/dashboard/assets.go index b2c1203234..8337cf080d 100644 --- a/dashboard/assets.go +++ b/dashboard/assets.go @@ -6,6 +6,7 @@ package dashboard import ( + "crypto/sha256" "fmt" "io/ioutil" "os" @@ -15,8 +16,9 @@ import ( ) type asset struct { - bytes []byte - info os.FileInfo + bytes []byte + info os.FileInfo + digest [sha256.Size]byte } type bindataFileInfo struct { @@ -82,7 +84,7 @@ func dashboardHtml() (*asset, error) { } info := bindataFileInfo{name: "dashboard.html", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x6b, 0xd9, 0xa6, 0xeb, 0x32, 0x49, 0x9b, 0xe5, 0x3a, 0xcb, 0x99, 0xd3, 0xb6, 0x69, 0x7f, 0xde, 0x35, 0x9d, 0x5, 0x96, 0x84, 0xc0, 0x14, 0xef, 0xbe, 0x58, 0x10, 0x5e, 0x40, 0xf2, 0x12, 0x97}} return a, nil } @@ -114,11 +116,11 @@ var _bundleJs = []byte((((((((((`!function(modules) { return __webpack_require__.d(getter, "a", getter), getter; }, __webpack_require__.o = function(object, property) { return Object.prototype.hasOwnProperty.call(object, property); - }, __webpack_require__.p = "", __webpack_require__(__webpack_require__.s = 336); + }, __webpack_require__.p = "", __webpack_require__(__webpack_require__.s = 331); }([ function(module, exports, __webpack_require__) { "use strict"; (function(process) { - "production" === process.env.NODE_ENV ? module.exports = __webpack_require__(337) : module.exports = __webpack_require__(338); + "production" === process.env.NODE_ENV ? module.exports = __webpack_require__(332) : module.exports = __webpack_require__(333); }).call(exports, __webpack_require__(2)); }, function(module, exports, __webpack_require__) { (function(process) { @@ -126,8 +128,8 @@ var _bundleJs = []byte((((((((((`!function(modules) { var REACT_ELEMENT_TYPE = "function" == typeof Symbol && Symbol.for && Symbol.for("react.element") || 60103, isValidElement = function(object) { return "object" == typeof object && null !== object && object.$$typeof === REACT_ELEMENT_TYPE; }; - module.exports = __webpack_require__(379)(isValidElement, !0); - } else module.exports = __webpack_require__(380)(); + module.exports = __webpack_require__(374)(isValidElement, !0); + } else module.exports = __webpack_require__(375)(); }).call(exports, __webpack_require__(2)); }, function(module, exports) { function defaultSetTimout() { @@ -247,6 +249,11 @@ var _bundleJs = []byte((((((((((`!function(modules) { } return Array.from(arr); } + function _objectWithoutProperties(obj, keys) { + var target = {}; + for (var i in obj) keys.indexOf(i) >= 0 || Object.prototype.hasOwnProperty.call(obj, i) && (target[i] = obj[i]); + return target; + } __webpack_require__.d(__webpack_exports__, "c", function() { return PRESENTATION_ATTRIBUTES; }), __webpack_require__.d(__webpack_exports__, "a", function() { @@ -282,7 +289,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }), __webpack_require__.d(__webpack_exports__, "o", function() { return parseChildIndex; }); - var __WEBPACK_IMPORTED_MODULE_0_lodash_isNil__ = __webpack_require__(20), __WEBPACK_IMPORTED_MODULE_0_lodash_isNil___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_isNil__), __WEBPACK_IMPORTED_MODULE_1_lodash_isString__ = __webpack_require__(165), __WEBPACK_IMPORTED_MODULE_1_lodash_isString___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_lodash_isString__), __WEBPACK_IMPORTED_MODULE_2_lodash_isObject__ = __webpack_require__(32), __WEBPACK_IMPORTED_MODULE_2_lodash_isObject___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_lodash_isObject__), __WEBPACK_IMPORTED_MODULE_3_lodash_isFunction__ = __webpack_require__(8), __WEBPACK_IMPORTED_MODULE_3_lodash_isFunction___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_lodash_isFunction__), __WEBPACK_IMPORTED_MODULE_4_lodash_isArray__ = __webpack_require__(12), __WEBPACK_IMPORTED_MODULE_4_lodash_isArray___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_lodash_isArray__), __WEBPACK_IMPORTED_MODULE_5_react__ = __webpack_require__(0), __WEBPACK_IMPORTED_MODULE_5_react___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_5_react__), __WEBPACK_IMPORTED_MODULE_6_prop_types__ = __webpack_require__(1), __WEBPACK_IMPORTED_MODULE_6_prop_types___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_6_prop_types__), __WEBPACK_IMPORTED_MODULE_7__DataUtils__ = __webpack_require__(9), __WEBPACK_IMPORTED_MODULE_8__PureRender__ = __webpack_require__(5), PRESENTATION_ATTRIBUTES = { + var __WEBPACK_IMPORTED_MODULE_0_lodash_isNil__ = __webpack_require__(20), __WEBPACK_IMPORTED_MODULE_0_lodash_isNil___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_isNil__), __WEBPACK_IMPORTED_MODULE_1_lodash_isString__ = __webpack_require__(164), __WEBPACK_IMPORTED_MODULE_1_lodash_isString___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_lodash_isString__), __WEBPACK_IMPORTED_MODULE_2_lodash_isObject__ = __webpack_require__(31), __WEBPACK_IMPORTED_MODULE_2_lodash_isObject___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_lodash_isObject__), __WEBPACK_IMPORTED_MODULE_3_lodash_isFunction__ = __webpack_require__(8), __WEBPACK_IMPORTED_MODULE_3_lodash_isFunction___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_lodash_isFunction__), __WEBPACK_IMPORTED_MODULE_4_lodash_isArray__ = __webpack_require__(12), __WEBPACK_IMPORTED_MODULE_4_lodash_isArray___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_lodash_isArray__), __WEBPACK_IMPORTED_MODULE_5_react__ = __webpack_require__(0), __WEBPACK_IMPORTED_MODULE_5_react___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_5_react__), __WEBPACK_IMPORTED_MODULE_6_prop_types__ = __webpack_require__(1), __WEBPACK_IMPORTED_MODULE_6_prop_types___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_6_prop_types__), __WEBPACK_IMPORTED_MODULE_7__DataUtils__ = __webpack_require__(9), __WEBPACK_IMPORTED_MODULE_8__PureRender__ = __webpack_require__(5), PRESENTATION_ATTRIBUTES = { alignmentBaseline: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.string, angle: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.number, baselineShift: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.string, @@ -438,7 +445,12 @@ var _bundleJs = []byte((((((((((`!function(modules) { entry && entry.type && __WEBPACK_IMPORTED_MODULE_1_lodash_isString___default()(entry.type) && SVG_TAGS.indexOf(entry.type) >= 0 && svgElements.push(entry); }), svgElements; }, isSingleChildEqual = function(nextChild, prevChild) { - return !(!__WEBPACK_IMPORTED_MODULE_0_lodash_isNil___default()(nextChild) || !__WEBPACK_IMPORTED_MODULE_0_lodash_isNil___default()(prevChild)) || !__WEBPACK_IMPORTED_MODULE_0_lodash_isNil___default()(nextChild) && !__WEBPACK_IMPORTED_MODULE_0_lodash_isNil___default()(prevChild) && Object(__WEBPACK_IMPORTED_MODULE_8__PureRender__.b)(nextChild.props, prevChild.props); + if (__WEBPACK_IMPORTED_MODULE_0_lodash_isNil___default()(nextChild) && __WEBPACK_IMPORTED_MODULE_0_lodash_isNil___default()(prevChild)) return !0; + if (!__WEBPACK_IMPORTED_MODULE_0_lodash_isNil___default()(nextChild) && !__WEBPACK_IMPORTED_MODULE_0_lodash_isNil___default()(prevChild)) { + var _ref = nextChild.props || {}, nextChildren = _ref.children, nextProps = _objectWithoutProperties(_ref, [ "children" ]), _ref2 = prevChild.props || {}, prevChildren = _ref2.children, prevProps = _objectWithoutProperties(_ref2, [ "children" ]); + return nextChildren && prevChildren ? Object(__WEBPACK_IMPORTED_MODULE_8__PureRender__.b)(nextProps, prevProps) && isChildrenEqual(nextChildren, prevChildren) : !nextChildren && !prevChildren && Object(__WEBPACK_IMPORTED_MODULE_8__PureRender__.b)(nextProps, prevProps); + } + return !1; }, isChildrenEqual = function isChildrenEqual(nextChildren, prevChildren) { if (nextChildren === prevChildren) return !0; if (__WEBPACK_IMPORTED_MODULE_5_react__.Children.count(nextChildren) !== __WEBPACK_IMPORTED_MODULE_5_react__.Children.count(prevChildren)) return !1; @@ -497,7 +509,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, function(module, exports, __webpack_require__) { "use strict"; exports.__esModule = !0; - var _assign = __webpack_require__(206), _assign2 = function(obj) { + var _assign = __webpack_require__(205), _assign2 = function(obj) { return obj && obj.__esModule ? obj : { default: obj }; @@ -515,7 +527,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { var tag = baseGetTag(value); return tag == funcTag || tag == genTag || tag == asyncTag || tag == proxyTag; } - var baseGetTag = __webpack_require__(42), isObject = __webpack_require__(32), asyncTag = "[object AsyncFunction]", funcTag = "[object Function]", genTag = "[object GeneratorFunction]", proxyTag = "[object Proxy]"; + var baseGetTag = __webpack_require__(42), isObject = __webpack_require__(31), asyncTag = "[object AsyncFunction]", funcTag = "[object Function]", genTag = "[object GeneratorFunction]", proxyTag = "[object Proxy]"; module.exports = isFunction; }, function(module, __webpack_exports__, __webpack_require__) { "use strict"; @@ -540,7 +552,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }), __webpack_require__.d(__webpack_exports__, "a", function() { return findEntryInArray; }); - var __WEBPACK_IMPORTED_MODULE_0_lodash_get__ = __webpack_require__(109), __WEBPACK_IMPORTED_MODULE_0_lodash_get___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_get__), __WEBPACK_IMPORTED_MODULE_1_lodash_isArray__ = __webpack_require__(12), __WEBPACK_IMPORTED_MODULE_1_lodash_isArray___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_lodash_isArray__), __WEBPACK_IMPORTED_MODULE_2_lodash_isNaN__ = __webpack_require__(116), __WEBPACK_IMPORTED_MODULE_2_lodash_isNaN___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_lodash_isNaN__), __WEBPACK_IMPORTED_MODULE_3_lodash_isNumber__ = __webpack_require__(170), __WEBPACK_IMPORTED_MODULE_3_lodash_isNumber___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_lodash_isNumber__), __WEBPACK_IMPORTED_MODULE_4_lodash_isString__ = __webpack_require__(165), __WEBPACK_IMPORTED_MODULE_4_lodash_isString___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_lodash_isString__), mathSign = function(value) { + var __WEBPACK_IMPORTED_MODULE_0_lodash_get__ = __webpack_require__(165), __WEBPACK_IMPORTED_MODULE_0_lodash_get___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_get__), __WEBPACK_IMPORTED_MODULE_1_lodash_isArray__ = __webpack_require__(12), __WEBPACK_IMPORTED_MODULE_1_lodash_isArray___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_lodash_isArray__), __WEBPACK_IMPORTED_MODULE_2_lodash_isNaN__ = __webpack_require__(117), __WEBPACK_IMPORTED_MODULE_2_lodash_isNaN___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_lodash_isNaN__), __WEBPACK_IMPORTED_MODULE_3_lodash_isNumber__ = __webpack_require__(170), __WEBPACK_IMPORTED_MODULE_3_lodash_isNumber___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_lodash_isNumber__), __WEBPACK_IMPORTED_MODULE_4_lodash_isString__ = __webpack_require__(164), __WEBPACK_IMPORTED_MODULE_4_lodash_isString___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_lodash_isString__), mathSign = function(value) { return 0 === value ? 0 : value > 0 ? 1 : -1; }, isPercent = function(value) { return __WEBPACK_IMPORTED_MODULE_4_lodash_isString___default()(value) && value.indexOf("%") === value.length - 1; @@ -596,42 +608,38 @@ var _bundleJs = []byte((((((((((`!function(modules) { } Object.defineProperty(exports, "__esModule", { value: !0 - }), exports.sheetsManager = exports.preset = void 0; - var _keys = __webpack_require__(36), _keys2 = _interopRequireDefault(_keys), _extends2 = __webpack_require__(7), _extends3 = _interopRequireDefault(_extends2), _getPrototypeOf = __webpack_require__(26), _getPrototypeOf2 = _interopRequireDefault(_getPrototypeOf), _classCallCheck2 = __webpack_require__(27), _classCallCheck3 = _interopRequireDefault(_classCallCheck2), _createClass2 = __webpack_require__(28), _createClass3 = _interopRequireDefault(_createClass2), _possibleConstructorReturn2 = __webpack_require__(29), _possibleConstructorReturn3 = _interopRequireDefault(_possibleConstructorReturn2), _inherits2 = __webpack_require__(30), _inherits3 = _interopRequireDefault(_inherits2), _objectWithoutProperties2 = __webpack_require__(6), _objectWithoutProperties3 = _interopRequireDefault(_objectWithoutProperties2), _map = __webpack_require__(402), _map2 = _interopRequireDefault(_map), _minSafeInteger = __webpack_require__(418), _minSafeInteger2 = _interopRequireDefault(_minSafeInteger), _react = __webpack_require__(0), _react2 = _interopRequireDefault(_react), _propTypes = __webpack_require__(1), _propTypes2 = _interopRequireDefault(_propTypes), _warning = __webpack_require__(11), _warning2 = _interopRequireDefault(_warning), _hoistNonReactStatics = __webpack_require__(151), _hoistNonReactStatics2 = _interopRequireDefault(_hoistNonReactStatics), _wrapDisplayName = __webpack_require__(74), _wrapDisplayName2 = _interopRequireDefault(_wrapDisplayName), _getDisplayName = __webpack_require__(227), _getDisplayName2 = _interopRequireDefault(_getDisplayName), _contextTypes = __webpack_require__(421), _contextTypes2 = _interopRequireDefault(_contextTypes), _jss = __webpack_require__(229), _jssGlobal = __webpack_require__(444), _jssGlobal2 = _interopRequireDefault(_jssGlobal), _jssNested = __webpack_require__(445), _jssNested2 = _interopRequireDefault(_jssNested), _jssCamelCase = __webpack_require__(446), _jssCamelCase2 = _interopRequireDefault(_jssCamelCase), _jssDefaultUnit = __webpack_require__(447), _jssDefaultUnit2 = _interopRequireDefault(_jssDefaultUnit), _jssVendorPrefixer = __webpack_require__(449), _jssVendorPrefixer2 = _interopRequireDefault(_jssVendorPrefixer), _jssPropsSort = __webpack_require__(454), _jssPropsSort2 = _interopRequireDefault(_jssPropsSort), _ns = __webpack_require__(228), ns = function(obj) { + }), exports.sheetsManager = void 0; + var _keys = __webpack_require__(41), _keys2 = _interopRequireDefault(_keys), _extends2 = __webpack_require__(7), _extends3 = _interopRequireDefault(_extends2), _getPrototypeOf = __webpack_require__(26), _getPrototypeOf2 = _interopRequireDefault(_getPrototypeOf), _classCallCheck2 = __webpack_require__(27), _classCallCheck3 = _interopRequireDefault(_classCallCheck2), _createClass2 = __webpack_require__(28), _createClass3 = _interopRequireDefault(_createClass2), _possibleConstructorReturn2 = __webpack_require__(29), _possibleConstructorReturn3 = _interopRequireDefault(_possibleConstructorReturn2), _inherits2 = __webpack_require__(30), _inherits3 = _interopRequireDefault(_inherits2), _objectWithoutProperties2 = __webpack_require__(6), _objectWithoutProperties3 = _interopRequireDefault(_objectWithoutProperties2), _map = __webpack_require__(397), _map2 = _interopRequireDefault(_map), _minSafeInteger = __webpack_require__(413), _minSafeInteger2 = _interopRequireDefault(_minSafeInteger), _react = __webpack_require__(0), _react2 = _interopRequireDefault(_react), _propTypes = __webpack_require__(1), _propTypes2 = _interopRequireDefault(_propTypes), _warning = __webpack_require__(11), _warning2 = _interopRequireDefault(_warning), _hoistNonReactStatics = __webpack_require__(152), _hoistNonReactStatics2 = _interopRequireDefault(_hoistNonReactStatics), _getDisplayName = __webpack_require__(226), _getDisplayName2 = _interopRequireDefault(_getDisplayName), _wrapDisplayName = __webpack_require__(75), _wrapDisplayName2 = _interopRequireDefault(_wrapDisplayName), _contextTypes = __webpack_require__(416), _contextTypes2 = _interopRequireDefault(_contextTypes), _jss = __webpack_require__(228), _ns = __webpack_require__(227), ns = function(obj) { if (obj && obj.__esModule) return obj; var newObj = {}; if (null != obj) for (var key in obj) Object.prototype.hasOwnProperty.call(obj, key) && (newObj[key] = obj[key]); return newObj.default = obj, newObj; - }(_ns), _createMuiTheme = __webpack_require__(150), _createMuiTheme2 = _interopRequireDefault(_createMuiTheme), _themeListener = __webpack_require__(149), _themeListener2 = _interopRequireDefault(_themeListener), _createGenerateClassName = __webpack_require__(455), _createGenerateClassName2 = _interopRequireDefault(_createGenerateClassName), _getStylesCreator = __webpack_require__(456), _getStylesCreator2 = _interopRequireDefault(_getStylesCreator), preset = exports.preset = function() { - return { - plugins: [ (0, _jssGlobal2.default)(), (0, _jssNested2.default)(), (0, _jssCamelCase2.default)(), (0, - _jssDefaultUnit2.default)(), (0, _jssVendorPrefixer2.default)(), (0, _jssPropsSort2.default)() ] - }; - }, jss = (0, _jss.create)(preset()), generateClassName = (0, _createGenerateClassName2.default)(), indexCounter = _minSafeInteger2.default, sheetsManager = exports.sheetsManager = new _map2.default(), noopTheme = {}, defaultTheme = void 0, withStyles = function(stylesOrCreator) { + }(_ns), _jssPreset = __webpack_require__(439), _jssPreset2 = _interopRequireDefault(_jssPreset), _createMuiTheme = __webpack_require__(151), _createMuiTheme2 = _interopRequireDefault(_createMuiTheme), _themeListener = __webpack_require__(150), _themeListener2 = _interopRequireDefault(_themeListener), _createGenerateClassName = __webpack_require__(451), _createGenerateClassName2 = _interopRequireDefault(_createGenerateClassName), _getStylesCreator = __webpack_require__(452), _getStylesCreator2 = _interopRequireDefault(_getStylesCreator), jss = (0, + _jss.create)((0, _jssPreset2.default)()), generateClassName = (0, _createGenerateClassName2.default)(), indexCounter = _minSafeInteger2.default, sheetsManager = exports.sheetsManager = new _map2.default(), noopTheme = {}, defaultTheme = void 0, withStyles = function(stylesOrCreator) { var options = arguments.length > 1 && void 0 !== arguments[1] ? arguments[1] : {}; return function(Component) { - var _options$withTheme = options.withTheme, withTheme = void 0 !== _options$withTheme && _options$withTheme, flip = options.flip, name = options.name, styleSheetOptions = (0, + var _options$withTheme = options.withTheme, withTheme = void 0 !== _options$withTheme && _options$withTheme, _options$flip = options.flip, flip = void 0 === _options$flip ? null : _options$flip, name = options.name, styleSheetOptions = (0, _objectWithoutProperties3.default)(options, [ "withTheme", "flip", "name" ]), stylesCreator = (0, _getStylesCreator2.default)(stylesOrCreator), listenToTheme = stylesCreator.themingEnabled || withTheme || "string" == typeof name; - void 0 === stylesCreator.options.index && (indexCounter += 1, stylesCreator.options.index = indexCounter), - "production" !== process.env.NODE_ENV && (0, _warning2.default)(indexCounter < 0, [ "Material-UI: you might have a memory leak.", "The indexCounter is not supposed to grow that much." ].join(" ")); - var Style = function(_React$Component) { - function Style(props, context) { - (0, _classCallCheck3.default)(this, Style); - var _this = (0, _possibleConstructorReturn3.default)(this, (Style.__proto__ || (0, - _getPrototypeOf2.default)(Style)).call(this, props, context)); - _this.state = {}, _this.unsubscribeId = null, _this.jss = null, _this.sheetsManager = sheetsManager, - _this.disableStylesGeneration = !1, _this.stylesCreatorSaved = null, _this.theme = null, - _this.sheetOptions = null, _this.theme = null; + indexCounter += 1, stylesCreator.options.index = indexCounter, "production" !== process.env.NODE_ENV && (0, + _warning2.default)(indexCounter < 0, [ "Material-UI: you might have a memory leak.", "The indexCounter is not supposed to grow that much." ].join(" ")); + var WithStyles = function(_React$Component) { + function WithStyles(props, context) { + (0, _classCallCheck3.default)(this, WithStyles); + var _this = (0, _possibleConstructorReturn3.default)(this, (WithStyles.__proto__ || (0, + _getPrototypeOf2.default)(WithStyles)).call(this, props, context)); + _this.state = {}, _this.disableStylesGeneration = !1, _this.jss = null, _this.sheetOptions = null, + _this.sheetsManager = sheetsManager, _this.stylesCreatorSaved = null, _this.theme = null, + _this.unsubscribeId = null, _this.jss = _this.context[ns.jss] || jss; var muiThemeProviderOptions = _this.context.muiThemeProviderOptions; - return _this.jss = _this.context[ns.jss] || jss, muiThemeProviderOptions && (muiThemeProviderOptions.sheetsManager && (_this.sheetsManager = muiThemeProviderOptions.sheetsManager), + return muiThemeProviderOptions && (muiThemeProviderOptions.sheetsManager && (_this.sheetsManager = muiThemeProviderOptions.sheetsManager), _this.disableStylesGeneration = muiThemeProviderOptions.disableStylesGeneration), _this.stylesCreatorSaved = stylesCreator, _this.sheetOptions = (0, _extends3.default)({ generateClassName: generateClassName }, _this.context[ns.sheetOptions]), _this.theme = listenToTheme ? _themeListener2.default.initial(context) || getDefaultTheme() : noopTheme, _this; } - return (0, _inherits3.default)(Style, _React$Component), (0, _createClass3.default)(Style, [ { + return (0, _inherits3.default)(WithStyles, _React$Component), (0, _createClass3.default)(WithStyles, [ { key: "componentWillMount", value: function() { this.attach(this.theme); @@ -669,10 +677,11 @@ var _bundleJs = []byte((((((((((`!function(modules) { refs: 0, sheet: null }, sheetManager.set(theme, sheetManagerTheme)), 0 === sheetManagerTheme.refs) { - var styles = stylesCreatorSaved.create(theme, name), meta = void 0; - "production" !== process.env.NODE_ENV && (meta = name || (0, _getDisplayName2.default)(Component)); + var styles = stylesCreatorSaved.create(theme, name), meta = name; + "production" === process.env.NODE_ENV || meta || (meta = (0, _getDisplayName2.default)(Component)); var sheet = this.jss.createStyleSheet(styles, (0, _extends3.default)({ meta: meta, + classNamePrefix: meta, flip: "boolean" == typeof flip ? flip : "rtl" === theme.direction, link: !1 }, this.sheetOptions, stylesCreatorSaved.options, { @@ -723,17 +732,17 @@ var _bundleJs = []byte((((((((((`!function(modules) { ref: innerRef })); } - } ]), Style; + } ]), WithStyles; }(_react2.default.Component); - return Style.propTypes = "production" !== process.env.NODE_ENV ? { + return WithStyles.propTypes = "production" !== process.env.NODE_ENV ? { classes: _propTypes2.default.object, innerRef: _propTypes2.default.func - } : {}, Style.contextTypes = (0, _extends3.default)({ + } : {}, WithStyles.contextTypes = (0, _extends3.default)({ muiThemeProviderOptions: _propTypes2.default.object }, _contextTypes2.default, listenToTheme ? _themeListener2.default.contextTypes : {}), - "production" !== process.env.NODE_ENV && (Style.displayName = (0, _wrapDisplayName2.default)(Component, "withStyles")), - (0, _hoistNonReactStatics2.default)(Style, Component), "production" !== process.env.NODE_ENV && (Style.Naked = Component, - Style.options = options), Style; + "production" !== process.env.NODE_ENV && (WithStyles.displayName = (0, _wrapDisplayName2.default)(Component, "WithStyles")), + (0, _hoistNonReactStatics2.default)(WithStyles, Component), "production" !== process.env.NODE_ENV && (WithStyles.Naked = Component, + WithStyles.options = options), WithStyles; }; }; exports.default = withStyles; @@ -765,7 +774,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, function(module, exports, __webpack_require__) { "use strict"; exports.__esModule = !0; - var _defineProperty = __webpack_require__(142), _defineProperty2 = function(obj) { + var _defineProperty = __webpack_require__(143), _defineProperty2 = function(obj) { return obj && obj.__esModule ? obj : { default: obj }; @@ -803,7 +812,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }; Layer.propTypes = propTypes, __webpack_exports__.a = Layer; }, function(module, exports, __webpack_require__) { - var global = __webpack_require__(159), core = __webpack_require__(160), hide = __webpack_require__(246), redefine = __webpack_require__(521), ctx = __webpack_require__(524), $export = function(type, name, source) { + var global = __webpack_require__(158), core = __webpack_require__(159), hide = __webpack_require__(244), redefine = __webpack_require__(534), ctx = __webpack_require__(537), $export = function(type, name, source) { var key, own, out, exp, IS_FORCED = type & $export.F, IS_GLOBAL = type & $export.G, IS_STATIC = type & $export.S, IS_PROTO = type & $export.P, IS_BIND = type & $export.B, target = IS_GLOBAL ? global : IS_STATIC ? global[name] || (global[name] = {}) : (global[name] || {}).prototype, exports = IS_GLOBAL ? core : core[name] || (core[name] = {}), expProto = exports.prototype || (exports.prototype = {}); IS_GLOBAL && (source = name); for (key in source) own = !IS_FORCED && target && void 0 !== target[key], out = (own ? target : source)[key], @@ -891,8 +900,8 @@ var _bundleJs = []byte((((((((((`!function(modules) { }), __webpack_require__.d(__webpack_exports__, "y", function() { return parseDomainOfCategoryAxis; }); - var __WEBPACK_IMPORTED_MODULE_0_lodash_isEqual__ = __webpack_require__(34), __WEBPACK_IMPORTED_MODULE_0_lodash_isEqual___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_isEqual__), __WEBPACK_IMPORTED_MODULE_1_lodash_sortBy__ = __webpack_require__(285), __WEBPACK_IMPORTED_MODULE_1_lodash_sortBy___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_lodash_sortBy__), __WEBPACK_IMPORTED_MODULE_2_lodash_isNaN__ = __webpack_require__(116), __WEBPACK_IMPORTED_MODULE_2_lodash_isNaN___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_lodash_isNaN__), __WEBPACK_IMPORTED_MODULE_3_lodash_isString__ = __webpack_require__(165), __WEBPACK_IMPORTED_MODULE_3_lodash_isString___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_lodash_isString__), __WEBPACK_IMPORTED_MODULE_4_lodash_max__ = __webpack_require__(692), __WEBPACK_IMPORTED_MODULE_4_lodash_max___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_lodash_max__), __WEBPACK_IMPORTED_MODULE_5_lodash_min__ = __webpack_require__(288), __WEBPACK_IMPORTED_MODULE_5_lodash_min___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_5_lodash_min__), __WEBPACK_IMPORTED_MODULE_6_lodash_isArray__ = __webpack_require__(12), __WEBPACK_IMPORTED_MODULE_6_lodash_isArray___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_6_lodash_isArray__), __WEBPACK_IMPORTED_MODULE_7_lodash_isFunction__ = __webpack_require__(8), __WEBPACK_IMPORTED_MODULE_7_lodash_isFunction___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_7_lodash_isFunction__), __WEBPACK_IMPORTED_MODULE_8_lodash_get__ = __webpack_require__(109), __WEBPACK_IMPORTED_MODULE_8_lodash_get___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_8_lodash_get__), __WEBPACK_IMPORTED_MODULE_9_lodash_isNil__ = __webpack_require__(20), __WEBPACK_IMPORTED_MODULE_9_lodash_isNil___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_9_lodash_isNil__), __WEBPACK_IMPORTED_MODULE_10_recharts_scale__ = __webpack_require__(693), __WEBPACK_IMPORTED_MODULE_11_d3_scale__ = (__webpack_require__.n(__WEBPACK_IMPORTED_MODULE_10_recharts_scale__), - __webpack_require__(291)), __WEBPACK_IMPORTED_MODULE_12_d3_shape__ = __webpack_require__(173), __WEBPACK_IMPORTED_MODULE_13__DataUtils__ = __webpack_require__(9), __WEBPACK_IMPORTED_MODULE_14__cartesian_ReferenceDot__ = __webpack_require__(324), __WEBPACK_IMPORTED_MODULE_15__cartesian_ReferenceLine__ = __webpack_require__(325), __WEBPACK_IMPORTED_MODULE_16__cartesian_ReferenceArea__ = __webpack_require__(326), __WEBPACK_IMPORTED_MODULE_17__cartesian_ErrorBar__ = __webpack_require__(90), __WEBPACK_IMPORTED_MODULE_18__component_Legend__ = __webpack_require__(171), __WEBPACK_IMPORTED_MODULE_19__ReactUtils__ = __webpack_require__(4), _extends = Object.assign || function(target) { + var __WEBPACK_IMPORTED_MODULE_0_lodash_isEqual__ = __webpack_require__(34), __WEBPACK_IMPORTED_MODULE_0_lodash_isEqual___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_isEqual__), __WEBPACK_IMPORTED_MODULE_1_lodash_sortBy__ = __webpack_require__(281), __WEBPACK_IMPORTED_MODULE_1_lodash_sortBy___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_lodash_sortBy__), __WEBPACK_IMPORTED_MODULE_2_lodash_isNaN__ = __webpack_require__(117), __WEBPACK_IMPORTED_MODULE_2_lodash_isNaN___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_lodash_isNaN__), __WEBPACK_IMPORTED_MODULE_3_lodash_isString__ = __webpack_require__(164), __WEBPACK_IMPORTED_MODULE_3_lodash_isString___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_lodash_isString__), __WEBPACK_IMPORTED_MODULE_4_lodash_max__ = __webpack_require__(702), __WEBPACK_IMPORTED_MODULE_4_lodash_max___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_lodash_max__), __WEBPACK_IMPORTED_MODULE_5_lodash_min__ = __webpack_require__(284), __WEBPACK_IMPORTED_MODULE_5_lodash_min___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_5_lodash_min__), __WEBPACK_IMPORTED_MODULE_6_lodash_isArray__ = __webpack_require__(12), __WEBPACK_IMPORTED_MODULE_6_lodash_isArray___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_6_lodash_isArray__), __WEBPACK_IMPORTED_MODULE_7_lodash_isFunction__ = __webpack_require__(8), __WEBPACK_IMPORTED_MODULE_7_lodash_isFunction___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_7_lodash_isFunction__), __WEBPACK_IMPORTED_MODULE_8_lodash_get__ = __webpack_require__(165), __WEBPACK_IMPORTED_MODULE_8_lodash_get___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_8_lodash_get__), __WEBPACK_IMPORTED_MODULE_9_lodash_isNil__ = __webpack_require__(20), __WEBPACK_IMPORTED_MODULE_9_lodash_isNil___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_9_lodash_isNil__), __WEBPACK_IMPORTED_MODULE_10_recharts_scale__ = __webpack_require__(703), __WEBPACK_IMPORTED_MODULE_11_d3_scale__ = (__webpack_require__.n(__WEBPACK_IMPORTED_MODULE_10_recharts_scale__), + __webpack_require__(287)), __WEBPACK_IMPORTED_MODULE_12_d3_shape__ = __webpack_require__(173), __WEBPACK_IMPORTED_MODULE_13__DataUtils__ = __webpack_require__(9), __WEBPACK_IMPORTED_MODULE_14__cartesian_ReferenceDot__ = __webpack_require__(320), __WEBPACK_IMPORTED_MODULE_15__cartesian_ReferenceLine__ = __webpack_require__(321), __WEBPACK_IMPORTED_MODULE_16__cartesian_ReferenceArea__ = __webpack_require__(322), __WEBPACK_IMPORTED_MODULE_17__cartesian_ErrorBar__ = __webpack_require__(91), __WEBPACK_IMPORTED_MODULE_18__component_Legend__ = __webpack_require__(171), __WEBPACK_IMPORTED_MODULE_19__ReactUtils__ = __webpack_require__(4), _extends = Object.assign || function(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) Object.prototype.hasOwnProperty.call(source, key) && (target[key] = source[key]); @@ -1427,7 +1436,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { __webpack_exports__.a = newInterval; var t0 = new Date(), t1 = new Date(); }, function(module, exports, __webpack_require__) { - var global = __webpack_require__(24), core = __webpack_require__(17), ctx = __webpack_require__(47), hide = __webpack_require__(41), $export = function(type, name, source) { + var global = __webpack_require__(24), core = __webpack_require__(17), ctx = __webpack_require__(47), hide = __webpack_require__(40), $export = function(type, name, source) { var key, own, out, IS_FORCED = type & $export.F, IS_GLOBAL = type & $export.G, IS_STATIC = type & $export.S, IS_PROTO = type & $export.P, IS_BIND = type & $export.B, IS_WRAP = type & $export.W, exports = IS_GLOBAL ? core : core[name] || (core[name] = {}), expProto = exports.prototype, target = IS_GLOBAL ? global : IS_STATIC ? global[name] : (global[name] || {}).prototype; IS_GLOBAL && (source = name); for (key in source) (own = !IS_FORCED && target && void 0 !== target[key]) && key in exports || (out = own ? target[key] : source[key], @@ -1460,12 +1469,12 @@ var _bundleJs = []byte((((((((((`!function(modules) { } module.exports = isNil; }, function(module, exports, __webpack_require__) { - var store = __webpack_require__(139)("wks"), uid = __webpack_require__(97), Symbol = __webpack_require__(24).Symbol, USE_SYMBOL = "function" == typeof Symbol; + var store = __webpack_require__(140)("wks"), uid = __webpack_require__(98), Symbol = __webpack_require__(24).Symbol, USE_SYMBOL = "function" == typeof Symbol; (module.exports = function(name) { return store[name] || (store[name] = USE_SYMBOL && Symbol[name] || (USE_SYMBOL ? Symbol : uid)("Symbol." + name)); }).store = store; }, function(module, exports, __webpack_require__) { - var anObject = __webpack_require__(48), IE8_DOM_DEFINE = __webpack_require__(208), toPrimitive = __webpack_require__(133), dP = Object.defineProperty; + var anObject = __webpack_require__(48), IE8_DOM_DEFINE = __webpack_require__(207), toPrimitive = __webpack_require__(134), dP = Object.defineProperty; exports.f = __webpack_require__(25) ? Object.defineProperty : function(O, P, Attributes) { if (anObject(O), P = toPrimitive(P, !0), anObject(Attributes), IE8_DOM_DEFINE) try { return dP(O, P, Attributes); @@ -1604,7 +1613,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }); }, function(module, exports, __webpack_require__) { module.exports = { - default: __webpack_require__(355), + default: __webpack_require__(350), __esModule: !0 }; }, function(module, exports, __webpack_require__) { @@ -1615,7 +1624,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, function(module, exports, __webpack_require__) { "use strict"; exports.__esModule = !0; - var _defineProperty = __webpack_require__(142), _defineProperty2 = function(obj) { + var _defineProperty = __webpack_require__(143), _defineProperty2 = function(obj) { return obj && obj.__esModule ? obj : { default: obj }; @@ -1636,7 +1645,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, function(module, exports, __webpack_require__) { "use strict"; exports.__esModule = !0; - var _typeof2 = __webpack_require__(99), _typeof3 = function(obj) { + var _typeof2 = __webpack_require__(100), _typeof3 = function(obj) { return obj && obj.__esModule ? obj : { default: obj }; @@ -1653,7 +1662,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }; } exports.__esModule = !0; - var _setPrototypeOf = __webpack_require__(372), _setPrototypeOf2 = _interopRequireDefault(_setPrototypeOf), _create = __webpack_require__(376), _create2 = _interopRequireDefault(_create), _typeof2 = __webpack_require__(99), _typeof3 = _interopRequireDefault(_typeof2); + var _setPrototypeOf = __webpack_require__(367), _setPrototypeOf2 = _interopRequireDefault(_setPrototypeOf), _create = __webpack_require__(371), _create2 = _interopRequireDefault(_create), _typeof2 = __webpack_require__(100), _typeof3 = _interopRequireDefault(_typeof2); exports.default = function(subClass, superClass) { if ("function" != typeof superClass && null !== superClass) throw new TypeError("Super expression must either be null or a function, not " + (void 0 === superClass ? "undefined" : (0, _typeof3.default)(superClass))); @@ -1666,15 +1675,15 @@ var _bundleJs = []byte((((((((((`!function(modules) { } }), superClass && (_setPrototypeOf2.default ? (0, _setPrototypeOf2.default)(subClass, superClass) : subClass.__proto__ = superClass); }; -}, function(module, exports, __webpack_require__) { - var freeGlobal = __webpack_require__(248), freeSelf = "object" == typeof self && self && self.Object === Object && self, root = freeGlobal || freeSelf || Function("return this")(); - module.exports = root; }, function(module, exports) { function isObject(value) { var type = typeof value; return null != value && ("object" == type || "function" == type); } module.exports = isObject; +}, function(module, exports, __webpack_require__) { + var freeGlobal = __webpack_require__(242), freeSelf = "object" == typeof self && self && self.Object === Object && self, root = freeGlobal || freeSelf || Function("return this")(); + module.exports = root; }, function(module, exports, __webpack_require__) { "use strict"; function _interopRequireDefault(obj) { @@ -1685,7 +1694,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { Object.defineProperty(exports, "__esModule", { value: !0 }), exports.translateStyle = exports.AnimateGroup = exports.configBezier = exports.configSpring = void 0; - var _Animate = __webpack_require__(266), _Animate2 = _interopRequireDefault(_Animate), _easing = __webpack_require__(278), _util = __webpack_require__(122), _AnimateGroup = __webpack_require__(668), _AnimateGroup2 = _interopRequireDefault(_AnimateGroup); + var _Animate = __webpack_require__(263), _Animate2 = _interopRequireDefault(_Animate), _easing = __webpack_require__(275), _util = __webpack_require__(123), _AnimateGroup = __webpack_require__(679), _AnimateGroup2 = _interopRequireDefault(_AnimateGroup); exports.configSpring = _easing.configSpring, exports.configBezier = _easing.configBezier, exports.AnimateGroup = _AnimateGroup2.default, exports.translateStyle = _util.translateStyle, exports.default = _Animate2.default; @@ -1699,11 +1708,6 @@ var _bundleJs = []byte((((((((((`!function(modules) { module.exports = function(it) { return "object" == typeof it ? null !== it : "function" == typeof it; }; -}, function(module, exports, __webpack_require__) { - module.exports = { - default: __webpack_require__(383), - __esModule: !0 - }; }, function(module, exports) { function isObjectLike(value) { return null != value && "object" == typeof value; @@ -1711,32 +1715,32 @@ var _bundleJs = []byte((((((((((`!function(modules) { module.exports = isObjectLike; }, function(module, __webpack_exports__, __webpack_require__) { "use strict"; - var __WEBPACK_IMPORTED_MODULE_0__src_bisect__ = __webpack_require__(292); + var __WEBPACK_IMPORTED_MODULE_0__src_bisect__ = __webpack_require__(288); __webpack_require__.d(__webpack_exports__, "b", function() { return __WEBPACK_IMPORTED_MODULE_0__src_bisect__.a; }); - var __WEBPACK_IMPORTED_MODULE_1__src_ascending__ = __webpack_require__(63); + var __WEBPACK_IMPORTED_MODULE_1__src_ascending__ = __webpack_require__(64); __webpack_require__.d(__webpack_exports__, "a", function() { return __WEBPACK_IMPORTED_MODULE_1__src_ascending__.a; }); - var __WEBPACK_IMPORTED_MODULE_2__src_bisector__ = __webpack_require__(293); + var __WEBPACK_IMPORTED_MODULE_2__src_bisector__ = __webpack_require__(289); __webpack_require__.d(__webpack_exports__, "c", function() { return __WEBPACK_IMPORTED_MODULE_2__src_bisector__.a; }); - var __WEBPACK_IMPORTED_MODULE_18__src_quantile__ = (__webpack_require__(697), __webpack_require__(698), - __webpack_require__(295), __webpack_require__(297), __webpack_require__(699), __webpack_require__(702), - __webpack_require__(703), __webpack_require__(301), __webpack_require__(704), __webpack_require__(705), - __webpack_require__(706), __webpack_require__(707), __webpack_require__(302), __webpack_require__(294), - __webpack_require__(708), __webpack_require__(186)); + var __WEBPACK_IMPORTED_MODULE_18__src_quantile__ = (__webpack_require__(707), __webpack_require__(708), + __webpack_require__(291), __webpack_require__(293), __webpack_require__(709), __webpack_require__(712), + __webpack_require__(713), __webpack_require__(297), __webpack_require__(714), __webpack_require__(715), + __webpack_require__(716), __webpack_require__(717), __webpack_require__(298), __webpack_require__(290), + __webpack_require__(718), __webpack_require__(185)); __webpack_require__.d(__webpack_exports__, "d", function() { return __WEBPACK_IMPORTED_MODULE_18__src_quantile__.a; }); - var __WEBPACK_IMPORTED_MODULE_19__src_range__ = __webpack_require__(299); + var __WEBPACK_IMPORTED_MODULE_19__src_range__ = __webpack_require__(295); __webpack_require__.d(__webpack_exports__, "e", function() { return __WEBPACK_IMPORTED_MODULE_19__src_range__.a; }); - var __WEBPACK_IMPORTED_MODULE_23__src_ticks__ = (__webpack_require__(709), __webpack_require__(710), - __webpack_require__(711), __webpack_require__(300)); + var __WEBPACK_IMPORTED_MODULE_23__src_ticks__ = (__webpack_require__(719), __webpack_require__(720), + __webpack_require__(721), __webpack_require__(296)); __webpack_require__.d(__webpack_exports__, "h", function() { return __WEBPACK_IMPORTED_MODULE_23__src_ticks__.a; }), __webpack_require__.d(__webpack_exports__, "f", function() { @@ -1744,7 +1748,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }), __webpack_require__.d(__webpack_exports__, "g", function() { return __WEBPACK_IMPORTED_MODULE_23__src_ticks__.c; }); - __webpack_require__(303), __webpack_require__(296), __webpack_require__(712); + __webpack_require__(299), __webpack_require__(292), __webpack_require__(722); }, function(module, __webpack_exports__, __webpack_require__) { "use strict"; __webpack_require__.d(__webpack_exports__, "d", function() { @@ -1775,17 +1779,22 @@ var _bundleJs = []byte((((((((((`!function(modules) { return arg; }, module.exports = emptyFunction; }, function(module, exports, __webpack_require__) { - var dP = __webpack_require__(22), createDesc = __webpack_require__(70); + var dP = __webpack_require__(22), createDesc = __webpack_require__(71); module.exports = __webpack_require__(25) ? function(object, key, value) { return dP.f(object, key, createDesc(1, value)); } : function(object, key, value) { return object[key] = value, object; }; +}, function(module, exports, __webpack_require__) { + module.exports = { + default: __webpack_require__(378), + __esModule: !0 + }; }, function(module, exports, __webpack_require__) { function baseGetTag(value) { return null == value ? void 0 === value ? undefinedTag : nullTag : symToStringTag && symToStringTag in Object(value) ? getRawTag(value) : objectToString(value); } - var Symbol = __webpack_require__(77), getRawTag = __webpack_require__(543), objectToString = __webpack_require__(544), nullTag = "[object Null]", undefinedTag = "[object Undefined]", symToStringTag = Symbol ? Symbol.toStringTag : void 0; + var Symbol = __webpack_require__(77), getRawTag = __webpack_require__(520), objectToString = __webpack_require__(521), nullTag = "[object Null]", undefinedTag = "[object Undefined]", symToStringTag = Symbol ? Symbol.toStringTag : void 0; module.exports = baseGetTag; }, function(module, __webpack_exports__, __webpack_require__) { "use strict"; @@ -1811,7 +1820,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { className: __WEBPACK_IMPORTED_MODULE_5_classnames___default()("recharts-label", className) }, attrs, positionAttrs), label); } - var __WEBPACK_IMPORTED_MODULE_0_lodash_isObject__ = __webpack_require__(32), __WEBPACK_IMPORTED_MODULE_0_lodash_isObject___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_isObject__), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction__ = __webpack_require__(8), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_lodash_isFunction__), __WEBPACK_IMPORTED_MODULE_2_lodash_isNil__ = __webpack_require__(20), __WEBPACK_IMPORTED_MODULE_2_lodash_isNil___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_lodash_isNil__), __WEBPACK_IMPORTED_MODULE_3_react__ = __webpack_require__(0), __WEBPACK_IMPORTED_MODULE_3_react___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_react__), __WEBPACK_IMPORTED_MODULE_4_prop_types__ = __webpack_require__(1), __WEBPACK_IMPORTED_MODULE_4_prop_types___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_prop_types__), __WEBPACK_IMPORTED_MODULE_5_classnames__ = __webpack_require__(3), __WEBPACK_IMPORTED_MODULE_5_classnames___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_5_classnames__), __WEBPACK_IMPORTED_MODULE_6__Text__ = __webpack_require__(55), __WEBPACK_IMPORTED_MODULE_7__util_ReactUtils__ = __webpack_require__(4), __WEBPACK_IMPORTED_MODULE_8__util_DataUtils__ = __webpack_require__(9), __WEBPACK_IMPORTED_MODULE_9__util_PolarUtils__ = __webpack_require__(23), _extends = Object.assign || function(target) { + var __WEBPACK_IMPORTED_MODULE_0_lodash_isObject__ = __webpack_require__(31), __WEBPACK_IMPORTED_MODULE_0_lodash_isObject___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_isObject__), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction__ = __webpack_require__(8), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_lodash_isFunction__), __WEBPACK_IMPORTED_MODULE_2_lodash_isNil__ = __webpack_require__(20), __WEBPACK_IMPORTED_MODULE_2_lodash_isNil___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_lodash_isNil__), __WEBPACK_IMPORTED_MODULE_3_react__ = __webpack_require__(0), __WEBPACK_IMPORTED_MODULE_3_react___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_react__), __WEBPACK_IMPORTED_MODULE_4_prop_types__ = __webpack_require__(1), __WEBPACK_IMPORTED_MODULE_4_prop_types___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_prop_types__), __WEBPACK_IMPORTED_MODULE_5_classnames__ = __webpack_require__(3), __WEBPACK_IMPORTED_MODULE_5_classnames___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_5_classnames__), __WEBPACK_IMPORTED_MODULE_6__Text__ = __webpack_require__(55), __WEBPACK_IMPORTED_MODULE_7__util_ReactUtils__ = __webpack_require__(4), __WEBPACK_IMPORTED_MODULE_8__util_DataUtils__ = __webpack_require__(9), __WEBPACK_IMPORTED_MODULE_9__util_PolarUtils__ = __webpack_require__(23), _extends = Object.assign || function(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) Object.prototype.hasOwnProperty.call(source, key) && (target[key] = source[key]); @@ -2023,7 +2032,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { __webpack_exports__.a = Label; }, function(module, __webpack_exports__, __webpack_require__) { "use strict"; - var __WEBPACK_IMPORTED_MODULE_0__src_color__ = __webpack_require__(189); + var __WEBPACK_IMPORTED_MODULE_0__src_color__ = __webpack_require__(188); __webpack_require__.d(__webpack_exports__, "a", function() { return __WEBPACK_IMPORTED_MODULE_0__src_color__.e; }), __webpack_require__.d(__webpack_exports__, "f", function() { @@ -2031,13 +2040,13 @@ var _bundleJs = []byte((((((((((`!function(modules) { }), __webpack_require__.d(__webpack_exports__, "d", function() { return __WEBPACK_IMPORTED_MODULE_0__src_color__.f; }); - var __WEBPACK_IMPORTED_MODULE_1__src_lab__ = __webpack_require__(720); + var __WEBPACK_IMPORTED_MODULE_1__src_lab__ = __webpack_require__(730); __webpack_require__.d(__webpack_exports__, "e", function() { return __WEBPACK_IMPORTED_MODULE_1__src_lab__.a; }), __webpack_require__.d(__webpack_exports__, "c", function() { return __WEBPACK_IMPORTED_MODULE_1__src_lab__.b; }); - var __WEBPACK_IMPORTED_MODULE_2__src_cubehelix__ = __webpack_require__(721); + var __WEBPACK_IMPORTED_MODULE_2__src_cubehelix__ = __webpack_require__(731); __webpack_require__.d(__webpack_exports__, "b", function() { return __WEBPACK_IMPORTED_MODULE_2__src_cubehelix__.a; }); @@ -2073,7 +2082,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { })); })) : null; } - var __WEBPACK_IMPORTED_MODULE_0_lodash_isObject__ = __webpack_require__(32), __WEBPACK_IMPORTED_MODULE_0_lodash_isObject___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_isObject__), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction__ = __webpack_require__(8), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_lodash_isFunction__), __WEBPACK_IMPORTED_MODULE_2_lodash_isNil__ = __webpack_require__(20), __WEBPACK_IMPORTED_MODULE_2_lodash_isNil___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_lodash_isNil__), __WEBPACK_IMPORTED_MODULE_3_lodash_last__ = __webpack_require__(771), __WEBPACK_IMPORTED_MODULE_3_lodash_last___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_lodash_last__), __WEBPACK_IMPORTED_MODULE_4_lodash_isArray__ = __webpack_require__(12), __WEBPACK_IMPORTED_MODULE_4_lodash_isArray___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_lodash_isArray__), __WEBPACK_IMPORTED_MODULE_5_react__ = __webpack_require__(0), __WEBPACK_IMPORTED_MODULE_5_react___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_5_react__), __WEBPACK_IMPORTED_MODULE_6_prop_types__ = __webpack_require__(1), __WEBPACK_IMPORTED_MODULE_6_prop_types___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_6_prop_types__), __WEBPACK_IMPORTED_MODULE_7__Label__ = __webpack_require__(43), __WEBPACK_IMPORTED_MODULE_8__container_Layer__ = __webpack_require__(14), __WEBPACK_IMPORTED_MODULE_9__util_ReactUtils__ = __webpack_require__(4), __WEBPACK_IMPORTED_MODULE_10__util_ChartUtils__ = __webpack_require__(16), _extends = Object.assign || function(target) { + var __WEBPACK_IMPORTED_MODULE_0_lodash_isObject__ = __webpack_require__(31), __WEBPACK_IMPORTED_MODULE_0_lodash_isObject___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_isObject__), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction__ = __webpack_require__(8), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_lodash_isFunction__), __WEBPACK_IMPORTED_MODULE_2_lodash_isNil__ = __webpack_require__(20), __WEBPACK_IMPORTED_MODULE_2_lodash_isNil___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_lodash_isNil__), __WEBPACK_IMPORTED_MODULE_3_lodash_last__ = __webpack_require__(781), __WEBPACK_IMPORTED_MODULE_3_lodash_last___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_lodash_last__), __WEBPACK_IMPORTED_MODULE_4_lodash_isArray__ = __webpack_require__(12), __WEBPACK_IMPORTED_MODULE_4_lodash_isArray___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_lodash_isArray__), __WEBPACK_IMPORTED_MODULE_5_react__ = __webpack_require__(0), __WEBPACK_IMPORTED_MODULE_5_react___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_5_react__), __WEBPACK_IMPORTED_MODULE_6_prop_types__ = __webpack_require__(1), __WEBPACK_IMPORTED_MODULE_6_prop_types___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_6_prop_types__), __WEBPACK_IMPORTED_MODULE_7__Label__ = __webpack_require__(43), __WEBPACK_IMPORTED_MODULE_8__container_Layer__ = __webpack_require__(14), __WEBPACK_IMPORTED_MODULE_9__util_ReactUtils__ = __webpack_require__(4), __WEBPACK_IMPORTED_MODULE_10__util_ChartUtils__ = __webpack_require__(16), _extends = Object.assign || function(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) Object.prototype.hasOwnProperty.call(source, key) && (target[key] = source[key]); @@ -2157,7 +2166,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { } }), superClass && (Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass); } - var __WEBPACK_IMPORTED_MODULE_0_lodash_sortBy__ = __webpack_require__(285), __WEBPACK_IMPORTED_MODULE_0_lodash_sortBy___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_sortBy__), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction__ = __webpack_require__(8), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_lodash_isFunction__), __WEBPACK_IMPORTED_MODULE_2_lodash_get__ = __webpack_require__(109), __WEBPACK_IMPORTED_MODULE_2_lodash_get___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_lodash_get__), __WEBPACK_IMPORTED_MODULE_3_lodash_range__ = __webpack_require__(333), __WEBPACK_IMPORTED_MODULE_3_lodash_range___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_lodash_range__), __WEBPACK_IMPORTED_MODULE_4_lodash_throttle__ = __webpack_require__(779), __WEBPACK_IMPORTED_MODULE_4_lodash_throttle___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_lodash_throttle__), __WEBPACK_IMPORTED_MODULE_5_lodash_isNil__ = __webpack_require__(20), __WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_5_lodash_isNil__), __WEBPACK_IMPORTED_MODULE_6_react__ = __webpack_require__(0), __WEBPACK_IMPORTED_MODULE_6_react___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_6_react__), __WEBPACK_IMPORTED_MODULE_7_prop_types__ = __webpack_require__(1), __WEBPACK_IMPORTED_MODULE_7_prop_types___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_7_prop_types__), __WEBPACK_IMPORTED_MODULE_8_classnames__ = __webpack_require__(3), __WEBPACK_IMPORTED_MODULE_8_classnames___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_8_classnames__), __WEBPACK_IMPORTED_MODULE_9__container_Surface__ = __webpack_require__(76), __WEBPACK_IMPORTED_MODULE_10__container_Layer__ = __webpack_require__(14), __WEBPACK_IMPORTED_MODULE_11__component_Tooltip__ = __webpack_require__(121), __WEBPACK_IMPORTED_MODULE_12__component_Legend__ = __webpack_require__(171), __WEBPACK_IMPORTED_MODULE_13__shape_Curve__ = __webpack_require__(65), __WEBPACK_IMPORTED_MODULE_14__shape_Cross__ = __webpack_require__(327), __WEBPACK_IMPORTED_MODULE_15__shape_Sector__ = __webpack_require__(127), __WEBPACK_IMPORTED_MODULE_16__shape_Dot__ = __webpack_require__(57), __WEBPACK_IMPORTED_MODULE_17__shape_Rectangle__ = __webpack_require__(64), __WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__ = __webpack_require__(4), __WEBPACK_IMPORTED_MODULE_19__cartesian_CartesianAxis__ = __webpack_require__(334), __WEBPACK_IMPORTED_MODULE_20__cartesian_Brush__ = __webpack_require__(332), __WEBPACK_IMPORTED_MODULE_21__util_DOMUtils__ = __webpack_require__(185), __WEBPACK_IMPORTED_MODULE_22__util_DataUtils__ = __webpack_require__(9), __WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__ = __webpack_require__(16), __WEBPACK_IMPORTED_MODULE_24__util_PolarUtils__ = __webpack_require__(23), __WEBPACK_IMPORTED_MODULE_25__util_PureRender__ = __webpack_require__(5), __WEBPACK_IMPORTED_MODULE_26__util_Events__ = __webpack_require__(780), _extends = Object.assign || function(target) { + var __WEBPACK_IMPORTED_MODULE_0_lodash_sortBy__ = __webpack_require__(281), __WEBPACK_IMPORTED_MODULE_0_lodash_sortBy___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_sortBy__), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction__ = __webpack_require__(8), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_lodash_isFunction__), __WEBPACK_IMPORTED_MODULE_2_lodash_range__ = __webpack_require__(329), __WEBPACK_IMPORTED_MODULE_2_lodash_range___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_lodash_range__), __WEBPACK_IMPORTED_MODULE_3_lodash_throttle__ = __webpack_require__(790), __WEBPACK_IMPORTED_MODULE_3_lodash_throttle___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_lodash_throttle__), __WEBPACK_IMPORTED_MODULE_4_lodash_isNil__ = __webpack_require__(20), __WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_lodash_isNil__), __WEBPACK_IMPORTED_MODULE_5_react__ = __webpack_require__(0), __WEBPACK_IMPORTED_MODULE_5_react___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_5_react__), __WEBPACK_IMPORTED_MODULE_6_prop_types__ = __webpack_require__(1), __WEBPACK_IMPORTED_MODULE_6_prop_types___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_6_prop_types__), __WEBPACK_IMPORTED_MODULE_7_classnames__ = __webpack_require__(3), __WEBPACK_IMPORTED_MODULE_7_classnames___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_7_classnames__), __WEBPACK_IMPORTED_MODULE_8__container_Surface__ = __webpack_require__(78), __WEBPACK_IMPORTED_MODULE_9__container_Layer__ = __webpack_require__(14), __WEBPACK_IMPORTED_MODULE_10__component_Tooltip__ = __webpack_require__(122), __WEBPACK_IMPORTED_MODULE_11__component_Legend__ = __webpack_require__(171), __WEBPACK_IMPORTED_MODULE_12__shape_Curve__ = __webpack_require__(66), __WEBPACK_IMPORTED_MODULE_13__shape_Cross__ = __webpack_require__(323), __WEBPACK_IMPORTED_MODULE_14__shape_Sector__ = __webpack_require__(128), __WEBPACK_IMPORTED_MODULE_15__shape_Dot__ = __webpack_require__(57), __WEBPACK_IMPORTED_MODULE_16__shape_Rectangle__ = __webpack_require__(65), __WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__ = __webpack_require__(4), __WEBPACK_IMPORTED_MODULE_18__cartesian_CartesianAxis__ = __webpack_require__(330), __WEBPACK_IMPORTED_MODULE_19__cartesian_Brush__ = __webpack_require__(328), __WEBPACK_IMPORTED_MODULE_20__util_DOMUtils__ = __webpack_require__(184), __WEBPACK_IMPORTED_MODULE_21__util_DataUtils__ = __webpack_require__(9), __WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__ = __webpack_require__(16), __WEBPACK_IMPORTED_MODULE_23__util_PolarUtils__ = __webpack_require__(23), __WEBPACK_IMPORTED_MODULE_24__util_PureRender__ = __webpack_require__(5), __WEBPACK_IMPORTED_MODULE_25__util_Events__ = __webpack_require__(791), _extends = Object.assign || function(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) Object.prototype.hasOwnProperty.call(source, key) && (target[key] = source[key]); @@ -2195,22 +2204,22 @@ var _bundleJs = []byte((((((((((`!function(modules) { props: props }, defaultState, { updateId: 0 - }))), _this.uniqueChartId = __WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(props.id) ? Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.j)("recharts") : props.id, - props.throttleDelay && (_this.triggeredAfterMouseMove = __WEBPACK_IMPORTED_MODULE_4_lodash_throttle___default()(_this.triggeredAfterMouseMove, props.throttleDelay)), + }))), _this.uniqueChartId = __WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(props.id) ? Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.j)("recharts") : props.id, + props.throttleDelay && (_this.triggeredAfterMouseMove = __WEBPACK_IMPORTED_MODULE_3_lodash_throttle___default()(_this.triggeredAfterMouseMove, props.throttleDelay)), _this; } return _inherits(CategoricalChartWrapper, _Component), _createClass(CategoricalChartWrapper, [ { key: "componentDidMount", value: function() { - __WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(this.props.syncId) || this.addListener(); + __WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(this.props.syncId) || this.addListener(); } }, { key: "componentWillReceiveProps", value: function(nextProps) { var _props = this.props, data = _props.data, children = _props.children, width = _props.width, height = _props.height, layout = _props.layout, stackOffset = _props.stackOffset, margin = _props.margin, updateId = this.state.updateId; - if (nextProps.data === data && nextProps.width === width && nextProps.height === height && nextProps.layout === layout && nextProps.stackOffset === stackOffset && Object(__WEBPACK_IMPORTED_MODULE_25__util_PureRender__.b)(nextProps.margin, margin)) { - if (!Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.m)(nextProps.children, children)) { - var hasGlobalData = !__WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(nextProps.data), newUpdateId = hasGlobalData ? updateId : updateId + 1, _state = this.state, dataStartIndex = _state.dataStartIndex, dataEndIndex = _state.dataEndIndex, _defaultState = _extends({}, this.constructor.createDefaultState(nextProps), { + if (nextProps.data === data && nextProps.width === width && nextProps.height === height && nextProps.layout === layout && nextProps.stackOffset === stackOffset && Object(__WEBPACK_IMPORTED_MODULE_24__util_PureRender__.b)(nextProps.margin, margin)) { + if (!Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.m)(nextProps.children, children)) { + var hasGlobalData = !__WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(nextProps.data), newUpdateId = hasGlobalData ? updateId : updateId + 1, _state = this.state, dataStartIndex = _state.dataStartIndex, dataEndIndex = _state.dataEndIndex, _defaultState = _extends({}, this.constructor.createDefaultState(nextProps), { dataEndIndex: dataEndIndex, dataStartIndex: dataStartIndex }); @@ -2232,19 +2241,19 @@ var _bundleJs = []byte((((((((((`!function(modules) { updateId: updateId + 1 })))); } - __WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(this.props.syncId) && !__WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(nextProps.syncId) && this.addListener(), - !__WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(this.props.syncId) && __WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(nextProps.syncId) && this.removeListener(); + __WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(this.props.syncId) && !__WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(nextProps.syncId) && this.addListener(), + !__WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(this.props.syncId) && __WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(nextProps.syncId) && this.removeListener(); } }, { key: "componentWillUnmount", value: function() { - __WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(this.props.syncId) || this.removeListener(), + __WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(this.props.syncId) || this.removeListener(), "function" == typeof this.triggeredAfterMouseMove.cancel && this.triggeredAfterMouseMove.cancel(); } }, { key: "getAxisMap", value: function(props, _ref2) { - var _ref2$axisType = _ref2.axisType, axisType = void 0 === _ref2$axisType ? "xAxis" : _ref2$axisType, AxisComp = _ref2.AxisComp, graphicalItems = _ref2.graphicalItems, stackGroups = _ref2.stackGroups, dataStartIndex = _ref2.dataStartIndex, dataEndIndex = _ref2.dataEndIndex, children = props.children, axisIdKey = axisType + "Id", axes = Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.h)(children, AxisComp), axisMap = {}; + var _ref2$axisType = _ref2.axisType, axisType = void 0 === _ref2$axisType ? "xAxis" : _ref2$axisType, AxisComp = _ref2.AxisComp, graphicalItems = _ref2.graphicalItems, stackGroups = _ref2.stackGroups, dataStartIndex = _ref2.dataStartIndex, dataEndIndex = _ref2.dataEndIndex, children = props.children, axisIdKey = axisType + "Id", axes = Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.h)(children, AxisComp), axisMap = {}; return axes && axes.length ? axisMap = this.getAxisMapByAxes(props, { axes: axes, graphicalItems: graphicalItems, @@ -2266,7 +2275,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, { key: "getAxisMapByAxes", value: function(props, _ref3) { - var _this2 = this, axes = _ref3.axes, graphicalItems = _ref3.graphicalItems, axisType = _ref3.axisType, axisIdKey = _ref3.axisIdKey, stackGroups = _ref3.stackGroups, dataStartIndex = _ref3.dataStartIndex, dataEndIndex = _ref3.dataEndIndex, layout = props.layout, children = props.children, stackOffset = props.stackOffset, isCategorial = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.x)(layout, axisType); + var _this2 = this, axes = _ref3.axes, graphicalItems = _ref3.graphicalItems, axisType = _ref3.axisType, axisIdKey = _ref3.axisIdKey, stackGroups = _ref3.stackGroups, dataStartIndex = _ref3.dataStartIndex, dataEndIndex = _ref3.dataEndIndex, layout = props.layout, children = props.children, stackOffset = props.stackOffset, isCategorial = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.x)(layout, axisType); return axes.reduce(function(result, child) { var _child$props = child.props, type = _child$props.type, dataKey = _child$props.dataKey, allowDataOverflow = _child$props.allowDataOverflow, allowDuplicatedCategory = _child$props.allowDuplicatedCategory, scale = _child$props.scale, ticks = _child$props.ticks, axisId = child.props[axisIdKey], displayedData = _this2.constructor.getDisplayedData(props, { graphicalItems: graphicalItems.filter(function(item) { @@ -2278,28 +2287,28 @@ var _bundleJs = []byte((((((((((`!function(modules) { if (!result[axisId]) { var domain = void 0, duplicateDomain = void 0, categoricalDomain = void 0; if (dataKey) { - if (domain = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.n)(displayedData, dataKey, type), + if (domain = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.n)(displayedData, dataKey, type), "category" === type && isCategorial) { - var duplicate = Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.d)(domain); - allowDuplicatedCategory && duplicate ? (duplicateDomain = domain, domain = __WEBPACK_IMPORTED_MODULE_3_lodash_range___default()(0, len)) : allowDuplicatedCategory || (domain = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.y)(child.props.domain, domain, child).reduce(function(finalDomain, entry) { + var duplicate = Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.d)(domain); + allowDuplicatedCategory && duplicate ? (duplicateDomain = domain, domain = __WEBPACK_IMPORTED_MODULE_2_lodash_range___default()(0, len)) : allowDuplicatedCategory || (domain = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.y)(child.props.domain, domain, child).reduce(function(finalDomain, entry) { return finalDomain.indexOf(entry) >= 0 ? finalDomain : [].concat(_toConsumableArray(finalDomain), [ entry ]); }, [])); } else if ("category" === type) domain = allowDuplicatedCategory ? domain.filter(function(entry) { - return "" !== entry && !__WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(entry); - }) : Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.y)(child.props.domain, domain, child).reduce(function(finalDomain, entry) { - return finalDomain.indexOf(entry) >= 0 || "" === entry || __WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(entry) ? finalDomain : [].concat(_toConsumableArray(finalDomain), [ entry ]); + return "" !== entry && !__WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(entry); + }) : Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.y)(child.props.domain, domain, child).reduce(function(finalDomain, entry) { + return finalDomain.indexOf(entry) >= 0 || "" === entry || __WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(entry) ? finalDomain : [].concat(_toConsumableArray(finalDomain), [ entry ]); }, []); else if ("number" === type) { - var errorBarsDomain = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.z)(displayedData, graphicalItems.filter(function(item) { + var errorBarsDomain = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.z)(displayedData, graphicalItems.filter(function(item) { return item.props[axisIdKey] === axisId && !item.props.hide; }), dataKey, axisType); errorBarsDomain && (domain = errorBarsDomain); } - !isCategorial || "number" !== type && "auto" === scale || (categoricalDomain = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.n)(displayedData, dataKey, "category")); - } else domain = isCategorial ? __WEBPACK_IMPORTED_MODULE_3_lodash_range___default()(0, len) : stackGroups && stackGroups[axisId] && stackGroups[axisId].hasStack && "number" === type ? "expand" === stackOffset ? [ 0, 1 ] : Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.p)(stackGroups[axisId].stackGroups, dataStartIndex, dataEndIndex) : Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.o)(displayedData, graphicalItems.filter(function(item) { + !isCategorial || "number" !== type && "auto" === scale || (categoricalDomain = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.n)(displayedData, dataKey, "category")); + } else domain = isCategorial ? __WEBPACK_IMPORTED_MODULE_2_lodash_range___default()(0, len) : stackGroups && stackGroups[axisId] && stackGroups[axisId].hasStack && "number" === type ? "expand" === stackOffset ? [ 0, 1 ] : Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.p)(stackGroups[axisId].stackGroups, dataStartIndex, dataEndIndex) : Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.o)(displayedData, graphicalItems.filter(function(item) { return item.props[axisIdKey] === axisId && !item.props.hide; }), type, !0); - return "number" === type && (domain = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.e)(children, domain, axisId, axisType, ticks), - child.props.domain && (domain = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.B)(child.props.domain, domain, allowDataOverflow))), + return "number" === type && (domain = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.e)(children, domain, axisId, axisType, ticks), + child.props.domain && (domain = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.B)(child.props.domain, domain, allowDataOverflow))), _extends({}, result, _defineProperty({}, axisId, _extends({}, child.props, { axisType: axisType, domain: domain, @@ -2320,16 +2329,16 @@ var _bundleJs = []byte((((((((((`!function(modules) { graphicalItems: graphicalItems, dataStartIndex: dataStartIndex, dataEndIndex: dataEndIndex - }), len = displayedData.length, isCategorial = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.x)(layout, axisType), index = -1; + }), len = displayedData.length, isCategorial = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.x)(layout, axisType), index = -1; return graphicalItems.reduce(function(result, child) { var axisId = child.props[axisIdKey]; if (!result[axisId]) { index++; var domain = void 0; - return isCategorial ? domain = __WEBPACK_IMPORTED_MODULE_3_lodash_range___default()(0, len) : stackGroups && stackGroups[axisId] && stackGroups[axisId].hasStack ? (domain = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.p)(stackGroups[axisId].stackGroups, dataStartIndex, dataEndIndex), - domain = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.e)(children, domain, axisId, axisType)) : (domain = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.B)(Axis.defaultProps.domain, Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.o)(displayedData, graphicalItems.filter(function(item) { + return isCategorial ? domain = __WEBPACK_IMPORTED_MODULE_2_lodash_range___default()(0, len) : stackGroups && stackGroups[axisId] && stackGroups[axisId].hasStack ? (domain = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.p)(stackGroups[axisId].stackGroups, dataStartIndex, dataEndIndex), + domain = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.e)(children, domain, axisId, axisType)) : (domain = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.B)(Axis.defaultProps.domain, Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.o)(displayedData, graphicalItems.filter(function(item) { return item.props[axisIdKey] === axisId && !item.props.hide; - }), "number"), Axis.defaultProps.allowDataOverflow), domain = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.e)(children, domain, axisId, axisType)), + }), "number"), Axis.defaultProps.allowDataOverflow), domain = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.e)(children, domain, axisId, axisType)), _extends({}, result, _defineProperty({}, axisId, _extends({ axisType: axisType }, Axis.defaultProps, { @@ -2347,9 +2356,9 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, { key: "getActiveCoordinate", value: function(tooltipTicks, activeIndex, rangeObj) { - var layout = this.props.layout, entry = __WEBPACK_IMPORTED_MODULE_2_lodash_get___default()(tooltipTicks.filter(function(tick) { + var layout = this.props.layout, entry = tooltipTicks.find(function(tick) { return tick && tick.index === activeIndex; - }), "[0]"); + }); if (entry) { if ("horizontal" === layout) return { x: entry.coordinate, @@ -2361,13 +2370,13 @@ var _bundleJs = []byte((((((((((`!function(modules) { }; if ("centric" === layout) { var _angle = entry.coordinate, _radius = rangeObj.radius; - return _extends({}, rangeObj, Object(__WEBPACK_IMPORTED_MODULE_24__util_PolarUtils__.e)(rangeObj.cx, rangeObj.cy, _radius, _angle), { + return _extends({}, rangeObj, Object(__WEBPACK_IMPORTED_MODULE_23__util_PolarUtils__.e)(rangeObj.cx, rangeObj.cy, _radius, _angle), { angle: _angle, radius: _radius }); } var radius = entry.coordinate, angle = rangeObj.angle; - return _extends({}, rangeObj, Object(__WEBPACK_IMPORTED_MODULE_24__util_PolarUtils__.e)(rangeObj.cx, rangeObj.cy, radius, angle), { + return _extends({}, rangeObj, Object(__WEBPACK_IMPORTED_MODULE_23__util_PolarUtils__.e)(rangeObj.cx, rangeObj.cy, radius, angle), { angle: angle, radius: radius }); @@ -2378,17 +2387,17 @@ var _bundleJs = []byte((((((((((`!function(modules) { key: "getMouseInfo", value: function(event) { if (!this.container) return null; - var containerOffset = Object(__WEBPACK_IMPORTED_MODULE_21__util_DOMUtils__.b)(this.container), e = Object(__WEBPACK_IMPORTED_MODULE_21__util_DOMUtils__.a)(event, containerOffset), rangeObj = this.inRange(e.chartX, e.chartY); + var containerOffset = Object(__WEBPACK_IMPORTED_MODULE_20__util_DOMUtils__.b)(this.container), e = Object(__WEBPACK_IMPORTED_MODULE_20__util_DOMUtils__.a)(event, containerOffset), rangeObj = this.inRange(e.chartX, e.chartY); if (!rangeObj) return null; var _state2 = this.state, xAxisMap = _state2.xAxisMap, yAxisMap = _state2.yAxisMap; if ("axis" !== eventType && xAxisMap && yAxisMap) { - var xScale = Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.b)(xAxisMap).scale, yScale = Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.b)(yAxisMap).scale, xValue = xScale && xScale.invert ? xScale.invert(e.chartX) : null, yValue = yScale && yScale.invert ? yScale.invert(e.chartY) : null; + var xScale = Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.b)(xAxisMap).scale, yScale = Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.b)(yAxisMap).scale, xValue = xScale && xScale.invert ? xScale.invert(e.chartX) : null, yValue = yScale && yScale.invert ? yScale.invert(e.chartY) : null; return _extends({}, e, { xValue: xValue, yValue: yValue }); } - var _state3 = this.state, ticks = _state3.orderedTooltipTicks, axis = _state3.tooltipAxis, tooltipTicks = _state3.tooltipTicks, pos = this.calculateTooltipPos(rangeObj), activeIndex = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.b)(pos, ticks, tooltipTicks, axis); + var _state3 = this.state, ticks = _state3.orderedTooltipTicks, axis = _state3.tooltipAxis, tooltipTicks = _state3.tooltipTicks, pos = this.calculateTooltipPos(rangeObj), activeIndex = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.b)(pos, ticks, tooltipTicks, axis); if (activeIndex >= 0 && tooltipTicks) { var activeLabel = tooltipTicks[activeIndex] && tooltipTicks[activeIndex].value, activePayload = this.getTooltipContent(activeIndex, activeLabel), activeCoordinate = this.getActiveCoordinate(ticks, activeIndex, rangeObj); return _extends({}, e, { @@ -2407,14 +2416,14 @@ var _bundleJs = []byte((((((((((`!function(modules) { return activeIndex < 0 || !graphicalItems || !graphicalItems.length || activeIndex >= displayedData.length ? null : graphicalItems.reduce(function(result, child) { if (child.props.hide) return result; var _child$props2 = child.props, dataKey = _child$props2.dataKey, name = _child$props2.name, unit = _child$props2.unit, formatter = _child$props2.formatter, data = _child$props2.data, payload = void 0; - return payload = tooltipAxis.dataKey && !tooltipAxis.allowDuplicatedCategory ? Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.a)(data || displayedData, tooltipAxis.dataKey, activeLabel) : displayedData[activeIndex], - payload ? [].concat(_toConsumableArray(result), [ _extends({}, Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.k)(child), { + return payload = tooltipAxis.dataKey && !tooltipAxis.allowDuplicatedCategory ? Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.a)(data || displayedData, tooltipAxis.dataKey, activeLabel) : displayedData[activeIndex], + payload ? [].concat(_toConsumableArray(result), [ _extends({}, Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.k)(child), { dataKey: dataKey, unit: unit, formatter: formatter, name: name || dataKey, - color: Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.r)(child), - value: Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.w)(payload, dataKey), + color: Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.r)(child), + value: Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.w)(payload, dataKey), payload: payload }) ]) : result; }, []); @@ -2422,7 +2431,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, { key: "getFormatItems", value: function(props, currentState) { - var _this3 = this, graphicalItems = currentState.graphicalItems, stackGroups = currentState.stackGroups, offset = currentState.offset, updateId = currentState.updateId, dataStartIndex = currentState.dataStartIndex, dataEndIndex = currentState.dataEndIndex, barSize = props.barSize, layout = props.layout, barGap = props.barGap, barCategoryGap = props.barCategoryGap, globalMaxBarSize = props.maxBarSize, _getAxisNameByLayout = this.getAxisNameByLayout(layout), numericAxisName = _getAxisNameByLayout.numericAxisName, cateAxisName = _getAxisNameByLayout.cateAxisName, hasBar = this.constructor.hasBar(graphicalItems), sizeList = hasBar && Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.i)({ + var _this3 = this, graphicalItems = currentState.graphicalItems, stackGroups = currentState.stackGroups, offset = currentState.offset, updateId = currentState.updateId, dataStartIndex = currentState.dataStartIndex, dataEndIndex = currentState.dataEndIndex, barSize = props.barSize, layout = props.layout, barGap = props.barGap, barCategoryGap = props.barCategoryGap, globalMaxBarSize = props.maxBarSize, _getAxisNameByLayout = this.getAxisNameByLayout(layout), numericAxisName = _getAxisNameByLayout.numericAxisName, cateAxisName = _getAxisNameByLayout.cateAxisName, hasBar = this.constructor.hasBar(graphicalItems), sizeList = hasBar && Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.i)({ barSize: barSize, stackGroups: stackGroups }), formatedItems = []; @@ -2433,9 +2442,9 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, item), _item$props = item.props, dataKey = _item$props.dataKey, childMaxBarSize = _item$props.maxBarSize, numericAxisId = item.props[numericAxisName + "Id"], cateAxisId = item.props[cateAxisName + "Id"], axisObj = axisComponents.reduce(function(result, entry) { var _extends4, axisMap = currentState[entry.axisType + "Map"], id = item.props[entry.axisType + "Id"], axis = axisMap && axisMap[id]; return _extends({}, result, (_extends4 = {}, _defineProperty(_extends4, entry.axisType, axis), - _defineProperty(_extends4, entry.axisType + "Ticks", Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.u)(axis)), + _defineProperty(_extends4, entry.axisType + "Ticks", Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.u)(axis)), _extends4)); - }, {}), cateAxis = axisObj[cateAxisName], cateTicks = axisObj[cateAxisName + "Ticks"], stackedData = stackGroups && stackGroups[numericAxisId] && stackGroups[numericAxisId].hasStack && Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.t)(item, stackGroups[numericAxisId].stackGroups), bandSize = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.g)(cateAxis, cateTicks), maxBarSize = __WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(childMaxBarSize) ? globalMaxBarSize : childMaxBarSize, barPosition = hasBar && Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.h)({ + }, {}), cateAxis = axisObj[cateAxisName], cateTicks = axisObj[cateAxisName + "Ticks"], stackedData = stackGroups && stackGroups[numericAxisId] && stackGroups[numericAxisId].hasStack && Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.t)(item, stackGroups[numericAxisId].stackGroups), bandSize = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.g)(cateAxis, cateTicks), maxBarSize = __WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(childMaxBarSize) ? globalMaxBarSize : childMaxBarSize, barPosition = hasBar && Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.h)({ barGap: barGap, barCategoryGap: barCategoryGap, bandSize: bandSize, @@ -2457,13 +2466,13 @@ var _bundleJs = []byte((((((((((`!function(modules) { layout: layout, dataStartIndex: dataStartIndex, dataEndIndex: dataEndIndex, - onItemMouseLeave: Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.d)(_this3.handleItemMouseLeave, null, item.props.onMouseLeave), - onItemMouseEnter: Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.d)(_this3.handleItemMouseEnter, null, item.props.onMouseEnter) + onItemMouseLeave: Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.d)(_this3.handleItemMouseLeave, null, item.props.onMouseLeave), + onItemMouseEnter: Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.d)(_this3.handleItemMouseEnter, null, item.props.onMouseEnter) })), (_extends5 = { key: item.key || "item-" + index }, _defineProperty(_extends5, numericAxisName, axisObj[numericAxisName]), _defineProperty(_extends5, cateAxisName, axisObj[cateAxisName]), _defineProperty(_extends5, "animationId", updateId), _extends5)), - childIndex: Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.o)(item, props.children), + childIndex: Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.o)(item, props.children), item: item }); } @@ -2488,9 +2497,9 @@ var _bundleJs = []byte((((((((((`!function(modules) { var layout = this.props.layout, _state6 = this.state, activeCoordinate = _state6.activeCoordinate, offset = _state6.offset, x1 = void 0, y1 = void 0, x2 = void 0, y2 = void 0; if ("horizontal" === layout) x1 = activeCoordinate.x, x2 = x1, y1 = offset.top, y2 = offset.top + offset.height; else if ("vertical" === layout) y1 = activeCoordinate.y, - y2 = y1, x1 = offset.left, x2 = offset.left + offset.width; else if (!__WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(activeCoordinate.cx) || !__WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(activeCoordinate.cy)) { + y2 = y1, x1 = offset.left, x2 = offset.left + offset.width; else if (!__WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(activeCoordinate.cx) || !__WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(activeCoordinate.cy)) { if ("centric" !== layout) { - var _cx = activeCoordinate.cx, _cy = activeCoordinate.cy, radius = activeCoordinate.radius, startAngle = activeCoordinate.startAngle, endAngle = activeCoordinate.endAngle, startPoint = Object(__WEBPACK_IMPORTED_MODULE_24__util_PolarUtils__.e)(_cx, _cy, radius, startAngle), endPoint = Object(__WEBPACK_IMPORTED_MODULE_24__util_PolarUtils__.e)(_cx, _cy, radius, endAngle); + var _cx = activeCoordinate.cx, _cy = activeCoordinate.cy, radius = activeCoordinate.radius, startAngle = activeCoordinate.startAngle, endAngle = activeCoordinate.endAngle, startPoint = Object(__WEBPACK_IMPORTED_MODULE_23__util_PolarUtils__.e)(_cx, _cy, radius, startAngle), endPoint = Object(__WEBPACK_IMPORTED_MODULE_23__util_PolarUtils__.e)(_cx, _cy, radius, endAngle); return { points: [ startPoint, endPoint ], cx: _cx, @@ -2500,7 +2509,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { endAngle: endAngle }; } - var cx = activeCoordinate.cx, cy = activeCoordinate.cy, innerRadius = activeCoordinate.innerRadius, outerRadius = activeCoordinate.outerRadius, angle = activeCoordinate.angle, innerPoint = Object(__WEBPACK_IMPORTED_MODULE_24__util_PolarUtils__.e)(cx, cy, innerRadius, angle), outerPoint = Object(__WEBPACK_IMPORTED_MODULE_24__util_PolarUtils__.e)(cx, cy, outerRadius, angle); + var cx = activeCoordinate.cx, cy = activeCoordinate.cy, innerRadius = activeCoordinate.innerRadius, outerRadius = activeCoordinate.outerRadius, angle = activeCoordinate.angle, innerPoint = Object(__WEBPACK_IMPORTED_MODULE_23__util_PolarUtils__.e)(cx, cy, innerRadius, angle), outerPoint = Object(__WEBPACK_IMPORTED_MODULE_23__util_PolarUtils__.e)(cx, cy, outerRadius, angle); x1 = innerPoint.x, y1 = innerPoint.y, x2 = outerPoint.x, y2 = outerPoint.y; } return [ { @@ -2547,8 +2556,8 @@ var _bundleJs = []byte((((((((((`!function(modules) { } var _state7 = this.state, angleAxisMap = _state7.angleAxisMap, radiusAxisMap = _state7.radiusAxisMap; if (angleAxisMap && radiusAxisMap) { - var angleAxis = Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.b)(angleAxisMap); - return Object(__WEBPACK_IMPORTED_MODULE_24__util_PolarUtils__.d)({ + var angleAxis = Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.b)(angleAxisMap); + return Object(__WEBPACK_IMPORTED_MODULE_23__util_PolarUtils__.d)({ x: x, y: y }, angleAxis); @@ -2558,22 +2567,22 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, { key: "parseEventsOfWrapper", value: function() { - var children = this.props.children, tooltipItem = Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.i)(children, __WEBPACK_IMPORTED_MODULE_11__component_Tooltip__.a), tooltipEvents = tooltipItem && "axis" === eventType ? { + var children = this.props.children, tooltipItem = Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.i)(children, __WEBPACK_IMPORTED_MODULE_10__component_Tooltip__.a), tooltipEvents = tooltipItem && "axis" === eventType ? { onMouseEnter: this.handleMouseEnter, onMouseMove: this.handleMouseMove, onMouseLeave: this.handleMouseLeave, onTouchMove: this.handleTouchMove - } : {}, outerEvents = Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.e)(this.props, this.handleOuterEvent); + } : {}, outerEvents = Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.e)(this.props, this.handleOuterEvent); return _extends({}, outerEvents, tooltipEvents); } }, { key: "updateStateOfAxisMapsOffsetAndStackGroups", value: function(_ref5) { var _this4 = this, props = _ref5.props, dataStartIndex = _ref5.dataStartIndex, dataEndIndex = _ref5.dataEndIndex, updateId = _ref5.updateId; - if (!Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.q)({ + if (!Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.q)({ props: props })) return null; - var children = props.children, layout = props.layout, stackOffset = props.stackOffset, data = props.data, reverseStackOrder = props.reverseStackOrder, _getAxisNameByLayout2 = this.getAxisNameByLayout(layout), numericAxisName = _getAxisNameByLayout2.numericAxisName, cateAxisName = _getAxisNameByLayout2.cateAxisName, graphicalItems = Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.h)(children, GraphicalChild), stackGroups = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.s)(data, graphicalItems, numericAxisName + "Id", cateAxisName + "Id", stackOffset, reverseStackOrder), axisObj = axisComponents.reduce(function(result, entry) { + var children = props.children, layout = props.layout, stackOffset = props.stackOffset, data = props.data, reverseStackOrder = props.reverseStackOrder, _getAxisNameByLayout2 = this.getAxisNameByLayout(layout), numericAxisName = _getAxisNameByLayout2.numericAxisName, cateAxisName = _getAxisNameByLayout2.cateAxisName, graphicalItems = Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.h)(children, GraphicalChild), stackGroups = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.s)(data, graphicalItems, numericAxisName + "Id", cateAxisName + "Id", stackOffset, reverseStackOrder), axisObj = axisComponents.reduce(function(result, entry) { var name = entry.axisType + "Map"; return _extends({}, result, _defineProperty({}, name, _this4.getAxisMap(props, _extends({}, entry, { graphicalItems: graphicalItems, @@ -2606,19 +2615,19 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, { key: "addListener", value: function() { - __WEBPACK_IMPORTED_MODULE_26__util_Events__.b.on(__WEBPACK_IMPORTED_MODULE_26__util_Events__.a, this.handleReceiveSyncEvent), - __WEBPACK_IMPORTED_MODULE_26__util_Events__.b.setMaxListeners && __WEBPACK_IMPORTED_MODULE_26__util_Events__.b._maxListeners && __WEBPACK_IMPORTED_MODULE_26__util_Events__.b.setMaxListeners(__WEBPACK_IMPORTED_MODULE_26__util_Events__.b._maxListeners + 1); + __WEBPACK_IMPORTED_MODULE_25__util_Events__.b.on(__WEBPACK_IMPORTED_MODULE_25__util_Events__.a, this.handleReceiveSyncEvent), + __WEBPACK_IMPORTED_MODULE_25__util_Events__.b.setMaxListeners && __WEBPACK_IMPORTED_MODULE_25__util_Events__.b._maxListeners && __WEBPACK_IMPORTED_MODULE_25__util_Events__.b.setMaxListeners(__WEBPACK_IMPORTED_MODULE_25__util_Events__.b._maxListeners + 1); } }, { key: "removeListener", value: function() { - __WEBPACK_IMPORTED_MODULE_26__util_Events__.b.removeListener(__WEBPACK_IMPORTED_MODULE_26__util_Events__.a, this.handleReceiveSyncEvent), - __WEBPACK_IMPORTED_MODULE_26__util_Events__.b.setMaxListeners && __WEBPACK_IMPORTED_MODULE_26__util_Events__.b._maxListeners && __WEBPACK_IMPORTED_MODULE_26__util_Events__.b.setMaxListeners(__WEBPACK_IMPORTED_MODULE_26__util_Events__.b._maxListeners - 1); + __WEBPACK_IMPORTED_MODULE_25__util_Events__.b.removeListener(__WEBPACK_IMPORTED_MODULE_25__util_Events__.a, this.handleReceiveSyncEvent), + __WEBPACK_IMPORTED_MODULE_25__util_Events__.b.setMaxListeners && __WEBPACK_IMPORTED_MODULE_25__util_Events__.b._maxListeners && __WEBPACK_IMPORTED_MODULE_25__util_Events__.b.setMaxListeners(__WEBPACK_IMPORTED_MODULE_25__util_Events__.b._maxListeners - 1); } }, { key: "calculateOffset", value: function(_ref6) { - var props = _ref6.props, graphicalItems = _ref6.graphicalItems, _ref6$xAxisMap = _ref6.xAxisMap, xAxisMap = void 0 === _ref6$xAxisMap ? {} : _ref6$xAxisMap, _ref6$yAxisMap = _ref6.yAxisMap, yAxisMap = void 0 === _ref6$yAxisMap ? {} : _ref6$yAxisMap, width = props.width, height = props.height, children = props.children, margin = props.margin || {}, brushItem = Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.i)(children, __WEBPACK_IMPORTED_MODULE_20__cartesian_Brush__.a), legendItem = Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.i)(children, __WEBPACK_IMPORTED_MODULE_12__component_Legend__.a), offsetH = Object.keys(yAxisMap).reduce(function(result, id) { + var props = _ref6.props, graphicalItems = _ref6.graphicalItems, _ref6$xAxisMap = _ref6.xAxisMap, xAxisMap = void 0 === _ref6$xAxisMap ? {} : _ref6$xAxisMap, _ref6$yAxisMap = _ref6.yAxisMap, yAxisMap = void 0 === _ref6$yAxisMap ? {} : _ref6$yAxisMap, width = props.width, height = props.height, children = props.children, margin = props.margin || {}, brushItem = Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.i)(children, __WEBPACK_IMPORTED_MODULE_19__cartesian_Brush__.a), legendItem = Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.i)(children, __WEBPACK_IMPORTED_MODULE_11__component_Legend__.a), offsetH = Object.keys(yAxisMap).reduce(function(result, id) { var entry = yAxisMap[id], orientation = entry.orientation; return entry.mirror || entry.hide ? result : _extends({}, result, _defineProperty({}, orientation, result[orientation] + entry.width)); }, { @@ -2631,10 +2640,10 @@ var _bundleJs = []byte((((((((((`!function(modules) { top: margin.top || 0, bottom: margin.bottom || 0 }), offset = _extends({}, offsetV, offsetH), brushBottom = offset.bottom; - if (brushItem && (offset.bottom += brushItem.props.height || __WEBPACK_IMPORTED_MODULE_20__cartesian_Brush__.a.defaultProps.height), + if (brushItem && (offset.bottom += brushItem.props.height || __WEBPACK_IMPORTED_MODULE_19__cartesian_Brush__.a.defaultProps.height), legendItem && this.legendInstance) { var legendBox = this.legendInstance.getBBox(); - offset = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.a)(offset, graphicalItems, props, legendBox); + offset = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.a)(offset, graphicalItems, props, legendBox); } return _extends({ brushBottom: brushBottom @@ -2647,14 +2656,14 @@ var _bundleJs = []byte((((((((((`!function(modules) { key: "triggerSyncEvent", value: function(data) { var syncId = this.props.syncId; - __WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(syncId) || __WEBPACK_IMPORTED_MODULE_26__util_Events__.b.emit(__WEBPACK_IMPORTED_MODULE_26__util_Events__.a, syncId, this.uniqueChartId, data); + __WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(syncId) || __WEBPACK_IMPORTED_MODULE_25__util_Events__.b.emit(__WEBPACK_IMPORTED_MODULE_25__util_Events__.a, syncId, this.uniqueChartId, data); } }, { key: "filterFormatItem", value: function(item, displayName, childIndex) { for (var formatedGraphicalItems = this.state.formatedGraphicalItems, i = 0, len = formatedGraphicalItems.length; i < len; i++) { var entry = formatedGraphicalItems[i]; - if (entry.item === item || entry.props.key === item.key || displayName === Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.j)(entry.item.type) && childIndex === entry.childIndex) return entry; + if (entry.item === item || entry.props.key === item.key || displayName === Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.j)(entry.item.type) && childIndex === entry.childIndex) return entry; } return null; } @@ -2662,7 +2671,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { key: "renderAxis", value: function(axisOptions, element, displayName, index) { var _props2 = this.props, width = _props2.width, height = _props2.height; - return __WEBPACK_IMPORTED_MODULE_6_react___default.a.createElement(__WEBPACK_IMPORTED_MODULE_19__cartesian_CartesianAxis__.a, _extends({}, axisOptions, { + return __WEBPACK_IMPORTED_MODULE_5_react___default.a.createElement(__WEBPACK_IMPORTED_MODULE_18__cartesian_CartesianAxis__.a, _extends({}, axisOptions, { className: "recharts-" + axisOptions.axisType + " " + axisOptions.axisType, key: element.key || displayName + "-" + index, viewBox: { @@ -2677,7 +2686,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, { key: "renderLegend", value: function() { - var _this5 = this, formatedGraphicalItems = this.state.formatedGraphicalItems, _props3 = this.props, children = _props3.children, width = _props3.width, height = _props3.height, margin = this.props.margin || {}, legendWidth = width - (margin.left || 0) - (margin.right || 0), legendHeight = height - (margin.top || 0) - (margin.bottom || 0), props = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.q)({ + var _this5 = this, formatedGraphicalItems = this.state.formatedGraphicalItems, _props3 = this.props, children = _props3.children, width = _props3.width, height = _props3.height, margin = this.props.margin || {}, legendWidth = width - (margin.left || 0) - (margin.right || 0), legendHeight = height - (margin.top || 0) - (margin.bottom || 0), props = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.q)({ children: children, formatedGraphicalItems: formatedGraphicalItems, legendWidth: legendWidth, @@ -2686,7 +2695,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }); if (!props) return null; var item = props.item, otherProps = _objectWithoutProperties(props, [ "item" ]); - return Object(__WEBPACK_IMPORTED_MODULE_6_react__.cloneElement)(item, _extends({}, otherProps, { + return Object(__WEBPACK_IMPORTED_MODULE_5_react__.cloneElement)(item, _extends({}, otherProps, { chartWidth: width, chartHeight: height, margin: margin, @@ -2699,10 +2708,10 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, { key: "renderTooltip", value: function() { - var children = this.props.children, tooltipItem = Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.i)(children, __WEBPACK_IMPORTED_MODULE_11__component_Tooltip__.a); + var children = this.props.children, tooltipItem = Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.i)(children, __WEBPACK_IMPORTED_MODULE_10__component_Tooltip__.a); if (!tooltipItem) return null; var _state8 = this.state, isTooltipActive = _state8.isTooltipActive, activeCoordinate = _state8.activeCoordinate, activePayload = _state8.activePayload, activeLabel = _state8.activeLabel, offset = _state8.offset; - return Object(__WEBPACK_IMPORTED_MODULE_6_react__.cloneElement)(tooltipItem, { + return Object(__WEBPACK_IMPORTED_MODULE_5_react__.cloneElement)(tooltipItem, { viewBox: _extends({}, offset, { x: offset.left, y: offset.top @@ -2717,8 +2726,8 @@ var _bundleJs = []byte((((((((((`!function(modules) { key: "renderActiveDot", value: function(option, props) { var dot = void 0; - return dot = Object(__WEBPACK_IMPORTED_MODULE_6_react__.isValidElement)(option) ? Object(__WEBPACK_IMPORTED_MODULE_6_react__.cloneElement)(option, props) : __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction___default()(option) ? option(props) : __WEBPACK_IMPORTED_MODULE_6_react___default.a.createElement(__WEBPACK_IMPORTED_MODULE_16__shape_Dot__.a, props), - __WEBPACK_IMPORTED_MODULE_6_react___default.a.createElement(__WEBPACK_IMPORTED_MODULE_10__container_Layer__.a, { + return dot = Object(__WEBPACK_IMPORTED_MODULE_5_react__.isValidElement)(option) ? Object(__WEBPACK_IMPORTED_MODULE_5_react__.cloneElement)(option, props) : __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction___default()(option) ? option(props) : __WEBPACK_IMPORTED_MODULE_5_react___default.a.createElement(__WEBPACK_IMPORTED_MODULE_15__shape_Dot__.a, props), + __WEBPACK_IMPORTED_MODULE_5_react___default.a.createElement(__WEBPACK_IMPORTED_MODULE_9__container_Layer__.a, { className: "recharts-active-dot", key: props.key }, dot); @@ -2732,13 +2741,13 @@ var _bundleJs = []byte((((((((((`!function(modules) { cx: activePoint.x, cy: activePoint.y, r: 4, - fill: Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.r)(item.item), + fill: Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.r)(item.item), strokeWidth: 2, stroke: "#fff", payload: activePoint.payload, value: activePoint.value, key: key + "-activePoint-" + childIndex - }, Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.k)(activeDot), Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.e)(activeDot)); + }, Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.k)(activeDot), Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.e)(activeDot)); return result.push(this.renderActiveDot(activeDot, dotProps, childIndex)), basePoint ? result.push(this.renderActiveDot(activeDot, _extends({}, dotProps, { cx: basePoint.x, cy: basePoint.y, @@ -2749,8 +2758,8 @@ var _bundleJs = []byte((((((((((`!function(modules) { key: "render", value: function() { var _this6 = this; - if (!Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.q)(this)) return null; - var _props4 = this.props, children = _props4.children, className = _props4.className, width = _props4.width, height = _props4.height, style = _props4.style, compact = _props4.compact, others = _objectWithoutProperties(_props4, [ "children", "className", "width", "height", "style", "compact" ]), attrs = Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.k)(others), map = { + if (!Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.q)(this)) return null; + var _props4 = this.props, children = _props4.children, className = _props4.className, width = _props4.width, height = _props4.height, style = _props4.style, compact = _props4.compact, others = _objectWithoutProperties(_props4, [ "children", "className", "width", "height", "style", "compact" ]), attrs = Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.k)(others), map = { CartesianGrid: { handler: this.renderGrid, once: !0 @@ -2810,13 +2819,13 @@ var _bundleJs = []byte((((((((((`!function(modules) { handler: this.renderPolarAxis } }; - if (compact) return __WEBPACK_IMPORTED_MODULE_6_react___default.a.createElement(__WEBPACK_IMPORTED_MODULE_9__container_Surface__.a, _extends({}, attrs, { + if (compact) return __WEBPACK_IMPORTED_MODULE_5_react___default.a.createElement(__WEBPACK_IMPORTED_MODULE_8__container_Surface__.a, _extends({}, attrs, { width: width, height: height - }), Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.p)(children, map)); + }), Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.p)(children, map)); var events = this.parseEventsOfWrapper(); - return __WEBPACK_IMPORTED_MODULE_6_react___default.a.createElement("div", _extends({ - className: __WEBPACK_IMPORTED_MODULE_8_classnames___default()("recharts-wrapper", className), + return __WEBPACK_IMPORTED_MODULE_5_react___default.a.createElement("div", _extends({ + className: __WEBPACK_IMPORTED_MODULE_7_classnames___default()("recharts-wrapper", className), style: _extends({}, style, { position: "relative", cursor: "default", @@ -2827,43 +2836,43 @@ var _bundleJs = []byte((((((((((`!function(modules) { ref: function(node) { _this6.container = node; } - }), __WEBPACK_IMPORTED_MODULE_6_react___default.a.createElement(__WEBPACK_IMPORTED_MODULE_9__container_Surface__.a, _extends({}, attrs, { + }), __WEBPACK_IMPORTED_MODULE_5_react___default.a.createElement(__WEBPACK_IMPORTED_MODULE_8__container_Surface__.a, _extends({}, attrs, { width: width, height: height - }), Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.p)(children, map)), this.renderLegend(), this.renderTooltip()); + }), Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.p)(children, map)), this.renderLegend(), this.renderTooltip()); } } ]), CategoricalChartWrapper; - }(__WEBPACK_IMPORTED_MODULE_6_react__.Component), _class.displayName = chartName, + }(__WEBPACK_IMPORTED_MODULE_5_react__.Component), _class.displayName = chartName, _class.propTypes = _extends({ - syncId: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.oneOfType([ __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.string, __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.number ]), - compact: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.bool, - width: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.number, - height: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.number, - data: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.arrayOf(__WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.object), - layout: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.oneOf([ "horizontal", "vertical" ]), - stackOffset: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.oneOf([ "sign", "expand", "none", "wiggle", "silhouette" ]), - throttleDelay: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.number, - margin: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.shape({ - top: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.number, - right: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.number, - bottom: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.number, - left: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.number + syncId: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.oneOfType([ __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.string, __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.number ]), + compact: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.bool, + width: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.number, + height: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.number, + data: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.arrayOf(__WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.object), + layout: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.oneOf([ "horizontal", "vertical" ]), + stackOffset: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.oneOf([ "sign", "expand", "none", "wiggle", "silhouette" ]), + throttleDelay: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.number, + margin: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.shape({ + top: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.number, + right: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.number, + bottom: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.number, + left: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.number }), - barCategoryGap: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.oneOfType([ __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.number, __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.string ]), - barGap: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.oneOfType([ __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.number, __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.string ]), - barSize: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.oneOfType([ __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.number, __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.string ]), - maxBarSize: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.number, - style: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.object, - className: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.string, - children: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.oneOfType([ __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.arrayOf(__WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.node), __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.node ]), - onClick: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.func, - onMouseLeave: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.func, - onMouseEnter: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.func, - onMouseMove: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.func, - onMouseDown: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.func, - onMouseUp: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.func, - reverseStackOrder: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.bool, - id: __WEBPACK_IMPORTED_MODULE_7_prop_types___default.a.string + barCategoryGap: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.oneOfType([ __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.number, __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.string ]), + barGap: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.oneOfType([ __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.number, __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.string ]), + barSize: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.oneOfType([ __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.number, __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.string ]), + maxBarSize: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.number, + style: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.object, + className: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.string, + children: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.oneOfType([ __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.arrayOf(__WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.node), __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.node ]), + onClick: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.func, + onMouseLeave: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.func, + onMouseEnter: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.func, + onMouseMove: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.func, + onMouseDown: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.func, + onMouseUp: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.func, + reverseStackOrder: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.bool, + id: __WEBPACK_IMPORTED_MODULE_6_prop_types___default.a.string }, propTypes), _class.defaultProps = _extends({ layout: "horizontal", stackOffset: "none", @@ -2877,7 +2886,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, reverseStackOrder: !1 }, defaultProps), _class.createDefaultState = function(props) { - var children = props.children, brushItem = Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.i)(children, __WEBPACK_IMPORTED_MODULE_20__cartesian_Brush__.a); + var children = props.children, brushItem = Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.i)(children, __WEBPACK_IMPORTED_MODULE_19__cartesian_Brush__.a); return { chartX: 0, chartY: 0, @@ -2888,7 +2897,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }; }, _class.hasBar = function(graphicalItems) { return !(!graphicalItems || !graphicalItems.length) && graphicalItems.some(function(item) { - var name = Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.j)(item && item.type); + var name = Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.j)(item && item.type); return name && name.indexOf("Bar") >= 0; }); }, _class.getDisplayedData = function(props, _ref8, item) { @@ -2899,7 +2908,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { if (itemsData && itemsData.length > 0) return itemsData; if (item && item.props && item.props.data && item.props.data.length > 0) return item.props.data; var data = props.data; - return data && data.length && Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.g)(dataStartIndex) && Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.g)(dataEndIndex) ? data.slice(dataStartIndex, dataEndIndex + 1) : []; + return data && data.length && Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.g)(dataStartIndex) && Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.g)(dataEndIndex) ? data.slice(dataStartIndex, dataEndIndex + 1) : []; }, _initialiseProps = function() { var _this7 = this; this.handleLegendBBoxUpdate = function(box) { @@ -2916,7 +2925,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { var _props5 = _this7.props, syncId = _props5.syncId, layout = _props5.layout, updateId = _this7.state.updateId; if (syncId === cId && chartId !== _this7.uniqueChartId) { var dataStartIndex = data.dataStartIndex, dataEndIndex = data.dataEndIndex; - if (__WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(data.dataStartIndex) && __WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(data.dataEndIndex)) if (__WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(data.activeTooltipIndex)) _this7.setState(data); else { + if (__WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(data.dataStartIndex) && __WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(data.dataEndIndex)) if (__WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(data.activeTooltipIndex)) _this7.setState(data); else { var chartX = data.chartX, chartY = data.chartY, activeTooltipIndex = data.activeTooltipIndex, _state10 = _this7.state, offset = _state10.offset, tooltipTicks = _state10.tooltipTicks; if (!offset) return; var viewBox = _extends({}, offset, { @@ -2945,15 +2954,17 @@ var _bundleJs = []byte((((((((((`!function(modules) { var startIndex = _ref9.startIndex, endIndex = _ref9.endIndex; if (startIndex !== _this7.state.dataStartIndex || endIndex !== _this7.state.dataEndIndex) { var updateId = _this7.state.updateId; - _this7.setState(_extends({ - dataStartIndex: startIndex, - dataEndIndex: endIndex - }, _this7.updateStateOfAxisMapsOffsetAndStackGroups({ - props: _this7.props, - dataStartIndex: startIndex, - dataEndIndex: endIndex, - updateId: updateId - }))), _this7.triggerSyncEvent({ + _this7.setState(function() { + return _extends({ + dataStartIndex: startIndex, + dataEndIndex: endIndex + }, _this7.updateStateOfAxisMapsOffsetAndStackGroups({ + props: _this7.props, + dataStartIndex: startIndex, + dataEndIndex: endIndex, + updateId: updateId + })); + }), _this7.triggerSyncEvent({ dataStartIndex: startIndex, dataEndIndex: endIndex }); @@ -2974,18 +2985,22 @@ var _bundleJs = []byte((((((((((`!function(modules) { }; _this7.setState(nextState), _this7.triggerSyncEvent(nextState), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction___default()(onMouseMove) && onMouseMove(nextState, e); }, this.handleItemMouseEnter = function(el) { - _this7.setState({ - isTooltipActive: !0, - activeItem: el, - activePayload: el.tooltipPayload, - activeCoordinate: el.tooltipPosition || { - x: el.cx, - y: el.cy - } + _this7.setState(function() { + return { + isTooltipActive: !0, + activeItem: el, + activePayload: el.tooltipPayload, + activeCoordinate: el.tooltipPosition || { + x: el.cx, + y: el.cy + } + }; }); }, this.handleItemMouseLeave = function() { - _this7.setState({ - isTooltipActive: !1 + _this7.setState(function() { + return { + isTooltipActive: !1 + }; }); }, this.handleMouseMove = function(e) { e && __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction___default()(e.persist) && e.persist(), @@ -2996,7 +3011,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }; _this7.setState(nextState), _this7.triggerSyncEvent(nextState), __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction___default()(onMouseLeave) && onMouseLeave(nextState, e); }, this.handleOuterEvent = function(e) { - var eventName = Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.l)(e); + var eventName = Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.l)(e); if (eventName && __WEBPACK_IMPORTED_MODULE_1_lodash_isFunction___default()(_this7.props[eventName])) { var mouse = _this7.getMouseInfo(e); (0, _this7.props[eventName])(mouse, e); @@ -3020,8 +3035,8 @@ var _bundleJs = []byte((((((((((`!function(modules) { null != e.changedTouches && e.changedTouches.length > 0 && _this7.handleMouseMove(e.changedTouches[0]); }, this.verticalCoordinatesGenerator = function(_ref10) { var xAxis = _ref10.xAxis, width = _ref10.width, height = _ref10.height, offset = _ref10.offset; - return Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.m)(__WEBPACK_IMPORTED_MODULE_19__cartesian_CartesianAxis__.a.getTicks(_extends({}, __WEBPACK_IMPORTED_MODULE_19__cartesian_CartesianAxis__.a.defaultProps, xAxis, { - ticks: Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.u)(xAxis, !0), + return Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.m)(__WEBPACK_IMPORTED_MODULE_18__cartesian_CartesianAxis__.a.getTicks(_extends({}, __WEBPACK_IMPORTED_MODULE_18__cartesian_CartesianAxis__.a.defaultProps, xAxis, { + ticks: Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.u)(xAxis, !0), viewBox: { x: 0, y: 0, @@ -3031,8 +3046,8 @@ var _bundleJs = []byte((((((((((`!function(modules) { })), offset.left, offset.left + offset.width); }, this.horizontalCoordinatesGenerator = function(_ref11) { var yAxis = _ref11.yAxis, width = _ref11.width, height = _ref11.height, offset = _ref11.offset; - return Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.m)(__WEBPACK_IMPORTED_MODULE_19__cartesian_CartesianAxis__.a.getTicks(_extends({}, __WEBPACK_IMPORTED_MODULE_19__cartesian_CartesianAxis__.a.defaultProps, yAxis, { - ticks: Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.u)(yAxis, !0), + return Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.m)(__WEBPACK_IMPORTED_MODULE_18__cartesian_CartesianAxis__.a.getTicks(_extends({}, __WEBPACK_IMPORTED_MODULE_18__cartesian_CartesianAxis__.a.defaultProps, yAxis, { + ticks: Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.u)(yAxis, !0), viewBox: { x: 0, y: 0, @@ -3041,23 +3056,23 @@ var _bundleJs = []byte((((((((((`!function(modules) { } })), offset.top, offset.top + offset.height); }, this.axesTicksGenerator = function(axis) { - return Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.u)(axis, !0); + return Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.u)(axis, !0); }, this.tooltipTicksGenerator = function(axisMap) { - var axis = Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.b)(axisMap), tooltipTicks = Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.u)(axis, !1, !0); + var axis = Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.b)(axisMap), tooltipTicks = Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.u)(axis, !1, !0); return { tooltipTicks: tooltipTicks, orderedTooltipTicks: __WEBPACK_IMPORTED_MODULE_0_lodash_sortBy___default()(tooltipTicks, function(o) { return o.coordinate; }), tooltipAxis: axis, - tooltipAxisBandSize: Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.g)(axis) + tooltipAxisBandSize: Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.g)(axis) }; }, this.renderCursor = function(element) { var _state11 = _this7.state, isTooltipActive = _state11.isTooltipActive, activeCoordinate = _state11.activeCoordinate, activePayload = _state11.activePayload, offset = _state11.offset; if (!(element && element.props.cursor && isTooltipActive && activeCoordinate)) return null; - var layout = _this7.props.layout, restProps = void 0, cursorComp = __WEBPACK_IMPORTED_MODULE_13__shape_Curve__.a; - if ("ScatterChart" === chartName) restProps = activeCoordinate, cursorComp = __WEBPACK_IMPORTED_MODULE_14__shape_Cross__.a; else if ("BarChart" === chartName) restProps = _this7.getCursorRectangle(), - cursorComp = __WEBPACK_IMPORTED_MODULE_17__shape_Rectangle__.a; else if ("radial" === layout) { + var layout = _this7.props.layout, restProps = void 0, cursorComp = __WEBPACK_IMPORTED_MODULE_12__shape_Curve__.a; + if ("ScatterChart" === chartName) restProps = activeCoordinate, cursorComp = __WEBPACK_IMPORTED_MODULE_13__shape_Cross__.a; else if ("BarChart" === chartName) restProps = _this7.getCursorRectangle(), + cursorComp = __WEBPACK_IMPORTED_MODULE_16__shape_Rectangle__.a; else if ("radial" === layout) { var _getCursorPoints = _this7.getCursorPoints(), cx = _getCursorPoints.cx, cy = _getCursorPoints.cy, radius = _getCursorPoints.radius, startAngle = _getCursorPoints.startAngle, endAngle = _getCursorPoints.endAngle; restProps = { cx: cx, @@ -3066,24 +3081,24 @@ var _bundleJs = []byte((((((((((`!function(modules) { endAngle: endAngle, innerRadius: radius, outerRadius: radius - }, cursorComp = __WEBPACK_IMPORTED_MODULE_15__shape_Sector__.a; + }, cursorComp = __WEBPACK_IMPORTED_MODULE_14__shape_Sector__.a; } else restProps = { points: _this7.getCursorPoints() - }, cursorComp = __WEBPACK_IMPORTED_MODULE_13__shape_Curve__.a; + }, cursorComp = __WEBPACK_IMPORTED_MODULE_12__shape_Curve__.a; var key = element.key || "_recharts-cursor", cursorProps = _extends({ stroke: "#ccc" - }, offset, restProps, Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.k)(element.props.cursor), { + }, offset, restProps, Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.k)(element.props.cursor), { payload: activePayload, key: key, className: "recharts-tooltip-cursor" }); - return Object(__WEBPACK_IMPORTED_MODULE_6_react__.isValidElement)(element.props.cursor) ? Object(__WEBPACK_IMPORTED_MODULE_6_react__.cloneElement)(element.props.cursor, cursorProps) : Object(__WEBPACK_IMPORTED_MODULE_6_react__.createElement)(cursorComp, cursorProps); + return Object(__WEBPACK_IMPORTED_MODULE_5_react__.isValidElement)(element.props.cursor) ? Object(__WEBPACK_IMPORTED_MODULE_5_react__.cloneElement)(element.props.cursor, cursorProps) : Object(__WEBPACK_IMPORTED_MODULE_5_react__.createElement)(cursorComp, cursorProps); }, this.renderPolarAxis = function(element, displayName, index) { var axisType = element.type.axisType, axisMap = _this7.state[axisType + "Map"], axisOption = axisMap[element.props[axisType + "Id"]]; - return Object(__WEBPACK_IMPORTED_MODULE_6_react__.cloneElement)(element, _extends({}, axisOption, { + return Object(__WEBPACK_IMPORTED_MODULE_5_react__.cloneElement)(element, _extends({}, axisOption, { className: axisType, key: element.key || displayName + "-" + index, - ticks: Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.u)(axisOption, !0) + ticks: Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.u)(axisOption, !0) })); }, this.renderXAxis = function(element, displayName, index) { var xAxisMap = _this7.state.xAxisMap, axisObj = xAxisMap[element.props.xAxisId]; @@ -3092,13 +3107,13 @@ var _bundleJs = []byte((((((((((`!function(modules) { var yAxisMap = _this7.state.yAxisMap, axisObj = yAxisMap[element.props.yAxisId]; return _this7.renderAxis(axisObj, element, displayName, index); }, this.renderGrid = function(element) { - var _state12 = _this7.state, xAxisMap = _state12.xAxisMap, yAxisMap = _state12.yAxisMap, offset = _state12.offset, _props6 = _this7.props, width = _props6.width, height = _props6.height, xAxis = Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.b)(xAxisMap), yAxis = Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.b)(yAxisMap), props = element.props || {}; - return Object(__WEBPACK_IMPORTED_MODULE_6_react__.cloneElement)(element, { + var _state12 = _this7.state, xAxisMap = _state12.xAxisMap, yAxisMap = _state12.yAxisMap, offset = _state12.offset, _props6 = _this7.props, width = _props6.width, height = _props6.height, xAxis = Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.b)(xAxisMap), yAxis = Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.b)(yAxisMap), props = element.props || {}; + return Object(__WEBPACK_IMPORTED_MODULE_5_react__.cloneElement)(element, { key: element.key || "grid", - x: Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.g)(props.x) ? props.x : offset.left, - y: Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.g)(props.y) ? props.y : offset.top, - width: Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.g)(props.width) ? props.width : offset.width, - height: Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.g)(props.height) ? props.height : offset.height, + x: Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.g)(props.x) ? props.x : offset.left, + y: Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.g)(props.y) ? props.y : offset.top, + width: Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.g)(props.width) ? props.width : offset.width, + height: Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.g)(props.height) ? props.height : offset.height, xAxis: xAxis, yAxis: yAxis, offset: offset, @@ -3108,12 +3123,12 @@ var _bundleJs = []byte((((((((((`!function(modules) { horizontalCoordinatesGenerator: _this7.horizontalCoordinatesGenerator }); }, this.renderPolarGrid = function(element) { - var _state13 = _this7.state, radiusAxisMap = _state13.radiusAxisMap, angleAxisMap = _state13.angleAxisMap, radiusAxis = Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.b)(radiusAxisMap), angleAxis = Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.b)(angleAxisMap), cx = angleAxis.cx, cy = angleAxis.cy, innerRadius = angleAxis.innerRadius, outerRadius = angleAxis.outerRadius; - return Object(__WEBPACK_IMPORTED_MODULE_6_react__.cloneElement)(element, { - polarAngles: Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.u)(angleAxis, !0).map(function(entry) { + var _state13 = _this7.state, radiusAxisMap = _state13.radiusAxisMap, angleAxisMap = _state13.angleAxisMap, radiusAxis = Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.b)(radiusAxisMap), angleAxis = Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.b)(angleAxisMap), cx = angleAxis.cx, cy = angleAxis.cy, innerRadius = angleAxis.innerRadius, outerRadius = angleAxis.outerRadius; + return Object(__WEBPACK_IMPORTED_MODULE_5_react__.cloneElement)(element, { + polarAngles: Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.u)(angleAxis, !0).map(function(entry) { return entry.coordinate; }), - polarRadius: Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.u)(radiusAxis, !0).map(function(entry) { + polarRadius: Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.u)(radiusAxis, !0).map(function(entry) { return entry.coordinate; }), cx: cx, @@ -3124,13 +3139,13 @@ var _bundleJs = []byte((((((((((`!function(modules) { }); }, this.renderBrush = function(element) { var _props7 = _this7.props, margin = _props7.margin, data = _props7.data, _state14 = _this7.state, offset = _state14.offset, dataStartIndex = _state14.dataStartIndex, dataEndIndex = _state14.dataEndIndex, updateId = _state14.updateId; - return Object(__WEBPACK_IMPORTED_MODULE_6_react__.cloneElement)(element, { + return Object(__WEBPACK_IMPORTED_MODULE_5_react__.cloneElement)(element, { key: element.key || "_recharts-brush", - onChange: Object(__WEBPACK_IMPORTED_MODULE_23__util_ChartUtils__.d)(_this7.handleBrushChange, null, element.props.onChange), + onChange: Object(__WEBPACK_IMPORTED_MODULE_22__util_ChartUtils__.d)(_this7.handleBrushChange, null, element.props.onChange), data: data, - x: Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.g)(element.props.x) ? element.props.x : offset.left, - y: Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.g)(element.props.y) ? element.props.y : offset.top + offset.height + offset.brushBottom - (margin.bottom || 0), - width: Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.g)(element.props.width) ? element.props.width : offset.width, + x: Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.g)(element.props.x) ? element.props.x : offset.left, + y: Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.g)(element.props.y) ? element.props.y : offset.top + offset.height + offset.brushBottom - (margin.bottom || 0), + width: Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.g)(element.props.width) ? element.props.width : offset.width, startIndex: dataStartIndex, endIndex: dataEndIndex, updateId: "brush-" + updateId @@ -3138,7 +3153,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, this.renderReferenceElement = function(element, displayName, index) { if (!element) return null; var _state15 = _this7.state, xAxisMap = _state15.xAxisMap, yAxisMap = _state15.yAxisMap, offset = _state15.offset, _element$props = element.props, xAxisId = _element$props.xAxisId, yAxisId = _element$props.yAxisId; - return Object(__WEBPACK_IMPORTED_MODULE_6_react__.cloneElement)(element, { + return Object(__WEBPACK_IMPORTED_MODULE_5_react__.cloneElement)(element, { key: element.key || displayName + "-" + index, xAxis: xAxisMap[xAxisId], yAxis: yAxisMap[yAxisId], @@ -3152,12 +3167,12 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, this.renderGraphicChild = function(element, displayName, index) { var item = _this7.filterFormatItem(element, displayName, index); if (!item) return null; - var graphicalItem = Object(__WEBPACK_IMPORTED_MODULE_6_react__.cloneElement)(element, item.props), _state16 = _this7.state, isTooltipActive = _state16.isTooltipActive, tooltipAxis = _state16.tooltipAxis, activeTooltipIndex = _state16.activeTooltipIndex, activeLabel = _state16.activeLabel, children = _this7.props.children, tooltipItem = Object(__WEBPACK_IMPORTED_MODULE_18__util_ReactUtils__.i)(children, __WEBPACK_IMPORTED_MODULE_11__component_Tooltip__.a), _item$props2 = item.props, points = _item$props2.points, isRange = _item$props2.isRange, baseLine = _item$props2.baseLine, _item$item$props2 = item.item.props, activeDot = _item$item$props2.activeDot; + var graphicalItem = Object(__WEBPACK_IMPORTED_MODULE_5_react__.cloneElement)(element, item.props), _state16 = _this7.state, isTooltipActive = _state16.isTooltipActive, tooltipAxis = _state16.tooltipAxis, activeTooltipIndex = _state16.activeTooltipIndex, activeLabel = _state16.activeLabel, children = _this7.props.children, tooltipItem = Object(__WEBPACK_IMPORTED_MODULE_17__util_ReactUtils__.i)(children, __WEBPACK_IMPORTED_MODULE_10__component_Tooltip__.a), _item$props2 = item.props, points = _item$props2.points, isRange = _item$props2.isRange, baseLine = _item$props2.baseLine, _item$item$props2 = item.item.props, activeDot = _item$item$props2.activeDot; if (!_item$item$props2.hide && isTooltipActive && tooltipItem && activeDot && activeTooltipIndex >= 0) { var activePoint = void 0, basePoint = void 0; - if (tooltipAxis.dataKey && !tooltipAxis.allowDuplicatedCategory ? (activePoint = Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.a)(points, "payload." + tooltipAxis.dataKey, activeLabel), - basePoint = isRange && baseLine && Object(__WEBPACK_IMPORTED_MODULE_22__util_DataUtils__.a)(baseLine, "payload." + tooltipAxis.dataKey, activeLabel)) : (activePoint = points[activeTooltipIndex], - basePoint = isRange && baseLine && baseLine[activeTooltipIndex]), !__WEBPACK_IMPORTED_MODULE_5_lodash_isNil___default()(activePoint)) return [ graphicalItem ].concat(_toConsumableArray(_this7.renderActivePoints({ + if (tooltipAxis.dataKey && !tooltipAxis.allowDuplicatedCategory ? (activePoint = Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.a)(points, "payload." + tooltipAxis.dataKey, activeLabel), + basePoint = isRange && baseLine && Object(__WEBPACK_IMPORTED_MODULE_21__util_DataUtils__.a)(baseLine, "payload." + tooltipAxis.dataKey, activeLabel)) : (activePoint = points[activeTooltipIndex], + basePoint = isRange && baseLine && baseLine[activeTooltipIndex]), !__WEBPACK_IMPORTED_MODULE_4_lodash_isNil___default()(activePoint)) return [ graphicalItem ].concat(_toConsumableArray(_this7.renderActivePoints({ item: item, activePoint: activePoint, basePoint: basePoint, @@ -3171,7 +3186,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }; __webpack_exports__.a = generateCategoricalChart; }, function(module, exports, __webpack_require__) { - var aFunction = __webpack_require__(207); + var aFunction = __webpack_require__(206); module.exports = function(fn, that, length) { if (aFunction(fn), void 0 === that) return fn; switch (length) { @@ -3268,7 +3283,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { Object.defineProperty(exports, "__esModule", { value: !0 }); - var _typeof2 = __webpack_require__(99), _typeof3 = _interopRequireDefault(_typeof2), _keys = __webpack_require__(36), _keys2 = _interopRequireDefault(_keys); + var _typeof2 = __webpack_require__(100), _typeof3 = _interopRequireDefault(_typeof2), _keys = __webpack_require__(41), _keys2 = _interopRequireDefault(_keys); exports.capitalizeFirstLetter = capitalizeFirstLetter, exports.contains = contains, exports.findIndex = findIndex, exports.find = find, exports.createChainedFunction = createChainedFunction; var _warning = __webpack_require__(11), _warning2 = _interopRequireDefault(_warning); @@ -3278,7 +3293,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { var value = getValue(object, key); return baseIsNative(value) ? value : void 0; } - var baseIsNative = __webpack_require__(551), getValue = __webpack_require__(554); + var baseIsNative = __webpack_require__(562), getValue = __webpack_require__(565); module.exports = getNative; }, function(module, __webpack_exports__, __webpack_require__) { "use strict"; @@ -3312,7 +3327,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { } }), superClass && (Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass); } - var _class, _temp2, __WEBPACK_IMPORTED_MODULE_0_lodash_isNil__ = __webpack_require__(20), __WEBPACK_IMPORTED_MODULE_0_lodash_isNil___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_isNil__), __WEBPACK_IMPORTED_MODULE_1_react__ = __webpack_require__(0), __WEBPACK_IMPORTED_MODULE_1_react___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_react__), __WEBPACK_IMPORTED_MODULE_2_prop_types__ = __webpack_require__(1), __WEBPACK_IMPORTED_MODULE_2_prop_types___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_prop_types__), __WEBPACK_IMPORTED_MODULE_3_reduce_css_calc__ = __webpack_require__(676), __WEBPACK_IMPORTED_MODULE_3_reduce_css_calc___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_reduce_css_calc__), __WEBPACK_IMPORTED_MODULE_4_classnames__ = __webpack_require__(3), __WEBPACK_IMPORTED_MODULE_4_classnames___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_classnames__), __WEBPACK_IMPORTED_MODULE_5__util_DataUtils__ = __webpack_require__(9), __WEBPACK_IMPORTED_MODULE_6__util_ReactUtils__ = __webpack_require__(4), __WEBPACK_IMPORTED_MODULE_7__util_DOMUtils__ = __webpack_require__(185), _extends = Object.assign || function(target) { + var _class, _temp2, __WEBPACK_IMPORTED_MODULE_0_lodash_isNil__ = __webpack_require__(20), __WEBPACK_IMPORTED_MODULE_0_lodash_isNil___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_0_lodash_isNil__), __WEBPACK_IMPORTED_MODULE_1_react__ = __webpack_require__(0), __WEBPACK_IMPORTED_MODULE_1_react___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_1_react__), __WEBPACK_IMPORTED_MODULE_2_prop_types__ = __webpack_require__(1), __WEBPACK_IMPORTED_MODULE_2_prop_types___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_2_prop_types__), __WEBPACK_IMPORTED_MODULE_3_reduce_css_calc__ = __webpack_require__(686), __WEBPACK_IMPORTED_MODULE_3_reduce_css_calc___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_3_reduce_css_calc__), __WEBPACK_IMPORTED_MODULE_4_classnames__ = __webpack_require__(3), __WEBPACK_IMPORTED_MODULE_4_classnames___default = __webpack_require__.n(__WEBPACK_IMPORTED_MODULE_4_classnames__), __WEBPACK_IMPORTED_MODULE_5__util_DataUtils__ = __webpack_require__(9), __WEBPACK_IMPORTED_MODULE_6__util_ReactUtils__ = __webpack_require__(4), __WEBPACK_IMPORTED_MODULE_7__util_DOMUtils__ = __webpack_require__(184), _extends = Object.assign || function(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) Object.prototype.hasOwnProperty.call(source, key) && (target[key] = source[key]); @@ -3532,12 +3547,12 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, _class = _temp)) || _class; __webpack_exports__.a = Dot; }, function(module, exports, __webpack_require__) { - var IObject = __webpack_require__(134), defined = __webpack_require__(136); + var IObject = __webpack_require__(135), defined = __webpack_require__(137); module.exports = function(it) { return IObject(defined(it)); }; }, function(module, exports, __webpack_require__) { - var defined = __webpack_require__(136); + var defined = __webpack_require__(137); module.exports = function(it) { return Object(defined(it)); }; @@ -3576,7 +3591,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { return protoProps && defineProperties(Constructor.prototype, protoProps), staticProps && defineProperties(Constructor, staticProps), Constructor; }; - }(), _warning = __webpack_require__(11), _warning2 = _interopRequireDefault(_warning), _toCss = __webpack_require__(152), _toCss2 = _interopRequireDefault(_toCss), _toCssValue = __webpack_require__(153), _toCssValue2 = _interopRequireDefault(_toCssValue), StyleRule = function() { + }(), _warning = __webpack_require__(11), _warning2 = _interopRequireDefault(_warning), _toCss = __webpack_require__(153), _toCss2 = _interopRequireDefault(_toCss), _toCssValue = __webpack_require__(105), _toCssValue2 = _interopRequireDefault(_toCssValue), StyleRule = function() { function StyleRule(key, style, options) { _classCallCheck(this, StyleRule), this.type = "style", this.isProcessed = !1; var sheet = options.sheet, Renderer = options.Renderer, selector = options.selector; @@ -3638,11 +3653,69 @@ var _bundleJs = []byte((((((((((`!function(modules) { } ]), StyleRule; }(); exports.default = StyleRule; +}, function(module, exports, __webpack_require__) { + "use strict"; + Object.defineProperty(exports, "__esModule", { + value: !0 + }); + var _extends = Object.assign || function(target) { + for (var i = 1; i < arguments.length; i++) { + var source = arguments[i]; + for (var key in source) Object.prototype.hasOwnProperty.call(source, key) && (target[key] = source[key]); + } + return target; + }, menuSkeletons = [ { + id: "home", + menu: { + title: "Home", + icon: "home" + } + }, { + id: "chain", + menu: { + title: "Chain", + icon: "link" + } + }, { + id: "txpool", + menu: { + title: "TxPool", + icon: "credit-card" + } + }, { + id: "network", + menu: { + title: "Network", + icon: "globe" + } + }, { + id: "system", + menu: { + title: "System", + icon: "tachometer" + } + }, { + id: "logs", + menu: { + title: "Logs", + icon: "list" + } + } ]; + exports.MENU = new Map(menuSkeletons.map(function(_ref) { + var id = _ref.id, menu = _ref.menu; + return [ id, _extends({ + id: id + }, menu) ]; + })), exports.DURATION = 200, exports.styles = { + light: { + color: "rgba(255, 255, 255, 0.54)" + } + }; }, function(module, exports, __webpack_require__) { function isSymbol(value) { return "symbol" == typeof value || isObjectLike(value) && baseGetTag(value) == symbolTag; } - var baseGetTag = __webpack_require__(42), isObjectLike = __webpack_require__(37), symbolTag = "[object Symbol]"; + var baseGetTag = __webpack_require__(42), isObjectLike = __webpack_require__(36), symbolTag = "[object Symbol]"; module.exports = isSymbol; }, function(module, exports) { function identity(value) { @@ -3719,12 +3792,12 @@ var _bundleJs = []byte((((((((((`!function(modules) { return _inherits(Rectangle, _Component), _createClass(Rectangle, [ { key: "componentDidMount", value: function() { - if (this.node && this.node.getTotalLength) { + if (this.node && this.node.getTotalLength) try { var totalLength = this.node.getTotalLength(); totalLength && this.setState({ totalLength: totalLength }); - } + } catch (err) {} } }, { key: "render", @@ -4167,7 +4240,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }; }; }, function(module, exports, __webpack_require__) { - var $keys = __webpack_require__(210), enumBugKeys = __webpack_require__(140); + var $keys = __webpack_require__(209), enumBugKeys = __webpack_require__(141); module.exports = Object.keys || function(O) { return $keys(O, enumBugKeys); }; @@ -4185,7 +4258,8 @@ var _bundleJs = []byte((((((((((`!function(modules) { return "@media (min-width:" + ("number" == typeof values[key] ? values[key] : key) + unit + ")"; } function down(key) { - return "@media (max-width:" + (("number" == typeof values[key] ? values[key] : key) - step / 100) + unit + ")"; + var endIndex = keys.indexOf(key) + 1, upperbound = values[keys[endIndex]]; + return endIndex === keys.length ? up("xs") : "@media (max-width:" + (("number" == typeof upperbound && endIndex > 0 ? upperbound : key) - step / 100) + unit + ")"; } function between(start, end) { var endIndex = keys.indexOf(end) + 1; @@ -4224,7 +4298,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, function(module, exports, __webpack_require__) { "use strict"; exports.__esModule = !0; - var _getDisplayName = __webpack_require__(227), _getDisplayName2 = function(obj) { + var _getDisplayName = __webpack_require__(226), _getDisplayName2 = function(obj) { return obj && obj.__esModule ? obj : { default: obj }; @@ -4263,7 +4337,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { return protoProps && defineProperties(Constructor.prototype, protoProps), staticProps && defineProperties(Constructor, staticProps), Constructor; }; - }(), _createRule = __webpack_require__(104), _createRule2 = _interopRequireDefault(_createRule), _linkRule = __webpack_require__(232), _linkRule2 = _interopRequireDefault(_linkRule), _StyleRule = __webpack_require__(60), _StyleRule2 = _interopRequireDefault(_StyleRule), _escape = __webpack_require__(429), _escape2 = _interopRequireDefault(_escape), RuleList = function() { + }(), _createRule = __webpack_require__(106), _createRule2 = _interopRequireDefault(_createRule), _linkRule = __webpack_require__(231), _linkRule2 = _interopRequireDefault(_linkRule), _StyleRule = __webpack_require__(60), _StyleRule2 = _interopRequireDefault(_StyleRule), _escape = __webpack_require__(424), _escape2 = _interopRequireDefault(_escape), RuleList = function() { function RuleList(options) { _classCallCheck(this, RuleList), this.map = {}, this.raw = {}, this.index = [], this.options = options, this.classes = options.classes; @@ -4349,6 +4423,9 @@ var _bundleJs = []byte((((((((((`!function(modules) { } ]), RuleList; }(); exports.default = RuleList; +}, function(module, exports, __webpack_require__) { + var root = __webpack_require__(32), Symbol = root.Symbol; + module.exports = Symbol; }, function(module, __webpack_exports__, __webpack_require__) { "use strict"; function _objectWithoutProperties(obj, keys) { @@ -4392,12 +4469,9 @@ var _bundleJs = []byte((((((((((`!function(modules) { children: __WEBPACK_IMPORTED_MODULE_1_prop_types___default.a.oneOfType([ __WEBPACK_IMPORTED_MODULE_1_prop_types___default.a.arrayOf(__WEBPACK_IMPORTED_MODULE_1_prop_types___default.a.node), __WEBPACK_IMPORTED_MODULE_1_prop_types___default.a.node ]) }; Surface.propTypes = propTypes, __webpack_exports__.a = Surface; -}, function(module, exports, __webpack_require__) { - var root = __webpack_require__(31), Symbol = root.Symbol; - module.exports = Symbol; }, function(module, __webpack_exports__, __webpack_require__) { "use strict"; - var __WEBPACK_IMPORTED_MODULE_0__src_path__ = __webpack_require__(573); + var __WEBPACK_IMPORTED_MODULE_0__src_path__ = __webpack_require__(584); __webpack_require__.d(__webpack_exports__, "a", function() { return __WEBPACK_IMPORTED_MODULE_0__src_path__.a; }); @@ -4455,7 +4529,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { function baseIteratee(value) { return "function" == typeof value ? value : null == value ? identity : "object" == typeof value ? isArray(value) ? baseMatchesProperty(value[0], value[1]) : baseMatches(value) : property(value); } - var baseMatches = __webpack_require__(658), baseMatchesProperty = __webpack_require__(661), identity = __webpack_require__(62), isArray = __webpack_require__(12), property = __webpack_require__(665); + var baseMatches = __webpack_require__(669), baseMatchesProperty = __webpack_require__(672), identity = __webpack_require__(63), isArray = __webpack_require__(12), property = __webpack_require__(676); module.exports = baseIteratee; }, function(module, __webpack_exports__, __webpack_require__) { "use strict"; @@ -4506,29 +4580,29 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, linearish(scale); } __webpack_exports__.b = linearish, __webpack_exports__.a = linear; - var __WEBPACK_IMPORTED_MODULE_0_d3_array__ = __webpack_require__(38), __WEBPACK_IMPORTED_MODULE_1_d3_interpolate__ = __webpack_require__(87), __WEBPACK_IMPORTED_MODULE_2__continuous__ = __webpack_require__(125), __WEBPACK_IMPORTED_MODULE_3__tickFormat__ = __webpack_require__(732); + var __WEBPACK_IMPORTED_MODULE_0_d3_array__ = __webpack_require__(37), __WEBPACK_IMPORTED_MODULE_1_d3_interpolate__ = __webpack_require__(88), __WEBPACK_IMPORTED_MODULE_2__continuous__ = __webpack_require__(126), __WEBPACK_IMPORTED_MODULE_3__tickFormat__ = __webpack_require__(742); }, function(module, __webpack_exports__, __webpack_require__) { "use strict"; - var __WEBPACK_IMPORTED_MODULE_0__src_value__ = __webpack_require__(188); + var __WEBPACK_IMPORTED_MODULE_0__src_value__ = __webpack_require__(187); __webpack_require__.d(__webpack_exports__, "a", function() { return __WEBPACK_IMPORTED_MODULE_0__src_value__.a; }); - var __WEBPACK_IMPORTED_MODULE_5__src_number__ = (__webpack_require__(309), __webpack_require__(191), - __webpack_require__(307), __webpack_require__(310), __webpack_require__(124)); + var __WEBPACK_IMPORTED_MODULE_5__src_number__ = (__webpack_require__(305), __webpack_require__(190), + __webpack_require__(303), __webpack_require__(306), __webpack_require__(125)); __webpack_require__.d(__webpack_exports__, "c", function() { return __WEBPACK_IMPORTED_MODULE_5__src_number__.a; }); - var __WEBPACK_IMPORTED_MODULE_7__src_round__ = (__webpack_require__(311), __webpack_require__(722)); + var __WEBPACK_IMPORTED_MODULE_7__src_round__ = (__webpack_require__(307), __webpack_require__(732)); __webpack_require__.d(__webpack_exports__, "d", function() { return __WEBPACK_IMPORTED_MODULE_7__src_round__.a; }); - var __WEBPACK_IMPORTED_MODULE_15__src_cubehelix__ = (__webpack_require__(312), __webpack_require__(723), - __webpack_require__(726), __webpack_require__(306), __webpack_require__(727), __webpack_require__(728), - __webpack_require__(729), __webpack_require__(730)); + var __WEBPACK_IMPORTED_MODULE_15__src_cubehelix__ = (__webpack_require__(308), __webpack_require__(733), + __webpack_require__(736), __webpack_require__(302), __webpack_require__(737), __webpack_require__(738), + __webpack_require__(739), __webpack_require__(740)); __webpack_require__.d(__webpack_exports__, "b", function() { return __WEBPACK_IMPORTED_MODULE_15__src_cubehelix__.a; }); - __webpack_require__(731); + __webpack_require__(741); }, function(module, __webpack_exports__, __webpack_require__) { "use strict"; function linear(a, d) { @@ -4555,7 +4629,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { return d ? linear(a, d) : Object(__WEBPACK_IMPORTED_MODULE_0__constant__.a)(isNaN(a) ? b : a); } __webpack_exports__.c = hue, __webpack_exports__.b = gamma, __webpack_exports__.a = nogamma; - var __WEBPACK_IMPORTED_MODULE_0__constant__ = __webpack_require__(308); + var __WEBPACK_IMPORTED_MODULE_0__constant__ = __webpack_require__(304); }, function(module, __webpack_exports__, __webpack_require__) { "use strict"; __webpack_exports__.a = function(s) { @@ -4750,7 +4824,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }, function(module, exports, __webpack_require__) { "use strict"; (function(process) { - var emptyFunction = __webpack_require__(40), warning = emptyFunction; + var emptyFunction = __webpack_require__(39), warning = emptyFunction; if ("production" !== process.env.NODE_ENV) { var printWarning = function(format) { for (var _len = arguments.length, args = Array(_len > 1 ? _len - 1 : 0), _key = 1; _key < _len; _key++) args[_key - 1] = arguments[_key]; @@ -4785,7 +4859,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { } } } - "production" === process.env.NODE_ENV ? (checkDCE(), module.exports = __webpack_require__(339)) : module.exports = __webpack_require__(342); + "production" === process.env.NODE_ENV ? (checkDCE(), module.exports = __webpack_require__(334)) : module.exports = __webpack_require__(337); }).call(exports, __webpack_require__(2)); }, function(module, exports, __webpack_require__) { "use strict"; @@ -4803,7 +4877,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { var hasOwnProperty = Object.prototype.hasOwnProperty; module.exports = shallowEqual; }, function(module, exports, __webpack_require__) { - var toInteger = __webpack_require__(137), min = Math.min; + var toInteger = __webpack_require__(138), min = Math.min; module.exports = function(it) { return it > 0 ? min(toInteger(it), 9007199254740991) : 0; }; @@ -4822,7 +4896,7 @@ var _bundleJs = []byte((((((((((`!function(modules) { }; } exports.__esModule = !0; - var _iterator = __webpack_require__(357), _iterator2 = _interopRequireDefault(_iterator), _symbol = __webpack_require__(365), _symbol2 = _interopRequireDefault(_symbol), _typeof = "function" == typeof _symbol2.default && "symbol" == typeof _iterator2.default ? function(obj) { + var _iterator = __webpack_require__(352), _iterator2 = _interopRequireDefault(_iterator), _symbol = __webpack_require__(360), _symbol2 = _interopRequireDefault(_symbol), _typeof = "function" == typeof _symbol2.default && "symbol" == typeof _iterator2.default ? function(obj) { return typeof obj; } : function(obj) { return obj && "function" == typeof _symbol2.default && obj.constructor === _symbol2.default && obj !== _symbol2.default.prototype ? "symbol" : typeof obj; @@ -4833,9 +4907,9 @@ var _bundleJs = []byte((((((((((`!function(modules) { return obj && "function" == typeof _symbol2.default && obj.constructor === _symbol2.default && obj !== _symbol2.default.prototype ? "symbol" : void 0 === obj ? "undefined" : _typeof(obj); }; }, function(module, exports, __webpack_require__) { - var anObject = __webpack_require__(48), dPs = __webpack_require__(361), enumBugKeys = __webpack_require__(140), IE_PROTO = __webpack_require__(138)("IE_PROTO"), Empty = function() {}, createDict = function() { - var iframeDocument, iframe = __webpack_require__(209)("iframe"), i = enumBugKeys.length; - for (iframe.style.display = "none", __webpack_require__(362).appendChild(iframe), + var anObject = __webpack_require__(48), dPs = __webpack_require__(356), enumBugKeys = __webpack_require__(141), IE_PROTO = __webpack_require__(139)("IE_PROTO"), Empty = function() {}, createDict = function() { + var iframeDocument, iframe = __webpack_require__(208)("iframe"), i = enumBugKeys.length; + for (iframe.style.display = "none", __webpack_require__(357).appendChild(iframe), iframe.src = "javascript:", iframeDocument = iframe.contentWindow.document, iframeDocument.open(), iframeDocument.write("