From 6e0667fa06caa15a5333bc4c902315c9d56a3739 Mon Sep 17 00:00:00 2001 From: Vlad Date: Mon, 26 Feb 2018 13:58:04 +0100 Subject: [PATCH 01/33] whisper: wnode updated - all messages are saved if savedir param is given --- cmd/wnode/main.go | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/cmd/wnode/main.go b/cmd/wnode/main.go index 971b1c0ab8..8694219c57 100644 --- a/cmd/wnode/main.go +++ b/cmd/wnode/main.go @@ -76,14 +76,14 @@ var ( // cmd arguments var ( - bootstrapMode = flag.Bool("standalone", false, "boostrap node: don't actively connect to peers, wait for incoming connections") - forwarderMode = flag.Bool("forwarder", false, "forwarder mode: only forward messages, neither send nor decrypt messages") + bootstrapMode = flag.Bool("standalone", false, "boostrap node: does not initiate connection to peers, just waits for incoming connections") + forwarderMode = flag.Bool("forwarder", false, "forwarder mode: only forwards messages, neither encrypts nor decrypts messages") mailServerMode = flag.Bool("mailserver", false, "mail server mode: delivers expired messages on demand") requestMail = flag.Bool("mailclient", false, "request expired messages from the bootstrap server") asymmetricMode = flag.Bool("asym", false, "use asymmetric encryption") generateKey = flag.Bool("generatekey", false, "generate and show the private key") fileExMode = flag.Bool("fileexchange", false, "file exchange mode") - testMode = flag.Bool("test", false, "use of predefined parameters for diagnostics") + testMode = flag.Bool("test", false, "use of predefined parameters for diagnostics (password, etc.)") echoMode = flag.Bool("echo", false, "echo mode: prints some arguments for diagnostics") argVerbosity = flag.Int("verbosity", int(log.LvlError), "log verbosity level") @@ -99,7 +99,7 @@ var ( argIDFile = flag.String("idfile", "", "file name with node id (private key)") argEnode = flag.String("boot", "", "bootstrap node you want to connect to (e.g. enode://e454......08d50@52.176.211.200:16428)") argTopic = flag.String("topic", "", "topic in hexadecimal format (e.g. 70a4beef)") - argSaveDir = flag.String("savedir", "", "directory where incoming messages will be saved as files") + argSaveDir = flag.String("savedir", "", "directory where all incoming messages will be saved as files") ) func main() { @@ -548,20 +548,16 @@ func messageLoop() { for { select { case <-ticker.C: - messages := sf.Retrieve() + m1 := sf.Retrieve() + m2 := af.Retrieve() + messages := append(m1, m2...) for _, msg := range messages { - if *fileExMode || len(msg.Payload) > 2048 { + // NB: it is possible that *fileExMode == false && len(*argSaveDir) > 0 + if len(*argSaveDir) > 0 { writeMessageToFile(*argSaveDir, msg) - } else { - printMessageInfo(msg) } - } - messages = af.Retrieve() - for _, msg := range messages { - if *fileExMode || len(msg.Payload) > 2048 { - writeMessageToFile(*argSaveDir, msg) - } else { + if !*fileExMode && len(msg.Payload) <= 2048 { printMessageInfo(msg) } } From 61c9730b2de2719059f0fb2ce8f965c0cde34527 Mon Sep 17 00:00:00 2001 From: JU HYEONG PARK Date: Tue, 27 Feb 2018 01:22:46 +0900 Subject: [PATCH 02/33] p2p: fix doEncHandshake documentation (#16184) --- p2p/rlpx.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/p2p/rlpx.go b/p2p/rlpx.go index e65a0b6047..1889edac99 100644 --- a/p2p/rlpx.go +++ b/p2p/rlpx.go @@ -121,10 +121,6 @@ func (t *rlpx) close(err error) { t.fd.Close() } -// doEncHandshake runs the protocol handshake using authenticated -// messages. the protocol handshake is the first authenticated message -// and also verifies whether the encryption handshake 'worked' and the -// remote side actually provided the right public key. func (t *rlpx) doProtoHandshake(our *protoHandshake) (their *protoHandshake, err error) { // Writing our handshake happens concurrently, we prefer // returning the handshake read error. If the remote side @@ -175,6 +171,10 @@ func readProtocolHandshake(rw MsgReader, our *protoHandshake) (*protoHandshake, return &hs, nil } +// doEncHandshake runs the protocol handshake using authenticated +// messages. the protocol handshake is the first authenticated message +// and also verifies whether the encryption handshake 'worked' and the +// remote side actually provided the right public key. func (t *rlpx) doEncHandshake(prv *ecdsa.PrivateKey, dial *discover.Node) (discover.NodeID, error) { var ( sec secrets From f4e676cccdeaf781a449410482e75267672db61b Mon Sep 17 00:00:00 2001 From: Vlad Date: Mon, 26 Feb 2018 19:26:36 +0100 Subject: [PATCH 03/33] whipser: comments updated --- cmd/wnode/main.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cmd/wnode/main.go b/cmd/wnode/main.go index 8694219c57..0f86adb81c 100644 --- a/cmd/wnode/main.go +++ b/cmd/wnode/main.go @@ -76,8 +76,8 @@ var ( // cmd arguments var ( - bootstrapMode = flag.Bool("standalone", false, "boostrap node: does not initiate connection to peers, just waits for incoming connections") - forwarderMode = flag.Bool("forwarder", false, "forwarder mode: only forwards messages, neither encrypts nor decrypts messages") + bootstrapMode = flag.Bool("standalone", false, "boostrap node: don't initiate connection to peers, just wait for incoming connections") + forwarderMode = flag.Bool("forwarder", false, "forwarder mode: only forward messages, neither encrypt nor decrypt messages") mailServerMode = flag.Bool("mailserver", false, "mail server mode: delivers expired messages on demand") requestMail = flag.Bool("mailclient", false, "request expired messages from the bootstrap server") asymmetricMode = flag.Bool("asym", false, "use asymmetric encryption") @@ -552,7 +552,9 @@ func messageLoop() { m2 := af.Retrieve() messages := append(m1, m2...) for _, msg := range messages { - // NB: it is possible that *fileExMode == false && len(*argSaveDir) > 0 + // All messages are saved upon specifying argSaveDir. + // fileExMode only specifies how messages are displayed on the console after they are saved. + // if fileExMode == true, only the hashes are displayed, since messages might be too big. if len(*argSaveDir) > 0 { writeMessageToFile(*argSaveDir, msg) } From 2e9c8fd4fbd5d0de0ced03961d268c7492917860 Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Tue, 27 Feb 2018 05:52:59 -0500 Subject: [PATCH 04/33] eth, les: allow exceeding maxPeers for trusted peers (#16189) Fixes #3326, #14472 --- eth/handler.go | 3 ++- les/handler.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/eth/handler.go b/eth/handler.go index c2426544f6..3fae0cd00d 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -249,7 +249,8 @@ func (pm *ProtocolManager) newPeer(pv int, p *p2p.Peer, rw p2p.MsgReadWriter) *p // handle is the callback invoked to manage the life cycle of an eth peer. When // this function terminates, the peer is disconnected. func (pm *ProtocolManager) handle(p *peer) error { - if pm.peers.Len() >= pm.maxPeers { + // Ignore maxPeers if this is a trusted peer + if pm.peers.Len() >= pm.maxPeers && !p.Peer.Info().Network.Trusted { return p2p.DiscTooManyPeers } p.Log().Debug("Ethereum peer connected", "name", p.Name()) diff --git a/les/handler.go b/les/handler.go index 864abe605a..9627f392be 100644 --- a/les/handler.go +++ b/les/handler.go @@ -260,7 +260,8 @@ func (pm *ProtocolManager) newPeer(pv int, nv uint64, p *p2p.Peer, rw p2p.MsgRea // handle is the callback invoked to manage the life cycle of a les peer. When // this function terminates, the peer is disconnected. func (pm *ProtocolManager) handle(p *peer) error { - if pm.peers.Len() >= pm.maxPeers { + // Ignore maxPeers if this is a trusted peer + if pm.peers.Len() >= pm.maxPeers && !p.Peer.Info().Network.Trusted { return p2p.DiscTooManyPeers } From c41f1a3e23e603ffc01d787d7509adf1712633b3 Mon Sep 17 00:00:00 2001 From: Saulius Grigaitis Date: Tue, 27 Feb 2018 12:56:51 +0200 Subject: [PATCH 05/33] puppeth: fix Parity Chain Spec parameter GasLimitBoundDivision (#16188) --- cmd/puppeth/genesis.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/cmd/puppeth/genesis.go b/cmd/puppeth/genesis.go index f747f47395..1974a94aa2 100644 --- a/cmd/puppeth/genesis.go +++ b/cmd/puppeth/genesis.go @@ -168,19 +168,18 @@ type parityChainSpec struct { Engine struct { Ethash struct { Params struct { - MinimumDifficulty *hexutil.Big `json:"minimumDifficulty"` - DifficultyBoundDivisor *hexutil.Big `json:"difficultyBoundDivisor"` - GasLimitBoundDivisor hexutil.Uint64 `json:"gasLimitBoundDivisor"` - DurationLimit *hexutil.Big `json:"durationLimit"` - BlockReward *hexutil.Big `json:"blockReward"` - HomesteadTransition uint64 `json:"homesteadTransition"` - EIP150Transition uint64 `json:"eip150Transition"` - EIP160Transition uint64 `json:"eip160Transition"` - EIP161abcTransition uint64 `json:"eip161abcTransition"` - EIP161dTransition uint64 `json:"eip161dTransition"` - EIP649Reward *hexutil.Big `json:"eip649Reward"` - EIP100bTransition uint64 `json:"eip100bTransition"` - EIP649Transition uint64 `json:"eip649Transition"` + MinimumDifficulty *hexutil.Big `json:"minimumDifficulty"` + DifficultyBoundDivisor *hexutil.Big `json:"difficultyBoundDivisor"` + DurationLimit *hexutil.Big `json:"durationLimit"` + BlockReward *hexutil.Big `json:"blockReward"` + HomesteadTransition uint64 `json:"homesteadTransition"` + EIP150Transition uint64 `json:"eip150Transition"` + EIP160Transition uint64 `json:"eip160Transition"` + EIP161abcTransition uint64 `json:"eip161abcTransition"` + EIP161dTransition uint64 `json:"eip161dTransition"` + EIP649Reward *hexutil.Big `json:"eip649Reward"` + EIP100bTransition uint64 `json:"eip100bTransition"` + EIP649Transition uint64 `json:"eip649Transition"` } `json:"params"` } `json:"Ethash"` } `json:"engine"` @@ -188,6 +187,7 @@ type parityChainSpec struct { Params struct { MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"` MinGasLimit hexutil.Uint64 `json:"minGasLimit"` + GasLimitBoundDivisor hexutil.Uint64 `json:"gasLimitBoundDivisor"` NetworkID hexutil.Uint64 `json:"networkID"` MaxCodeSize uint64 `json:"maxCodeSize"` EIP155Transition uint64 `json:"eip155Transition"` @@ -270,7 +270,6 @@ func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []strin } spec.Engine.Ethash.Params.MinimumDifficulty = (*hexutil.Big)(params.MinimumDifficulty) spec.Engine.Ethash.Params.DifficultyBoundDivisor = (*hexutil.Big)(params.DifficultyBoundDivisor) - spec.Engine.Ethash.Params.GasLimitBoundDivisor = (hexutil.Uint64)(params.GasLimitBoundDivisor) spec.Engine.Ethash.Params.DurationLimit = (*hexutil.Big)(params.DurationLimit) spec.Engine.Ethash.Params.BlockReward = (*hexutil.Big)(ethash.FrontierBlockReward) spec.Engine.Ethash.Params.HomesteadTransition = genesis.Config.HomesteadBlock.Uint64() @@ -284,6 +283,7 @@ func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []strin spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize) spec.Params.MinGasLimit = (hexutil.Uint64)(params.MinGasLimit) + spec.Params.GasLimitBoundDivisor = (hexutil.Uint64)(params.GasLimitBoundDivisor) spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainId.Uint64()) spec.Params.MaxCodeSize = params.MaxCodeSize spec.Params.EIP155Transition = genesis.Config.EIP155Block.Uint64() From dd389e595f3fa1c9644478b4c21a5127e916ec04 Mon Sep 17 00:00:00 2001 From: Elad Nachmias Date: Tue, 27 Feb 2018 12:04:47 +0100 Subject: [PATCH 06/33] eth: added travis build badge (#16117) * eth: added travis build status for master branch * README: fix travis badge order, link to CI --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index aad14ed7d8..3d0d4d35d4 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ Official golang implementation of the Ethereum protocol. https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667 )](https://godoc.org/github.com/ethereum/go-ethereum) [![Go Report Card](https://goreportcard.com/badge/github.com/ethereum/go-ethereum)](https://goreportcard.com/report/github.com/ethereum/go-ethereum) +[![Travis](https://travis-ci.org/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.org/ethereum/go-ethereum) [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/ethereum/go-ethereum?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) Automated builds are available for stable releases and the unstable master branch. From 18bb3da55e4d2f6b5e9142b7f5d5bd76f2fb78b0 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 27 Feb 2018 14:30:07 +0100 Subject: [PATCH 07/33] node: fill StandardCounters as part of debugapi/metrics (#16054) --- node/api.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/node/api.go b/node/api.go index 992a7c4166..a3b8bc0bb5 100644 --- a/node/api.go +++ b/node/api.go @@ -308,6 +308,11 @@ func (api *PublicDebugAPI) Metrics(raw bool) (map[string]interface{}, error) { // Fill the counter with the metric details, formatting if requested if raw { switch metric := metric.(type) { + case metrics.Counter: + root[name] = map[string]interface{}{ + "Overall": float64(metric.Count()), + } + case metrics.Meter: root[name] = map[string]interface{}{ "AvgRate01Min": metric.Rate1(), @@ -338,6 +343,11 @@ func (api *PublicDebugAPI) Metrics(raw bool) (map[string]interface{}, error) { } } else { switch metric := metric.(type) { + case metrics.Counter: + root[name] = map[string]interface{}{ + "Overall": float64(metric.Count()), + } + case metrics.Meter: root[name] = map[string]interface{}{ "Avg01Min": format(metric.Rate1()*60, metric.Rate1()), From b574b5776695eb30e034fd8c7a468b3f03d4c6b9 Mon Sep 17 00:00:00 2001 From: Elad Nachmias Date: Tue, 27 Feb 2018 14:32:38 +0100 Subject: [PATCH 08/33] swarm: give correct error on 0x hash prefix (#16195) - added a case error struct that contains information about certain error cases in which we would like to output more information to the client - added a validation method that iterates and adds the information that is stored in the error cases --- .gitignore | 3 +++ swarm/api/http/error.go | 43 +++++++++++++++++++++++++++---- swarm/api/http/error_templates.go | 11 ++++++++ swarm/api/http/error_test.go | 40 ++++++++++++++++++++++++---- swarm/api/http/server.go | 22 ++++++++-------- 5 files changed, 98 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index 0763d8492c..b8d2929016 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,9 @@ profile.cov # IdeaIDE .idea +# VS Code +.vscode + # dashboard /dashboard/assets/flow-typed /dashboard/assets/node_modules diff --git a/swarm/api/http/error.go b/swarm/api/http/error.go index 831cf23fe2..9a65412cf9 100644 --- a/swarm/api/http/error.go +++ b/swarm/api/http/error.go @@ -35,6 +35,7 @@ import ( //templateMap holds a mapping of an HTTP error code to a template var templateMap map[int]*template.Template +var caseErrors []CaseError //metrics variables var ( @@ -51,6 +52,13 @@ type ErrorParams struct { Details template.HTML } +//a custom error case struct that would be used to store validators and +//additional error info to display with client responses. +type CaseError struct { + Validator func(*Request) bool + Msg func(*Request) string +} + //we init the error handling right on boot time, so lookup and http response is fast func init() { initErrHandling() @@ -74,6 +82,29 @@ func initErrHandling() { //assign formatted HTML to the code templateMap[code] = template.Must(template.New(fmt.Sprintf("%d", code)).Parse(tname)) } + + caseErrors = []CaseError{ + { + Validator: func(r *Request) bool { return r.uri != nil && r.uri.Addr != "" && strings.HasPrefix(r.uri.Addr, "0x") }, + Msg: func(r *Request) string { + uriCopy := r.uri + uriCopy.Addr = strings.TrimPrefix(uriCopy.Addr, "0x") + return fmt.Sprintf(`The requested hash seems to be prefixed with '0x'. You will be redirected to the correct URL within 5 seconds.
+ Please click here if your browser does not redirect you.`, "/"+uriCopy.String()) + }, + }} +} + +//ValidateCaseErrors is a method that process the request object through certain validators +//that assert if certain conditions are met for further information to log as an error +func ValidateCaseErrors(r *Request) string { + for _, err := range caseErrors { + if err.Validator(r) { + return err.Msg(r) + } + } + + return "" } //ShowMultipeChoices is used when a user requests a resource in a manifest which results @@ -82,10 +113,10 @@ func initErrHandling() { //For example, if the user requests bzz://read and that manifest contains entries //"readme.md" and "readinglist.txt", a HTML page is returned with this two links. //This only applies if the manifest has no default entry -func ShowMultipleChoices(w http.ResponseWriter, r *http.Request, list api.ManifestList) { +func ShowMultipleChoices(w http.ResponseWriter, r *Request, list api.ManifestList) { msg := "" if list.Entries == nil { - ShowError(w, r, "Internal Server Error", http.StatusInternalServerError) + ShowError(w, r, "Could not resolve", http.StatusInternalServerError) return } //make links relative @@ -102,7 +133,7 @@ func ShowMultipleChoices(w http.ResponseWriter, r *http.Request, list api.Manife //create clickable link for each entry msg += "" + e.Path + "
" } - respond(w, r, &ErrorParams{ + respond(w, &r.Request, &ErrorParams{ Code: http.StatusMultipleChoices, Details: template.HTML(msg), Timestamp: time.Now().Format(time.RFC1123), @@ -115,13 +146,15 @@ func ShowMultipleChoices(w http.ResponseWriter, r *http.Request, list api.Manife //The function just takes a string message which will be displayed in the error page. //The code is used to evaluate which template will be displayed //(and return the correct HTTP status code) -func ShowError(w http.ResponseWriter, r *http.Request, msg string, code int) { +func ShowError(w http.ResponseWriter, r *Request, msg string, code int) { + additionalMessage := ValidateCaseErrors(r) if code == http.StatusInternalServerError { log.Error(msg) } - respond(w, r, &ErrorParams{ + respond(w, &r.Request, &ErrorParams{ Code: code, Msg: msg, + Details: template.HTML(additionalMessage), Timestamp: time.Now().Format(time.RFC1123), template: getTemplate(code), }) diff --git a/swarm/api/http/error_templates.go b/swarm/api/http/error_templates.go index 0457cb8a70..cc9b996ba4 100644 --- a/swarm/api/http/error_templates.go +++ b/swarm/api/http/error_templates.go @@ -168,6 +168,11 @@ func GetGenericErrorPage() string { {{.Msg}} + + + {{.Details}} + + @@ -342,6 +347,12 @@ func GetNotFoundErrorPage() string { {{.Msg}} + + + {{.Details}} + + + diff --git a/swarm/api/http/error_test.go b/swarm/api/http/error_test.go index c2c8b908b8..dc545722e1 100644 --- a/swarm/api/http/error_test.go +++ b/swarm/api/http/error_test.go @@ -18,12 +18,13 @@ package http_test import ( "encoding/json" - "golang.org/x/net/html" "io/ioutil" "net/http" "strings" "testing" + "golang.org/x/net/html" + "github.com/ethereum/go-ethereum/swarm/testutil" ) @@ -96,8 +97,37 @@ func Test500Page(t *testing.T) { defer resp.Body.Close() respbody, err = ioutil.ReadAll(resp.Body) - if resp.StatusCode != 500 || !strings.Contains(string(respbody), "500") { - t.Fatalf("Invalid Status Code received, expected 500, got %d", resp.StatusCode) + if resp.StatusCode != 404 { + t.Fatalf("Invalid Status Code received, expected 404, got %d", resp.StatusCode) + } + + _, err = html.Parse(strings.NewReader(string(respbody))) + if err != nil { + t.Fatalf("HTML validation failed for error page returned!") + } +} +func Test500PageWith0xHashPrefix(t *testing.T) { + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() + + var resp *http.Response + var respbody []byte + + url := srv.URL + "/bzz:/0xthisShouldFailWith500CodeAndAHelpfulMessage" + resp, err := http.Get(url) + + if err != nil { + t.Fatalf("Request failed: %v", err) + } + defer resp.Body.Close() + respbody, err = ioutil.ReadAll(resp.Body) + + if resp.StatusCode != 404 { + t.Fatalf("Invalid Status Code received, expected 404, got %d", resp.StatusCode) + } + + if !strings.Contains(string(respbody), "The requested hash seems to be prefixed with") { + t.Fatalf("Did not receive the expected error message") } _, err = html.Parse(strings.NewReader(string(respbody))) @@ -127,8 +157,8 @@ func TestJsonResponse(t *testing.T) { defer resp.Body.Close() respbody, err = ioutil.ReadAll(resp.Body) - if resp.StatusCode != 500 { - t.Fatalf("Invalid Status Code received, expected 500, got %d", resp.StatusCode) + if resp.StatusCode != 404 { + t.Fatalf("Invalid Status Code received, expected 404, got %d", resp.StatusCode) } if !isJSON(string(respbody)) { diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index df90fd04f9..b8e7436cf0 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -336,7 +336,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { key, err := s.api.Resolve(r.uri) if err != nil { getFail.Inc(1) - s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + s.NotFound(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) return } @@ -421,7 +421,7 @@ func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) { key, err := s.api.Resolve(r.uri) if err != nil { getFilesFail.Inc(1) - s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + s.NotFound(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) return } @@ -494,7 +494,7 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) { key, err := s.api.Resolve(r.uri) if err != nil { getListFail.Inc(1) - s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + s.NotFound(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) return } @@ -598,7 +598,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { key, err := s.api.Resolve(r.uri) if err != nil { getFileFail.Inc(1) - s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + s.NotFound(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) return } @@ -628,7 +628,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { s.logDebug(fmt.Sprintf("Multiple choices! --> %v", list)) //show a nice page links to available entries - ShowMultipleChoices(w, &r.Request, list) + ShowMultipleChoices(w, r, list) return } @@ -693,7 +693,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { // strictly a traditional PUT request which replaces content // at a URI, and POST is more ubiquitous) if uri.Raw() || uri.DeprecatedRaw() { - ShowError(w, r, fmt.Sprintf("No PUT to %s allowed.", uri), http.StatusBadRequest) + ShowError(w, req, fmt.Sprintf("No PUT to %s allowed.", uri), http.StatusBadRequest) return } else { s.HandlePostFiles(w, req) @@ -701,7 +701,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { case "DELETE": if uri.Raw() || uri.DeprecatedRaw() { - ShowError(w, r, fmt.Sprintf("No DELETE to %s allowed.", uri), http.StatusBadRequest) + ShowError(w, req, fmt.Sprintf("No DELETE to %s allowed.", uri), http.StatusBadRequest) return } s.HandleDelete(w, req) @@ -725,7 +725,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.HandleGetFile(w, req) default: - ShowError(w, r, fmt.Sprintf("Method "+r.Method+" is not supported.", uri), http.StatusMethodNotAllowed) + ShowError(w, req, fmt.Sprintf("Method "+r.Method+" is not supported.", uri), http.StatusMethodNotAllowed) } } @@ -757,13 +757,13 @@ func (s *Server) logError(format string, v ...interface{}) { } func (s *Server) BadRequest(w http.ResponseWriter, r *Request, reason string) { - ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, reason), http.StatusBadRequest) + ShowError(w, r, fmt.Sprintf("Bad request %s %s: %s", r.Request.Method, r.uri, reason), http.StatusBadRequest) } func (s *Server) Error(w http.ResponseWriter, r *Request, err error) { - ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError) + ShowError(w, r, fmt.Sprintf("Error serving %s %s: %s", r.Request.Method, r.uri, err), http.StatusInternalServerError) } func (s *Server) NotFound(w http.ResponseWriter, r *Request, err error) { - ShowError(w, &r.Request, fmt.Sprintf("NOT FOUND error serving %s %s: %s", r.Method, r.uri, err), http.StatusNotFound) + ShowError(w, r, fmt.Sprintf("NOT FOUND error serving %s %s: %s", r.Request.Method, r.uri, err), http.StatusNotFound) } From dadf4d53abd8be13f59fb09820712d74ce8f89a7 Mon Sep 17 00:00:00 2001 From: Vlad Date: Tue, 27 Feb 2018 15:45:00 +0100 Subject: [PATCH 09/33] whisper: mailserver no longer supports the signature vaidation --- whisper/mailserver/mailserver.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/whisper/mailserver/mailserver.go b/whisper/mailserver/mailserver.go index 6555fd5c0b..a889c2bd23 100644 --- a/whisper/mailserver/mailserver.go +++ b/whisper/mailserver/mailserver.go @@ -17,7 +17,6 @@ package mailserver import ( - "bytes" "encoding/binary" "fmt" @@ -175,7 +174,10 @@ func (s *WMailServer) validateRequest(peerID []byte, request *whisper.Envelope) if len(src)-len(peerID) == 1 { src = src[1:] } - if !bytes.Equal(peerID, src) { + + // if you want to check the signature, you can do it here. e.g.: + // if !bytes.Equal(peerID, src) { + if src == nil { log.Warn(fmt.Sprintf("Wrong signature of p2p request")) return false, 0, 0, topic } From 5e30a5f66e126d35c00276abffe32d99182a1bc7 Mon Sep 17 00:00:00 2001 From: Vlad Date: Tue, 27 Feb 2018 15:52:10 +0100 Subject: [PATCH 10/33] whisper: test fixed --- whisper/mailserver/server_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/whisper/mailserver/server_test.go b/whisper/mailserver/server_test.go index c8e0a553a0..aeb317e597 100644 --- a/whisper/mailserver/server_test.go +++ b/whisper/mailserver/server_test.go @@ -167,7 +167,8 @@ func singleRequest(t *testing.T, server *WMailServer, env *whisper.Envelope, p * src[0]++ ok, lower, upper, topic = server.validateRequest(src, request) - if ok { + if !ok { + // request should be valid regardless of signature t.Fatalf("request validation false positive, seed: %d (lower: %d, upper: %d).", seed, lower, upper) } } From 17b0e226d3d6b3829a82ee1c142bd125cc9a0109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 27 Feb 2018 18:25:56 +0200 Subject: [PATCH 11/33] travis, build, consensus: drop support for Go 1.7 --- .travis.yml | 11 ----- build/ci.go | 6 +-- consensus/ethash/algorithm.go | 43 ++++++++++++++++ consensus/ethash/algorithm_go1.7.go | 47 ------------------ consensus/ethash/algorithm_go1.8.go | 63 ------------------------ consensus/ethash/algorithm_go1.8_test.go | 37 -------------- consensus/ethash/algorithm_test.go | 16 ++++++ consensus/ethash/consensus.go | 10 +--- 8 files changed, 64 insertions(+), 169 deletions(-) delete mode 100644 consensus/ethash/algorithm_go1.7.go delete mode 100644 consensus/ethash/algorithm_go1.8.go delete mode 100644 consensus/ethash/algorithm_go1.8_test.go diff --git a/.travis.yml b/.travis.yml index 3941fa785b..a76a78954d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,17 +3,6 @@ go_import_path: github.com/ethereum/go-ethereum sudo: false matrix: include: - - os: linux - dist: trusty - sudo: required - go: 1.7.x - script: - - sudo modprobe fuse - - sudo chmod 666 /dev/fuse - - sudo chown root:$USER /etc/fuse.conf - - go run build/ci.go install - - go run build/ci.go test -coverage - - os: linux dist: trusty sudo: required diff --git a/build/ci.go b/build/ci.go index 544483c42c..24b58c1ae4 100644 --- a/build/ci.go +++ b/build/ci.go @@ -182,13 +182,13 @@ func doInstall(cmdline []string) { // Check Go version. People regularly open issues about compilation // failure with outdated Go. This should save them the trouble. if !strings.Contains(runtime.Version(), "devel") { - // Figure out the minor version number since we can't textually compare (1.10 < 1.7) + // Figure out the minor version number since we can't textually compare (1.10 < 1.8) var minor int fmt.Sscanf(strings.TrimPrefix(runtime.Version(), "go1."), "%d", &minor) - if minor < 7 { + if minor < 8 { log.Println("You have Go version", runtime.Version()) - log.Println("go-ethereum requires at least Go version 1.7 and cannot") + log.Println("go-ethereum requires at least Go version 1.8 and cannot") log.Println("be compiled with an earlier version. Please upgrade your Go installation.") os.Exit(1) } diff --git a/consensus/ethash/algorithm.go b/consensus/ethash/algorithm.go index 10767bb312..905a7b1ea7 100644 --- a/consensus/ethash/algorithm.go +++ b/consensus/ethash/algorithm.go @@ -19,6 +19,7 @@ package ethash import ( "encoding/binary" "hash" + "math/big" "reflect" "runtime" "sync" @@ -47,6 +48,48 @@ const ( loopAccesses = 64 // Number of accesses in hashimoto loop ) +// cacheSize returns the size of the ethash verification cache that belongs to a certain +// block number. +func cacheSize(block uint64) uint64 { + epoch := int(block / epochLength) + if epoch < maxEpoch { + return cacheSizes[epoch] + } + return calcCacheSize(epoch) +} + +// calcCacheSize calculates the cache size for epoch. The cache size grows linearly, +// however, we always take the highest prime below the linearly growing threshold in order +// to reduce the risk of accidental regularities leading to cyclic behavior. +func calcCacheSize(epoch int) uint64 { + size := cacheInitBytes + cacheGrowthBytes*uint64(epoch) - hashBytes + for !new(big.Int).SetUint64(size / hashBytes).ProbablyPrime(1) { // Always accurate for n < 2^64 + size -= 2 * hashBytes + } + return size +} + +// datasetSize returns the size of the ethash mining dataset that belongs to a certain +// block number. +func datasetSize(block uint64) uint64 { + epoch := int(block / epochLength) + if epoch < maxEpoch { + return datasetSizes[epoch] + } + return calcDatasetSize(epoch) +} + +// calcDatasetSize calculates the dataset size for epoch. The dataset size grows linearly, +// however, we always take the highest prime below the linearly growing threshold in order +// to reduce the risk of accidental regularities leading to cyclic behavior. +func calcDatasetSize(epoch int) uint64 { + size := datasetInitBytes + datasetGrowthBytes*uint64(epoch) - mixBytes + for !new(big.Int).SetUint64(size / mixBytes).ProbablyPrime(1) { // Always accurate for n < 2^64 + size -= 2 * mixBytes + } + return size +} + // hasher is a repetitive hasher allowing the same hash data structures to be // reused between hash runs instead of requiring new ones to be created. type hasher func(dest []byte, data []byte) diff --git a/consensus/ethash/algorithm_go1.7.go b/consensus/ethash/algorithm_go1.7.go deleted file mode 100644 index c7f7f48e41..0000000000 --- a/consensus/ethash/algorithm_go1.7.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// +build !go1.8 - -package ethash - -// cacheSize calculates and returns the size of the ethash verification cache that -// belongs to a certain block number. The cache size grows linearly, however, we -// always take the highest prime below the linearly growing threshold in order to -// reduce the risk of accidental regularities leading to cyclic behavior. -func cacheSize(block uint64) uint64 { - // If we have a pre-generated value, use that - epoch := int(block / epochLength) - if epoch < maxEpoch { - return cacheSizes[epoch] - } - // We don't have a way to verify primes fast before Go 1.8 - panic("fast prime testing unsupported in Go < 1.8") -} - -// datasetSize calculates and returns the size of the ethash mining dataset that -// belongs to a certain block number. The dataset size grows linearly, however, we -// always take the highest prime below the linearly growing threshold in order to -// reduce the risk of accidental regularities leading to cyclic behavior. -func datasetSize(block uint64) uint64 { - // If we have a pre-generated value, use that - epoch := int(block / epochLength) - if epoch < maxEpoch { - return datasetSizes[epoch] - } - // We don't have a way to verify primes fast before Go 1.8 - panic("fast prime testing unsupported in Go < 1.8") -} diff --git a/consensus/ethash/algorithm_go1.8.go b/consensus/ethash/algorithm_go1.8.go deleted file mode 100644 index 975fdffe51..0000000000 --- a/consensus/ethash/algorithm_go1.8.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// +build go1.8 - -package ethash - -import "math/big" - -// cacheSize returns the size of the ethash verification cache that belongs to a certain -// block number. -func cacheSize(block uint64) uint64 { - epoch := int(block / epochLength) - if epoch < maxEpoch { - return cacheSizes[epoch] - } - return calcCacheSize(epoch) -} - -// calcCacheSize calculates the cache size for epoch. The cache size grows linearly, -// however, we always take the highest prime below the linearly growing threshold in order -// to reduce the risk of accidental regularities leading to cyclic behavior. -func calcCacheSize(epoch int) uint64 { - size := cacheInitBytes + cacheGrowthBytes*uint64(epoch) - hashBytes - for !new(big.Int).SetUint64(size / hashBytes).ProbablyPrime(1) { // Always accurate for n < 2^64 - size -= 2 * hashBytes - } - return size -} - -// datasetSize returns the size of the ethash mining dataset that belongs to a certain -// block number. -func datasetSize(block uint64) uint64 { - epoch := int(block / epochLength) - if epoch < maxEpoch { - return datasetSizes[epoch] - } - return calcDatasetSize(epoch) -} - -// calcDatasetSize calculates the dataset size for epoch. The dataset size grows linearly, -// however, we always take the highest prime below the linearly growing threshold in order -// to reduce the risk of accidental regularities leading to cyclic behavior. -func calcDatasetSize(epoch int) uint64 { - size := datasetInitBytes + datasetGrowthBytes*uint64(epoch) - mixBytes - for !new(big.Int).SetUint64(size / mixBytes).ProbablyPrime(1) { // Always accurate for n < 2^64 - size -= 2 * mixBytes - } - return size -} diff --git a/consensus/ethash/algorithm_go1.8_test.go b/consensus/ethash/algorithm_go1.8_test.go deleted file mode 100644 index 6648bd6a97..0000000000 --- a/consensus/ethash/algorithm_go1.8_test.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// +build go1.8 - -package ethash - -import "testing" - -// Tests whether the dataset size calculator works correctly by cross checking the -// hard coded lookup table with the value generated by it. -func TestSizeCalculations(t *testing.T) { - // Verify all the cache and dataset sizes from the lookup table. - for epoch, want := range cacheSizes { - if size := calcCacheSize(epoch); size != want { - t.Errorf("cache %d: cache size mismatch: have %d, want %d", epoch, size, want) - } - } - for epoch, want := range datasetSizes { - if size := calcDatasetSize(epoch); size != want { - t.Errorf("dataset %d: dataset size mismatch: have %d, want %d", epoch, size, want) - } - } -} diff --git a/consensus/ethash/algorithm_test.go b/consensus/ethash/algorithm_test.go index a54f3b582c..841e39233f 100644 --- a/consensus/ethash/algorithm_test.go +++ b/consensus/ethash/algorithm_test.go @@ -30,6 +30,22 @@ import ( "github.com/ethereum/go-ethereum/core/types" ) +// Tests whether the dataset size calculator works correctly by cross checking the +// hard coded lookup table with the value generated by it. +func TestSizeCalculations(t *testing.T) { + // Verify all the cache and dataset sizes from the lookup table. + for epoch, want := range cacheSizes { + if size := calcCacheSize(epoch); size != want { + t.Errorf("cache %d: cache size mismatch: have %d, want %d", epoch, size, want) + } + } + for epoch, want := range datasetSizes { + if size := calcDatasetSize(epoch); size != want { + t.Errorf("dataset %d: dataset size mismatch: have %d, want %d", epoch, size, want) + } + } +} + // Tests that verification caches can be correctly generated. func TestCacheGeneration(t *testing.T) { tests := []struct { diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 92a23d4a4d..3f860bf473 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -53,7 +53,6 @@ var ( errDuplicateUncle = errors.New("duplicate uncle") errUncleIsAncestor = errors.New("uncle is ancestor") errDanglingUncle = errors.New("uncle's parent is not ancestor") - errNonceOutOfRange = errors.New("nonce out of range") errInvalidDifficulty = errors.New("non-positive difficulty") errInvalidMixDigest = errors.New("invalid mix digest") errInvalidPoW = errors.New("invalid proof-of-work") @@ -474,18 +473,13 @@ func (ethash *Ethash) VerifySeal(chain consensus.ChainReader, header *types.Head if ethash.shared != nil { return ethash.shared.VerifySeal(chain, header) } - // Sanity check that the block number is below the lookup table size (60M blocks) - number := header.Number.Uint64() - if number/epochLength >= maxEpoch { - // Go < 1.7 cannot calculate new cache/dataset sizes (no fast prime check) - return errNonceOutOfRange - } // Ensure that we have a valid difficulty for the block if header.Difficulty.Sign() <= 0 { return errInvalidDifficulty } - // Recompute the digest and PoW value and verify against the header + number := header.Number.Uint64() + cache := ethash.cache(number) size := datasetSize(number) if ethash.config.PowMode == ModeTest { From 014d8d98370abf770b89f7987a92999133e3d4ea Mon Sep 17 00:00:00 2001 From: Vlad Date: Tue, 27 Feb 2018 21:16:15 +0100 Subject: [PATCH 12/33] whisper: message filtering optimized --- whisper/whisperv6/filter.go | 66 ++++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 11 deletions(-) diff --git a/whisper/whisperv6/filter.go b/whisper/whisperv6/filter.go index eb0c65fa3b..75c31209c9 100644 --- a/whisper/whisperv6/filter.go +++ b/whisper/whisperv6/filter.go @@ -35,6 +35,7 @@ type Filter struct { PoW float64 // Proof of work as described in the Whisper spec AllowP2P bool // Indicates whether this filter is interested in direct peer-to-peer messages SymKeyHash common.Hash // The Keccak256Hash of the symmetric key, needed for optimization + id string // unique identifier Messages map[common.Hash]*ReceivedMessage mutex sync.RWMutex @@ -42,16 +43,21 @@ type Filter struct { // Filters represents a collection of filters type Filters struct { - watchers map[string]*Filter - whisper *Whisper - mutex sync.RWMutex + watchers map[string]*Filter + topicMatcher map[TopicType]map[*Filter]struct{} + allTopicsMatcher map[*Filter]struct{} + + whisper *Whisper + mutex sync.RWMutex } // NewFilters returns a newly created filter collection func NewFilters(w *Whisper) *Filters { return &Filters{ - watchers: make(map[string]*Filter), - whisper: w, + watchers: make(map[string]*Filter), + topicMatcher: make(map[TopicType]map[*Filter]struct{}), + allTopicsMatcher: make(map[*Filter]struct{}), + whisper: w, } } @@ -81,7 +87,9 @@ func (fs *Filters) Install(watcher *Filter) (string, error) { watcher.SymKeyHash = crypto.Keccak256Hash(watcher.KeySym) } + watcher.id = id fs.watchers[id] = watcher + fs.addTopicMatcher(watcher) return id, err } @@ -91,12 +99,49 @@ func (fs *Filters) Uninstall(id string) bool { fs.mutex.Lock() defer fs.mutex.Unlock() if fs.watchers[id] != nil { + fs.removeFromTopicMatchers(fs.watchers[id]) delete(fs.watchers, id) return true } return false } +// addTopicMatcher adds a filter to the topic matchers +func (fs *Filters) addTopicMatcher(watcher *Filter) { + if len(watcher.Topics) == 0 { + fs.allTopicsMatcher[watcher] = struct{}{} + } else { + for _, t := range watcher.Topics { + topic := BytesToTopic(t) + if fs.topicMatcher[topic] == nil { + fs.topicMatcher[topic] = make(map[*Filter]struct{}) + } + fs.topicMatcher[topic][watcher] = struct{}{} + } + } +} + +// removeFromTopicMatchers removes a filter from the topic matchers +func (fs *Filters) removeFromTopicMatchers(watcher *Filter) { + delete(fs.allTopicsMatcher, watcher) + for _, topic := range watcher.Topics { + delete(fs.topicMatcher[BytesToTopic(topic)], watcher) + } +} + +// getWatchersByTopic returns a slice containing the filters that +// match a specific topic +func (fs *Filters) getWatchersByTopic(topic TopicType) []*Filter { + res := make([]*Filter, 0, len(fs.allTopicsMatcher)) + for watcher, _ := range fs.allTopicsMatcher { + res = append(res, watcher) + } + for watcher, _ := range fs.topicMatcher[topic] { + res = append(res, watcher) + } + return res +} + // Get returns a filter from the collection with a specific ID func (fs *Filters) Get(id string) *Filter { fs.mutex.RLock() @@ -112,11 +157,10 @@ func (fs *Filters) NotifyWatchers(env *Envelope, p2pMessage bool) { fs.mutex.RLock() defer fs.mutex.RUnlock() - i := -1 // only used for logging info - for _, watcher := range fs.watchers { - i++ + candidates := fs.getWatchersByTopic(env.Topic) + for _, watcher := range candidates { if p2pMessage && !watcher.AllowP2P { - log.Trace(fmt.Sprintf("msg [%x], filter [%d]: p2p messages are not allowed", env.Hash(), i)) + log.Trace(fmt.Sprintf("msg [%x], filter [%s]: p2p messages are not allowed", env.Hash(), watcher.id)) continue } @@ -128,10 +172,10 @@ func (fs *Filters) NotifyWatchers(env *Envelope, p2pMessage bool) { if match { msg = env.Open(watcher) if msg == nil { - log.Trace("processing message: failed to open", "message", env.Hash().Hex(), "filter", i) + log.Trace("processing message: failed to open", "message", env.Hash().Hex(), "filter", watcher.id) } } else { - log.Trace("processing message: does not match", "message", env.Hash().Hex(), "filter", i) + log.Trace("processing message: does not match", "message", env.Hash().Hex(), "filter", watcher.id) } } From c733792be4b49ec47c5307b1a3b8fa116ea16933 Mon Sep 17 00:00:00 2001 From: Vlad Date: Tue, 27 Feb 2018 23:38:20 +0100 Subject: [PATCH 13/33] whsiper: refactoring --- whisper/whisperv6/filter.go | 29 ++++++----------------------- whisper/whisperv6/filter_test.go | 25 +++++++++++++++---------- whisper/whisperv6/whisper.go | 18 ------------------ whisper/whisperv6/whisper_test.go | 11 +---------- 4 files changed, 22 insertions(+), 61 deletions(-) diff --git a/whisper/whisperv6/filter.go b/whisper/whisperv6/filter.go index 75c31209c9..d7b71795de 100644 --- a/whisper/whisperv6/filter.go +++ b/whisper/whisperv6/filter.go @@ -238,16 +238,17 @@ func (f *Filter) Retrieve() (all []*ReceivedMessage) { // MatchMessage checks if the filter matches an already decrypted // message (i.e. a Message that has already been handled by -// MatchEnvelope when checked by a previous filter) +// MatchEnvelope when checked by a previous filter). +// Topics are not checked here, since this is done by topic matchers. func (f *Filter) MatchMessage(msg *ReceivedMessage) bool { if f.PoW > 0 && msg.PoW < f.PoW { return false } if f.expectsAsymmetricEncryption() && msg.isAsymmetricEncryption() { - return IsPubKeyEqual(&f.KeyAsym.PublicKey, msg.Dst) && f.MatchTopic(msg.Topic) + return IsPubKeyEqual(&f.KeyAsym.PublicKey, msg.Dst) } else if f.expectsSymmetricEncryption() && msg.isSymmetricEncryption() { - return f.SymKeyHash == msg.SymKeyHash && f.MatchTopic(msg.Topic) + return f.SymKeyHash == msg.SymKeyHash } return false } @@ -255,27 +256,9 @@ func (f *Filter) MatchMessage(msg *ReceivedMessage) bool { // MatchEnvelope checks if it's worth decrypting the message. If // it returns `true`, client code is expected to attempt decrypting // the message and subsequently call MatchMessage. +// Topics are not checked here, since this is done by topic matchers. func (f *Filter) MatchEnvelope(envelope *Envelope) bool { - if f.PoW > 0 && envelope.pow < f.PoW { - return false - } - - return f.MatchTopic(envelope.Topic) -} - -// MatchTopic checks that the filter captures a given topic. -func (f *Filter) MatchTopic(topic TopicType) bool { - if len(f.Topics) == 0 { - // any topic matches - return true - } - - for _, bt := range f.Topics { - if matchSingleTopic(topic, bt) { - return true - } - } - return false + return f.PoW <= 0 || envelope.pow >= f.PoW } func matchSingleTopic(topic TopicType, bt []byte) bool { diff --git a/whisper/whisperv6/filter_test.go b/whisper/whisperv6/filter_test.go index e7230ef388..491e137bdb 100644 --- a/whisper/whisperv6/filter_test.go +++ b/whisper/whisperv6/filter_test.go @@ -303,9 +303,8 @@ func TestMatchEnvelope(t *testing.T) { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - params.Topic[0] = 0xFF // ensure mismatch + params.Topic[0] = 0xFF // topic mismatch - // mismatch with pseudo-random data msg, err := NewSentMessage(params) if err != nil { t.Fatalf("failed to create new message with seed %d: %s.", seed, err) @@ -315,11 +314,13 @@ func TestMatchEnvelope(t *testing.T) { t.Fatalf("failed Wrap with seed %d: %s.", seed, err) } match := fsym.MatchEnvelope(env) - if match { + if !match { + // topic mismatch should have no affect, as topics are handled by topic matchers t.Fatalf("failed MatchEnvelope symmetric with seed %d.", seed) } match = fasym.MatchEnvelope(env) - if match { + if !match { + // topic mismatch should have no affect, as topics are handled by topic matchers t.Fatalf("failed MatchEnvelope asymmetric with seed %d.", seed) } @@ -396,7 +397,7 @@ func TestMatchEnvelope(t *testing.T) { // asymmetric + matching topic: match fasym.Topics[i] = fasym.Topics[i+1] match = fasym.MatchEnvelope(env) - if match { + if !match { t.Fatalf("failed MatchEnvelope(asymmetric + matching topic) with seed %d.", seed) } @@ -431,7 +432,8 @@ func TestMatchEnvelope(t *testing.T) { // filter with topic + envelope without topic: mismatch fasym.Topics = fsym.Topics match = fasym.MatchEnvelope(env) - if match { + if !match { + // topic mismatch should have no affect, as topics are handled by topic matchers t.Fatalf("failed MatchEnvelope(filter without topic + envelope without topic) with seed %d.", seed) } } @@ -487,7 +489,8 @@ func TestMatchMessageSym(t *testing.T) { // topic mismatch f.Topics[index][0]++ - if f.MatchMessage(msg) { + if !f.MatchMessage(msg) { + // topic mismatch should have no affect, as topics are handled by topic matchers t.Fatalf("failed MatchEnvelope(topic mismatch) with seed %d.", seed) } f.Topics[index][0]-- @@ -580,7 +583,8 @@ func TestMatchMessageAsym(t *testing.T) { // topic mismatch f.Topics[index][0]++ - if f.MatchMessage(msg) { + if !f.MatchMessage(msg) { + // topic mismatch should have no affect, as topics are handled by topic matchers t.Fatalf("failed MatchEnvelope(topic mismatch) with seed %d.", seed) } f.Topics[index][0]-- @@ -829,8 +833,9 @@ func TestVariableTopics(t *testing.T) { f.Topics[i][lastTopicByte]++ match = f.MatchEnvelope(env) - if match { - t.Fatalf("MatchEnvelope symmetric with seed %d, step %d: false positive.", seed, i) + if !match { + // topic mismatch should have no affect, as topics are handled by topic matchers + t.Fatalf("MatchEnvelope symmetric with seed %d, step %d.", seed, i) } } } diff --git a/whisper/whisperv6/whisper.go b/whisper/whisperv6/whisper.go index b0b601f032..81f59b3cc7 100644 --- a/whisper/whisperv6/whisper.go +++ b/whisper/whisperv6/whisper.go @@ -928,24 +928,6 @@ func (whisper *Whisper) Envelopes() []*Envelope { return all } -// Messages iterates through all currently floating envelopes -// and retrieves all the messages, that this filter could decrypt. -func (whisper *Whisper) Messages(id string) []*ReceivedMessage { - result := make([]*ReceivedMessage, 0) - whisper.poolMu.RLock() - defer whisper.poolMu.RUnlock() - - if filter := whisper.filters.Get(id); filter != nil { - for _, env := range whisper.envelopes { - msg := filter.processEnvelope(env) - if msg != nil { - result = append(result, msg) - } - } - } - return result -} - // isEnvelopeCached checks if envelope with specific hash has already been received and cached. func (whisper *Whisper) isEnvelopeCached(hash common.Hash) bool { whisper.poolMu.Lock() diff --git a/whisper/whisperv6/whisper_test.go b/whisper/whisperv6/whisper_test.go index 99e5f0bbb4..3a219cd133 100644 --- a/whisper/whisperv6/whisper_test.go +++ b/whisper/whisperv6/whisper_test.go @@ -75,10 +75,6 @@ func TestWhisperBasic(t *testing.T) { if len(mail) != 0 { t.Fatalf("failed w.Envelopes().") } - m := w.Messages("non-existent") - if len(m) != 0 { - t.Fatalf("failed w.Messages.") - } derived := pbkdf2.Key([]byte(peerID), nil, 65356, aesKeyLength, sha256.New) if !validateDataIntegrity(derived, aesKeyLength) { @@ -593,7 +589,7 @@ func TestCustomization(t *testing.T) { } // check w.messages() - id, err := w.Subscribe(f) + _, err = w.Subscribe(f) if err != nil { t.Fatalf("failed subscribe with seed %d: %s.", seed, err) } @@ -602,11 +598,6 @@ func TestCustomization(t *testing.T) { if len(mail) > 0 { t.Fatalf("received premature mail") } - - mail = w.Messages(id) - if len(mail) != 2 { - t.Fatalf("failed to get whisper messages") - } } func TestSymmetricSendCycle(t *testing.T) { From a69cb3b4ff55c6ea2bafb7bd0ab0ff4afafb148f Mon Sep 17 00:00:00 2001 From: Vlad Date: Wed, 28 Feb 2018 00:39:38 +0100 Subject: [PATCH 14/33] whisper: comment updated --- whisper/mailserver/server_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/whisper/mailserver/server_test.go b/whisper/mailserver/server_test.go index aeb317e597..63864c1e6b 100644 --- a/whisper/mailserver/server_test.go +++ b/whisper/mailserver/server_test.go @@ -169,7 +169,7 @@ func singleRequest(t *testing.T, server *WMailServer, env *whisper.Envelope, p * ok, lower, upper, topic = server.validateRequest(src, request) if !ok { // request should be valid regardless of signature - t.Fatalf("request validation false positive, seed: %d (lower: %d, upper: %d).", seed, lower, upper) + t.Fatalf("request validation false negative, seed: %d (lower: %d, upper: %d).", seed, lower, upper) } } From cf52d5c91f4f9c583eecca63bce52c7df6dd2169 Mon Sep 17 00:00:00 2001 From: b00ris Date: Wed, 28 Feb 2018 09:50:36 +0300 Subject: [PATCH 15/33] whisper: fixed datarace --- whisper/whisperv6/peer_test.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/whisper/whisperv6/peer_test.go b/whisper/whisperv6/peer_test.go index 9ce5eed8bc..3d3dbd88aa 100644 --- a/whisper/whisperv6/peer_test.go +++ b/whisper/whisperv6/peer_test.go @@ -23,6 +23,7 @@ import ( mrand "math/rand" "net" "sync" + "sync/atomic" "testing" "time" @@ -71,7 +72,7 @@ var keys = []string{ } type TestData struct { - started int + started int64 counter [NumNodes]int mutex sync.RWMutex } @@ -240,9 +241,7 @@ func startServer(t *testing.T, s *p2p.Server) { t.Fatalf("failed to start the fisrt server.") } - result.mutex.Lock() - defer result.mutex.Unlock() - result.started++ + atomic.AddInt64(&result.started, 1) } func stopServers() { @@ -472,7 +471,10 @@ func checkPowExchange(t *testing.T) { func checkBloomFilterExchangeOnce(t *testing.T, mustPass bool) bool { for i, node := range nodes { for peer := range node.shh.peers { - if !bytes.Equal(peer.bloomFilter, masterBloomFilter) { + peer.bloomMu.Lock() + eqals := bytes.Equal(peer.bloomFilter, masterBloomFilter) + peer.bloomMu.Unlock() + if !eqals { if mustPass { t.Fatalf("node %d: failed to exchange bloom filter requirement in round %d. \n%x expected \n%x got", i, round, masterBloomFilter, peer.bloomFilter) @@ -500,11 +502,13 @@ func checkBloomFilterExchange(t *testing.T) { func waitForServersToStart(t *testing.T) { const iterations = 200 + var started int64 for j := 0; j < iterations; j++ { time.Sleep(50 * time.Millisecond) - if result.started == NumNodes { + started = atomic.LoadInt64(&result.started) + if started == NumNodes { return } } - t.Fatalf("Failed to start all the servers, running: %d", result.started) + t.Fatalf("Failed to start all the servers, running: %d", started) } From 98ec5e50115b47670c1f3ffb2cc890ce4838c4d6 Mon Sep 17 00:00:00 2001 From: Mark Rushakoff Date: Wed, 28 Feb 2018 02:20:07 -0800 Subject: [PATCH 16/33] core/asm: rename isAlphaNumeric to isLetter (#16212) The function would return false for numbers, so isLetter is a more accurate description of the behavior. --- core/asm/lexer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/asm/lexer.go b/core/asm/lexer.go index a34b2cbd8f..4054999504 100644 --- a/core/asm/lexer.go +++ b/core/asm/lexer.go @@ -206,7 +206,7 @@ func lexLine(l *lexer) stateFn { return lexComment case isSpace(r): l.ignore() - case isAlphaNumeric(r) || r == '_': + case isLetter(r) || r == '_': return lexElement case isNumber(r): return lexNumber @@ -278,7 +278,7 @@ func lexElement(l *lexer) stateFn { return lexLine } -func isAlphaNumeric(t rune) bool { +func isLetter(t rune) bool { return unicode.IsLetter(t) } From ba7b384019de17a72a8c24e3aaac8a98ff30dcc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 28 Feb 2018 12:40:15 +0200 Subject: [PATCH 17/33] internal/ethapi: fix getTransactionReceipt --- internal/ethapi/api.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index d021b127c5..20a060e727 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1035,14 +1035,14 @@ func (s *PublicTransactionPoolAPI) GetRawTransactionByHash(ctx context.Context, func (s *PublicTransactionPoolAPI) GetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) { tx, blockHash, blockNumber, index := core.GetTransaction(s.b.ChainDb(), hash) if tx == nil { - return nil, errors.New("unknown transaction") + return nil, nil } receipts, err := s.b.GetReceipts(ctx, blockHash) if err != nil { return nil, err } if len(receipts) <= int(index) { - return nil, errors.New("unknown receipt") + return nil, nil } receipt := receipts[index] From 62c239f608ec2ec3721c33d5755ca2287c71f085 Mon Sep 17 00:00:00 2001 From: b00ris Date: Wed, 28 Feb 2018 14:38:42 +0300 Subject: [PATCH 18/33] whisper: fix typo --- whisper/whisperv6/peer_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/whisper/whisperv6/peer_test.go b/whisper/whisperv6/peer_test.go index 3d3dbd88aa..6d7754d790 100644 --- a/whisper/whisperv6/peer_test.go +++ b/whisper/whisperv6/peer_test.go @@ -472,9 +472,9 @@ func checkBloomFilterExchangeOnce(t *testing.T, mustPass bool) bool { for i, node := range nodes { for peer := range node.shh.peers { peer.bloomMu.Lock() - eqals := bytes.Equal(peer.bloomFilter, masterBloomFilter) + equals := bytes.Equal(peer.bloomFilter, masterBloomFilter) peer.bloomMu.Unlock() - if !eqals { + if !equals { if mustPass { t.Fatalf("node %d: failed to exchange bloom filter requirement in round %d. \n%x expected \n%x got", i, round, masterBloomFilter, peer.bloomFilter) From d24d10a764457937f1bda72053c82d98ac95dd7a Mon Sep 17 00:00:00 2001 From: Vlad Date: Wed, 28 Feb 2018 15:05:35 +0100 Subject: [PATCH 19/33] whisper: style fixes --- whisper/whisperv6/filter.go | 15 +++++++++------ whisper/whisperv6/filter_test.go | 12 +----------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/whisper/whisperv6/filter.go b/whisper/whisperv6/filter.go index d7b71795de..e4171f85c9 100644 --- a/whisper/whisperv6/filter.go +++ b/whisper/whisperv6/filter.go @@ -43,9 +43,10 @@ type Filter struct { // Filters represents a collection of filters type Filters struct { - watchers map[string]*Filter - topicMatcher map[TopicType]map[*Filter]struct{} - allTopicsMatcher map[*Filter]struct{} + watchers map[string]*Filter + + topicMatcher map[TopicType]map[*Filter]struct{} // map a topic to the filters that are interested in being notified when a message matches that topic + allTopicsMatcher map[*Filter]struct{} // list all the filters that will be notified of a new message, no matter what its topic is whisper *Whisper mutex sync.RWMutex @@ -106,7 +107,9 @@ func (fs *Filters) Uninstall(id string) bool { return false } -// addTopicMatcher adds a filter to the topic matchers +// addTopicMatcher adds a filter to the topic matchers. +// If the filter's Topics array is empty, it will be tried on every topic. +// Otherwise, it will be tried on the topics specified. func (fs *Filters) addTopicMatcher(watcher *Filter) { if len(watcher.Topics) == 0 { fs.allTopicsMatcher[watcher] = struct{}{} @@ -133,10 +136,10 @@ func (fs *Filters) removeFromTopicMatchers(watcher *Filter) { // match a specific topic func (fs *Filters) getWatchersByTopic(topic TopicType) []*Filter { res := make([]*Filter, 0, len(fs.allTopicsMatcher)) - for watcher, _ := range fs.allTopicsMatcher { + for watcher := range fs.allTopicsMatcher { res = append(res, watcher) } - for watcher, _ := range fs.topicMatcher[topic] { + for watcher := range fs.topicMatcher[topic] { res = append(res, watcher) } return res diff --git a/whisper/whisperv6/filter_test.go b/whisper/whisperv6/filter_test.go index 491e137bdb..0bb7986c39 100644 --- a/whisper/whisperv6/filter_test.go +++ b/whisper/whisperv6/filter_test.go @@ -313,16 +313,6 @@ func TestMatchEnvelope(t *testing.T) { if err != nil { t.Fatalf("failed Wrap with seed %d: %s.", seed, err) } - match := fsym.MatchEnvelope(env) - if !match { - // topic mismatch should have no affect, as topics are handled by topic matchers - t.Fatalf("failed MatchEnvelope symmetric with seed %d.", seed) - } - match = fasym.MatchEnvelope(env) - if !match { - // topic mismatch should have no affect, as topics are handled by topic matchers - t.Fatalf("failed MatchEnvelope asymmetric with seed %d.", seed) - } // encrypt symmetrically i := mrand.Int() % 4 @@ -338,7 +328,7 @@ func TestMatchEnvelope(t *testing.T) { } // symmetric + matching topic: match - match = fsym.MatchEnvelope(env) + match := fsym.MatchEnvelope(env) if !match { t.Fatalf("failed MatchEnvelope() symmetric with seed %d.", seed) } From 5a150e1b7724c91009a237ab0879cd64844b390d Mon Sep 17 00:00:00 2001 From: gluk256 Date: Thu, 1 Mar 2018 09:34:46 +0100 Subject: [PATCH 20/33] whisper: serious security issue fixed (#16219) The diagnostic tool was saving the unencrypted version of the messages, which is an obvious security flaw. As of this commit: * encrypted messages saved instead of plain text. * all messages are stored, even that created by the user of wnode. --- cmd/wnode/main.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/cmd/wnode/main.go b/cmd/wnode/main.go index 0f86adb81c..f8606bf824 100644 --- a/cmd/wnode/main.go +++ b/cmd/wnode/main.go @@ -594,19 +594,22 @@ func writeMessageToFile(dir string, msg *whisper.ReceivedMessage) { address = crypto.PubkeyToAddress(*msg.Src) } - if whisper.IsPubKeyEqual(msg.Src, &asymKey.PublicKey) { - // message from myself: don't save, only report - fmt.Printf("\n%s <%x>: message received: '%s'\n", timestamp, address, name) - } else if len(dir) > 0 { + // this is a sample code; uncomment if you don't want to save your own messages. + //if whisper.IsPubKeyEqual(msg.Src, &asymKey.PublicKey) { + // fmt.Printf("\n%s <%x>: message from myself received, not saved: '%s'\n", timestamp, address, name) + // return + //} + + if len(dir) > 0 { fullpath := filepath.Join(dir, name) - err := ioutil.WriteFile(fullpath, msg.Payload, 0644) + err := ioutil.WriteFile(fullpath, msg.Raw, 0644) if err != nil { fmt.Printf("\n%s {%x}: message received but not saved: %s\n", timestamp, address, err) } else { - fmt.Printf("\n%s {%x}: message received and saved as '%s' (%d bytes)\n", timestamp, address, name, len(msg.Payload)) + fmt.Printf("\n%s {%x}: message received and saved as '%s' (%d bytes)\n", timestamp, address, name, len(msg.Raw)) } } else { - fmt.Printf("\n%s {%x}: big message received (%d bytes), but not saved: %s\n", timestamp, address, len(msg.Payload), name) + fmt.Printf("\n%s {%x}: message received (%d bytes), but not saved: %s\n", timestamp, address, len(msg.Raw), name) } } From ee75a90ab41fd9a2e5676a2371e529ac2908befa Mon Sep 17 00:00:00 2001 From: Vlad Date: Thu, 1 Mar 2018 16:04:09 +0100 Subject: [PATCH 21/33] whisper: topics replaced by bloom filters --- cmd/wnode/main.go | 40 +++++++++++++++++++++++----- whisper/mailserver/mailserver.go | 44 ++++++++++++++++--------------- whisper/mailserver/server_test.go | 20 ++++++++------ whisper/whisperv6/doc.go | 2 +- whisper/whisperv6/envelope.go | 2 +- whisper/whisperv6/peer.go | 14 +++++----- whisper/whisperv6/peer_test.go | 6 ++--- whisper/whisperv6/whisper.go | 22 ++++++++-------- whisper/whisperv6/whisper_test.go | 18 ++++++------- 9 files changed, 100 insertions(+), 68 deletions(-) diff --git a/cmd/wnode/main.go b/cmd/wnode/main.go index f8606bf824..ccfdd46265 100644 --- a/cmd/wnode/main.go +++ b/cmd/wnode/main.go @@ -22,6 +22,7 @@ package main import ( "bufio" "crypto/ecdsa" + crand "crypto/rand" "crypto/sha512" "encoding/binary" "encoding/hex" @@ -48,6 +49,7 @@ import ( ) const quitCommand = "~Q" +const entropySize = 32 // singletons var ( @@ -55,6 +57,7 @@ var ( shh *whisper.Whisper done chan struct{} mailServer mailserver.WMailServer + entropy [entropySize]byte input = bufio.NewReader(os.Stdin) ) @@ -274,6 +277,11 @@ func initialize() { TrustedNodes: peers, }, } + + _, err = crand.Read(entropy[:]) + if err != nil { + utils.Fatalf("crypto/rand failed: %s", err) + } } func startServer() { @@ -614,10 +622,10 @@ func writeMessageToFile(dir string, msg *whisper.ReceivedMessage) { } func requestExpiredMessagesLoop() { - var key, peerID []byte + var key, peerID, bloom []byte var timeLow, timeUpp uint32 var t string - var xt, empty whisper.TopicType + var xt whisper.TopicType keyID, err := shh.AddSymKeyFromPassword(msPassword) if err != nil { @@ -640,18 +648,19 @@ func requestExpiredMessagesLoop() { utils.Fatalf("Failed to parse the topic: %s", err) } xt = whisper.BytesToTopic(x) + bloom = whisper.TopicToBloom(xt) + obfuscateBloom(bloom) + } else { + bloom = whisper.MakeFullNodeBloom() } if timeUpp == 0 { timeUpp = 0xFFFFFFFF } - data := make([]byte, 8+whisper.TopicLength) + data := make([]byte, 8, 8+whisper.BloomFilterSize) binary.BigEndian.PutUint32(data, timeLow) binary.BigEndian.PutUint32(data[4:], timeUpp) - copy(data[8:], xt[:]) - if xt == empty { - data = data[:8] - } + data = append(data, bloom...) var params whisper.MessageParams params.PoW = *argServerPoW @@ -685,3 +694,20 @@ func extractIDFromEnode(s string) []byte { } return n.ID[:] } + +// obfuscateBloom adds 16 random bits to the the bloom +// filter, in order to obfuscate the containing topics. +// it does so deterministically within every session. +// despite additional bits, it will match on average +// 32000 times less messages than full node's bloom filter. +func obfuscateBloom(bloom []byte) { + const half = entropySize / 2 + for i := 0; i < half; i++ { + x := int(entropy[i]) + if entropy[half+i] < 128 { + x += 256 + } + + bloom[x/8] = 1 << uint(x%8) // set the bit number X + } +} diff --git a/whisper/mailserver/mailserver.go b/whisper/mailserver/mailserver.go index a889c2bd23..57e6505ad1 100644 --- a/whisper/mailserver/mailserver.go +++ b/whisper/mailserver/mailserver.go @@ -107,17 +107,16 @@ func (s *WMailServer) DeliverMail(peer *whisper.Peer, request *whisper.Envelope) return } - ok, lower, upper, topic := s.validateRequest(peer.ID(), request) + ok, lower, upper, bloom := s.validateRequest(peer.ID(), request) if ok { - s.processRequest(peer, lower, upper, topic) + s.processRequest(peer, lower, upper, bloom) } } -func (s *WMailServer) processRequest(peer *whisper.Peer, lower, upper uint32, topic whisper.TopicType) []*whisper.Envelope { +func (s *WMailServer) processRequest(peer *whisper.Peer, lower, upper uint32, bloom []byte) []*whisper.Envelope { ret := make([]*whisper.Envelope, 0) var err error var zero common.Hash - var empty whisper.TopicType kl := NewDbKey(lower, zero) ku := NewDbKey(upper, zero) i := s.db.NewIterator(&util.Range{Start: kl.raw, Limit: ku.raw}, nil) @@ -130,7 +129,7 @@ func (s *WMailServer) processRequest(peer *whisper.Peer, lower, upper uint32, to log.Error(fmt.Sprintf("RLP decoding failed: %s", err)) } - if topic == empty || envelope.Topic == topic { + if whisper.BloomFilterMatch(bloom, envelope.Bloom()) { if peer == nil { // used for test purposes ret = append(ret, &envelope) @@ -152,22 +151,16 @@ func (s *WMailServer) processRequest(peer *whisper.Peer, lower, upper uint32, to return ret } -func (s *WMailServer) validateRequest(peerID []byte, request *whisper.Envelope) (bool, uint32, uint32, whisper.TopicType) { - var topic whisper.TopicType +func (s *WMailServer) validateRequest(peerID []byte, request *whisper.Envelope) (bool, uint32, uint32, []byte) { if s.pow > 0.0 && request.PoW() < s.pow { - return false, 0, 0, topic + return false, 0, 0, nil } f := whisper.Filter{KeySym: s.key} decrypted := request.Open(&f) if decrypted == nil { log.Warn(fmt.Sprintf("Failed to decrypt p2p request")) - return false, 0, 0, topic - } - - if len(decrypted.Payload) < 8 { - log.Warn(fmt.Sprintf("Undersized p2p request")) - return false, 0, 0, topic + return false, 0, 0, nil } src := crypto.FromECDSAPub(decrypted.Src) @@ -179,15 +172,24 @@ func (s *WMailServer) validateRequest(peerID []byte, request *whisper.Envelope) // if !bytes.Equal(peerID, src) { if src == nil { log.Warn(fmt.Sprintf("Wrong signature of p2p request")) - return false, 0, 0, topic + return false, 0, 0, nil + } + + var bloom []byte + payloadSize := len(decrypted.Payload) + if payloadSize < 8 { + log.Warn(fmt.Sprintf("Undersized p2p request")) + return false, 0, 0, nil + } else if payloadSize == 8 { + bloom = whisper.MakeFullNodeBloom() + } else if payloadSize < 8+whisper.BloomFilterSize { + log.Warn(fmt.Sprintf("Undersized bloom filter in p2p request")) + return false, 0, 0, nil + } else { + bloom = decrypted.Payload[8 : 8+whisper.BloomFilterSize] } lower := binary.BigEndian.Uint32(decrypted.Payload[:4]) upper := binary.BigEndian.Uint32(decrypted.Payload[4:8]) - - if len(decrypted.Payload) >= 8+whisper.TopicLength { - topic = whisper.BytesToTopic(decrypted.Payload[8:]) - } - - return true, lower, upper, topic + return true, lower, upper, bloom } diff --git a/whisper/mailserver/server_test.go b/whisper/mailserver/server_test.go index 63864c1e6b..d5b993afb9 100644 --- a/whisper/mailserver/server_test.go +++ b/whisper/mailserver/server_test.go @@ -17,6 +17,7 @@ package mailserver import ( + "bytes" "crypto/ecdsa" "encoding/binary" "io/ioutil" @@ -61,7 +62,7 @@ func generateEnvelope(t *testing.T) *whisper.Envelope { h := crypto.Keccak256Hash([]byte("test sample data")) params := &whisper.MessageParams{ KeySym: h[:], - Topic: whisper.TopicType{}, + Topic: whisper.TopicType{0x1F, 0x7E, 0xA1, 0x7F}, Payload: []byte("test payload"), PoW: powRequirement, WorkTime: 2, @@ -121,6 +122,7 @@ func deliverTest(t *testing.T, server *WMailServer, env *whisper.Envelope) { upp: birth + 1, key: testPeerID, } + singleRequest(t, server, env, p, true) p.low, p.upp = birth+1, 0xffffffff @@ -131,14 +133,14 @@ func deliverTest(t *testing.T, server *WMailServer, env *whisper.Envelope) { p.low = birth - 1 p.upp = birth + 1 - p.topic[0]++ + p.topic[0] = 0xFF singleRequest(t, server, env, p, false) } func singleRequest(t *testing.T, server *WMailServer, env *whisper.Envelope, p *ServerTestParams, expect bool) { request := createRequest(t, p) src := crypto.FromECDSAPub(&p.key.PublicKey) - ok, lower, upper, topic := server.validateRequest(src, request) + ok, lower, upper, bloom := server.validateRequest(src, request) if !ok { t.Fatalf("request validation failed, seed: %d.", seed) } @@ -148,12 +150,13 @@ func singleRequest(t *testing.T, server *WMailServer, env *whisper.Envelope, p * if upper != p.upp { t.Fatalf("request validation failed (upper bound), seed: %d.", seed) } - if topic != p.topic { + expectedBloom := whisper.TopicToBloom(p.topic) + if !bytes.Equal(bloom, expectedBloom) { t.Fatalf("request validation failed (topic), seed: %d.", seed) } var exist bool - mail := server.processRequest(nil, p.low, p.upp, p.topic) + mail := server.processRequest(nil, p.low, p.upp, bloom) for _, msg := range mail { if msg.Hash() == env.Hash() { exist = true @@ -166,7 +169,7 @@ func singleRequest(t *testing.T, server *WMailServer, env *whisper.Envelope, p * } src[0]++ - ok, lower, upper, topic = server.validateRequest(src, request) + ok, lower, upper, bloom = server.validateRequest(src, request) if !ok { // request should be valid regardless of signature t.Fatalf("request validation false negative, seed: %d (lower: %d, upper: %d).", seed, lower, upper) @@ -174,10 +177,11 @@ func singleRequest(t *testing.T, server *WMailServer, env *whisper.Envelope, p * } func createRequest(t *testing.T, p *ServerTestParams) *whisper.Envelope { - data := make([]byte, 8+whisper.TopicLength) + bloom := whisper.TopicToBloom(p.topic) + data := make([]byte, 8) binary.BigEndian.PutUint32(data, p.low) binary.BigEndian.PutUint32(data[4:], p.upp) - copy(data[8:], p.topic[:]) + data = append(data, bloom...) key, err := shh.GetSymKey(keyID) if err != nil { diff --git a/whisper/whisperv6/doc.go b/whisper/whisperv6/doc.go index d5d7fed609..066a9766d4 100644 --- a/whisper/whisperv6/doc.go +++ b/whisper/whisperv6/doc.go @@ -60,7 +60,7 @@ const ( aesKeyLength = 32 // in bytes aesNonceLength = 12 // in bytes; for more info please see cipher.gcmStandardNonceSize & aesgcm.NonceSize() keyIDSize = 32 // in bytes - bloomFilterSize = 64 // in bytes + BloomFilterSize = 64 // in bytes flagsLength = 1 EnvelopeHeaderLength = 20 diff --git a/whisper/whisperv6/envelope.go b/whisper/whisperv6/envelope.go index c7bea2bb90..2c80d47bc6 100644 --- a/whisper/whisperv6/envelope.go +++ b/whisper/whisperv6/envelope.go @@ -249,7 +249,7 @@ func (e *Envelope) Bloom() []byte { // TopicToBloom converts the topic (4 bytes) to the bloom filter (64 bytes) func TopicToBloom(topic TopicType) []byte { - b := make([]byte, bloomFilterSize) + b := make([]byte, BloomFilterSize) var index [3]int for j := 0; j < 3; j++ { index[j] = int(topic[j]) diff --git a/whisper/whisperv6/peer.go b/whisper/whisperv6/peer.go index 6d75290fde..2bf1c905b2 100644 --- a/whisper/whisperv6/peer.go +++ b/whisper/whisperv6/peer.go @@ -56,7 +56,7 @@ func newPeer(host *Whisper, remote *p2p.Peer, rw p2p.MsgReadWriter) *Peer { powRequirement: 0.0, known: set.New(), quit: make(chan struct{}), - bloomFilter: makeFullNodeBloom(), + bloomFilter: MakeFullNodeBloom(), fullNode: true, } } @@ -120,7 +120,7 @@ func (peer *Peer) handshake() error { err = s.Decode(&bloom) if err == nil { sz := len(bloom) - if sz != bloomFilterSize && sz != 0 { + if sz != BloomFilterSize && sz != 0 { return fmt.Errorf("peer [%x] sent bad status message: wrong bloom filter size %d", peer.ID(), sz) } peer.setBloomFilter(bloom) @@ -229,7 +229,7 @@ func (peer *Peer) notifyAboutBloomFilterChange(bloom []byte) error { func (peer *Peer) bloomMatch(env *Envelope) bool { peer.bloomMu.Lock() defer peer.bloomMu.Unlock() - return peer.fullNode || bloomFilterMatch(peer.bloomFilter, env.Bloom()) + return peer.fullNode || BloomFilterMatch(peer.bloomFilter, env.Bloom()) } func (peer *Peer) setBloomFilter(bloom []byte) { @@ -238,13 +238,13 @@ func (peer *Peer) setBloomFilter(bloom []byte) { peer.bloomFilter = bloom peer.fullNode = isFullNode(bloom) if peer.fullNode && peer.bloomFilter == nil { - peer.bloomFilter = makeFullNodeBloom() + peer.bloomFilter = MakeFullNodeBloom() } } -func makeFullNodeBloom() []byte { - bloom := make([]byte, bloomFilterSize) - for i := 0; i < bloomFilterSize; i++ { +func MakeFullNodeBloom() []byte { + bloom := make([]byte, BloomFilterSize) + for i := 0; i < BloomFilterSize; i++ { bloom[i] = 0xFF } return bloom diff --git a/whisper/whisperv6/peer_test.go b/whisper/whisperv6/peer_test.go index 6d7754d790..ec985ae65b 100644 --- a/whisper/whisperv6/peer_test.go +++ b/whisper/whisperv6/peer_test.go @@ -152,7 +152,7 @@ func resetParams(t *testing.T) { } func initBloom(t *testing.T) { - masterBloomFilter = make([]byte, bloomFilterSize) + masterBloomFilter = make([]byte, BloomFilterSize) _, err := mrand.Read(masterBloomFilter) if err != nil { t.Fatalf("rand failed: %s.", err) @@ -164,7 +164,7 @@ func initBloom(t *testing.T) { masterBloomFilter[i] = 0xFF } - if !bloomFilterMatch(masterBloomFilter, msgBloom) { + if !BloomFilterMatch(masterBloomFilter, msgBloom) { t.Fatalf("bloom mismatch on initBloom.") } } @@ -178,7 +178,7 @@ func initialize(t *testing.T) { for i := 0; i < NumNodes; i++ { var node TestNode - b := make([]byte, bloomFilterSize) + b := make([]byte, BloomFilterSize) copy(b, masterBloomFilter) node.shh = New(&DefaultConfig) node.shh.SetMinimumPoW(masterPow) diff --git a/whisper/whisperv6/whisper.go b/whisper/whisperv6/whisper.go index 81f59b3cc7..880cced098 100644 --- a/whisper/whisperv6/whisper.go +++ b/whisper/whisperv6/whisper.go @@ -232,11 +232,11 @@ func (whisper *Whisper) SetMaxMessageSize(size uint32) error { // SetBloomFilter sets the new bloom filter func (whisper *Whisper) SetBloomFilter(bloom []byte) error { - if len(bloom) != bloomFilterSize { + if len(bloom) != BloomFilterSize { return fmt.Errorf("invalid bloom filter size: %d", len(bloom)) } - b := make([]byte, bloomFilterSize) + b := make([]byte, BloomFilterSize) copy(b, bloom) whisper.settings.Store(bloomFilterIdx, b) @@ -558,14 +558,14 @@ func (whisper *Whisper) Subscribe(f *Filter) (string, error) { // updateBloomFilter recalculates the new value of bloom filter, // and informs the peers if necessary. func (whisper *Whisper) updateBloomFilter(f *Filter) { - aggregate := make([]byte, bloomFilterSize) + aggregate := make([]byte, BloomFilterSize) for _, t := range f.Topics { top := BytesToTopic(t) b := TopicToBloom(top) aggregate = addBloom(aggregate, b) } - if !bloomFilterMatch(whisper.BloomFilter(), aggregate) { + if !BloomFilterMatch(whisper.BloomFilter(), aggregate) { // existing bloom filter must be updated aggregate = addBloom(whisper.BloomFilter(), aggregate) whisper.SetBloomFilter(aggregate) @@ -701,7 +701,7 @@ func (whisper *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error { case bloomFilterExCode: var bloom []byte err := packet.Decode(&bloom) - if err == nil && len(bloom) != bloomFilterSize { + if err == nil && len(bloom) != BloomFilterSize { err = fmt.Errorf("wrong bloom filter size %d", len(bloom)) } @@ -779,11 +779,11 @@ func (whisper *Whisper) add(envelope *Envelope, isP2P bool) (bool, error) { } } - if !bloomFilterMatch(whisper.BloomFilter(), envelope.Bloom()) { + if !BloomFilterMatch(whisper.BloomFilter(), envelope.Bloom()) { // maybe the value was recently changed, and the peers did not adjust yet. // in this case the previous value is retrieved by BloomFilterTolerance() // for a short period of peer synchronization. - if !bloomFilterMatch(whisper.BloomFilterTolerance(), envelope.Bloom()) { + if !BloomFilterMatch(whisper.BloomFilterTolerance(), envelope.Bloom()) { return false, fmt.Errorf("envelope does not match bloom filter, hash=[%v], bloom: \n%x \n%x \n%x", envelope.Hash().Hex(), whisper.BloomFilter(), envelope.Bloom(), envelope.Topic) } @@ -1025,12 +1025,12 @@ func isFullNode(bloom []byte) bool { return true } -func bloomFilterMatch(filter, sample []byte) bool { +func BloomFilterMatch(filter, sample []byte) bool { if filter == nil { return true } - for i := 0; i < bloomFilterSize; i++ { + for i := 0; i < BloomFilterSize; i++ { f := filter[i] s := sample[i] if (f | s) != f { @@ -1042,8 +1042,8 @@ func bloomFilterMatch(filter, sample []byte) bool { } func addBloom(a, b []byte) []byte { - c := make([]byte, bloomFilterSize) - for i := 0; i < bloomFilterSize; i++ { + c := make([]byte, BloomFilterSize) + for i := 0; i < BloomFilterSize; i++ { c[i] = a[i] | b[i] } return c diff --git a/whisper/whisperv6/whisper_test.go b/whisper/whisperv6/whisper_test.go index 3a219cd133..7fe256309e 100644 --- a/whisper/whisperv6/whisper_test.go +++ b/whisper/whisperv6/whisper_test.go @@ -826,11 +826,11 @@ func TestSymmetricSendKeyMismatch(t *testing.T) { func TestBloom(t *testing.T) { topic := TopicType{0, 0, 255, 6} b := TopicToBloom(topic) - x := make([]byte, bloomFilterSize) + x := make([]byte, BloomFilterSize) x[0] = byte(1) x[32] = byte(1) - x[bloomFilterSize-1] = byte(128) - if !bloomFilterMatch(x, b) || !bloomFilterMatch(b, x) { + x[BloomFilterSize-1] = byte(128) + if !BloomFilterMatch(x, b) || !BloomFilterMatch(b, x) { t.Fatalf("bloom filter does not match the mask") } @@ -842,11 +842,11 @@ func TestBloom(t *testing.T) { if err != nil { t.Fatalf("math rand error") } - if !bloomFilterMatch(b, b) { + if !BloomFilterMatch(b, b) { t.Fatalf("bloom filter does not match self") } x = addBloom(x, b) - if !bloomFilterMatch(x, b) { + if !BloomFilterMatch(x, b) { t.Fatalf("bloom filter does not match combined bloom") } if !isFullNode(nil) { @@ -856,16 +856,16 @@ func TestBloom(t *testing.T) { if isFullNode(x) { t.Fatalf("isFullNode false positive") } - for i := 0; i < bloomFilterSize; i++ { + for i := 0; i < BloomFilterSize; i++ { b[i] = byte(255) } if !isFullNode(b) { t.Fatalf("isFullNode false negative") } - if bloomFilterMatch(x, b) { + if BloomFilterMatch(x, b) { t.Fatalf("bloomFilterMatch false positive") } - if !bloomFilterMatch(b, x) { + if !BloomFilterMatch(b, x) { t.Fatalf("bloomFilterMatch false negative") } @@ -879,7 +879,7 @@ func TestBloom(t *testing.T) { t.Fatalf("failed to set bloom filter: %s", err) } f = w.BloomFilter() - if !bloomFilterMatch(f, x) || !bloomFilterMatch(x, f) { + if !BloomFilterMatch(f, x) || !BloomFilterMatch(x, f) { t.Fatalf("retireved wrong bloom filter") } } From 3ca3fffdf01b94244ef6c2d93ed38a30da9fcb0a Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 1 Mar 2018 18:55:31 +0100 Subject: [PATCH 22/33] metrics: fix flaky Example metrics test (#16222) * metrics: add sleep to test in order to get predictable output * metrics: relax constraints on timer test --- metrics/metrics_test.go | 3 ++- metrics/timer_test.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/metrics/metrics_test.go b/metrics/metrics_test.go index 726fba3475..df36da0ade 100644 --- a/metrics/metrics_test.go +++ b/metrics/metrics_test.go @@ -6,6 +6,7 @@ import ( "log" "sync" "testing" + "time" ) const FANOUT = 128 @@ -114,7 +115,7 @@ func Example() { // Threadsafe registration t := GetOrRegisterTimer("db.get.latency", nil) - t.Time(func() {}) + t.Time(func() { time.Sleep(10 * time.Millisecond) }) t.Update(1) fmt.Println(c.Count()) diff --git a/metrics/timer_test.go b/metrics/timer_test.go index f85c9b803d..c1f0ff9388 100644 --- a/metrics/timer_test.go +++ b/metrics/timer_test.go @@ -47,8 +47,8 @@ func TestTimerStop(t *testing.T) { func TestTimerFunc(t *testing.T) { tm := NewTimer() tm.Time(func() { time.Sleep(50e6) }) - if max := tm.Max(); 45e6 > max || max > 55e6 { - t.Errorf("tm.Max(): 45e6 > %v || %v > 55e6\n", max, max) + if max := tm.Max(); 35e6 > max || max > 95e6 { + t.Errorf("tm.Max(): 35e6 > %v || %v > 95e6\n", max, max) } } From d520bf45038ec8f02dd255bea038c5785f7290e0 Mon Sep 17 00:00:00 2001 From: Zhenguo Niu Date: Fri, 2 Mar 2018 17:59:26 +0800 Subject: [PATCH 23/33] cmd/swarm: fix some typos in manifest cmd (#16227) Replace "atleast" with "at least" in the manifest error message. --- cmd/swarm/manifest.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/swarm/manifest.go b/cmd/swarm/manifest.go index aa276e0f9b..41a69a5d05 100644 --- a/cmd/swarm/manifest.go +++ b/cmd/swarm/manifest.go @@ -35,7 +35,7 @@ const bzzManifestJSON = "application/bzz-manifest+json" func add(ctx *cli.Context) { args := ctx.Args() if len(args) < 3 { - utils.Fatalf("Need atleast three arguments []") + utils.Fatalf("Need at least three arguments []") } var ( @@ -69,7 +69,7 @@ func update(ctx *cli.Context) { args := ctx.Args() if len(args) < 3 { - utils.Fatalf("Need atleast three arguments ") + utils.Fatalf("Need at least three arguments ") } var ( @@ -101,7 +101,7 @@ func update(ctx *cli.Context) { func remove(ctx *cli.Context) { args := ctx.Args() if len(args) < 2 { - utils.Fatalf("Need atleast two arguments ") + utils.Fatalf("Need at least two arguments ") } var ( From 6f13e515f40ef84ff277d373488c99de49e33f18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 2 Mar 2018 11:57:11 +0200 Subject: [PATCH 24/33] cmd/faucet: update state in background, skip when busy --- cmd/faucet/faucet.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index 095668c868..5bad09bbd5 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -533,9 +533,11 @@ func (f *faucet) loop() { } defer sub.Unsubscribe() - for { - select { - case head := <-heads: + // Start a goroutine to update the state from head notifications in the background + update := make(chan *types.Header) + + go func() { + for head := range update { // New chain head arrived, query the current stats and stream to clients var ( balance *big.Int @@ -588,6 +590,17 @@ func (f *faucet) loop() { } } f.lock.RUnlock() + } + }() + // Wait for various events and assing to the appropriate background threads + for { + select { + case head := <-heads: + // New head arrived, send if for state update if there's none running + select { + case update <- head: + default: + } case <-f.update: // Pending requests updated, stream to clients From 6219a338225f1a4dfb7e51212ec3dde6e32785ce Mon Sep 17 00:00:00 2001 From: Vlad Date: Fri, 2 Mar 2018 14:54:54 +0100 Subject: [PATCH 25/33] whisper: filereader mode introduced to wnode --- cmd/wnode/main.go | 37 +++++++++++++++++++++++++++++++++++ whisper/whisperv6/envelope.go | 4 ++++ 2 files changed, 41 insertions(+) diff --git a/cmd/wnode/main.go b/cmd/wnode/main.go index ccfdd46265..76590e7f5f 100644 --- a/cmd/wnode/main.go +++ b/cmd/wnode/main.go @@ -86,6 +86,7 @@ var ( asymmetricMode = flag.Bool("asym", false, "use asymmetric encryption") generateKey = flag.Bool("generatekey", false, "generate and show the private key") fileExMode = flag.Bool("fileexchange", false, "file exchange mode") + fileReader = flag.Bool("filereader", false, "load and decrypt messages saved as files, display as plain text") testMode = flag.Bool("test", false, "use of predefined parameters for diagnostics (password, etc.)") echoMode = flag.Bool("echo", false, "echo mode: prints some arguments for diagnostics") @@ -433,6 +434,8 @@ func run() { requestExpiredMessagesLoop() } else if *fileExMode { sendFilesLoop() + } else if *fileReader { + fileReaderLoop() } else { sendLoop() } @@ -483,6 +486,40 @@ func sendFilesLoop() { } } +func fileReaderLoop() { + watcher1 := shh.GetFilter(symFilterID) + watcher2 := shh.GetFilter(asymFilterID) + if watcher1 == nil && watcher2 == nil { + fmt.Println("Error: neither symmetric nor asymmetric filter is installed") + close(done) + return + } + + for { + s := scanLine("") + if s == quitCommand { + fmt.Println("Quit command received") + close(done) + return + } + raw, err := ioutil.ReadFile(s) + if err != nil { + fmt.Printf(">>> Error: %s \n", err) + } else { + env := whisper.Envelope{Data: raw} // the topic is zero + msg := env.Open(watcher1) // force-open envelope regardless of the topic + if msg == nil { + msg = env.Open(watcher2) + } + if msg == nil { + fmt.Printf(">>> Error: failed to decrypt the message \n") + } else { + printMessageInfo(msg) + } + } + } +} + func scanLine(prompt string) string { if len(prompt) > 0 { fmt.Print(prompt) diff --git a/whisper/whisperv6/envelope.go b/whisper/whisperv6/envelope.go index 2c80d47bc6..2f947f1a40 100644 --- a/whisper/whisperv6/envelope.go +++ b/whisper/whisperv6/envelope.go @@ -208,6 +208,10 @@ func (e *Envelope) OpenSymmetric(key []byte) (msg *ReceivedMessage, err error) { // Open tries to decrypt an envelope, and populates the message fields in case of success. func (e *Envelope) Open(watcher *Filter) (msg *ReceivedMessage) { + if watcher == nil { + return nil + } + // The API interface forbids filters doing both symmetric and asymmetric encryption. if watcher.expectsAsymmetricEncryption() && watcher.expectsSymmetricEncryption() { return nil From 12f4d284114a719e9a0779933e8770c352ed1767 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Sat, 3 Mar 2018 00:52:21 +0100 Subject: [PATCH 26/33] internal/debug: add support for mutex profiles (#16230) --- internal/debug/api.go | 27 +++++++++++++++++++++++---- internal/web3ext/web3ext.go | 15 +++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/internal/debug/api.go b/internal/debug/api.go index 3547b05641..048b7d7635 100644 --- a/internal/debug/api.go +++ b/internal/debug/api.go @@ -140,10 +140,9 @@ func (h *HandlerT) GoTrace(file string, nsec uint) error { return nil } -// BlockProfile turns on CPU profiling for nsec seconds and writes -// profile data to file. It uses a profile rate of 1 for most accurate -// information. If a different rate is desired, set the rate -// and write the profile manually. +// BlockProfile turns on goroutine profiling for nsec seconds and writes profile data to +// file. It uses a profile rate of 1 for most accurate information. If a different rate is +// desired, set the rate and write the profile manually. func (*HandlerT) BlockProfile(file string, nsec uint) error { runtime.SetBlockProfileRate(1) time.Sleep(time.Duration(nsec) * time.Second) @@ -162,6 +161,26 @@ func (*HandlerT) WriteBlockProfile(file string) error { return writeProfile("block", file) } +// MutexProfile turns on mutex profiling for nsec seconds and writes profile data to file. +// It uses a profile rate of 1 for most accurate information. If a different rate is +// desired, set the rate and write the profile manually. +func (*HandlerT) MutexProfile(file string, nsec uint) error { + runtime.SetMutexProfileFraction(1) + time.Sleep(time.Duration(nsec) * time.Second) + defer runtime.SetMutexProfileFraction(0) + return writeProfile("mutex", file) +} + +// SetMutexProfileFraction sets the rate of mutex profiling. +func (*HandlerT) SetMutexProfileFraction(rate int) { + runtime.SetMutexProfileFraction(rate) +} + +// WriteMutexProfile writes a goroutine blocking profile to the given file. +func (*HandlerT) WriteMutexProfile(file string) error { + return writeProfile("mutex", file) +} + // WriteMemProfile writes an allocation profile to the given file. // Note that the profiling rate cannot be set through the API, // it must be set on the command line. diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index a6b81b4c2a..9d6ce8c6c8 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -307,6 +307,21 @@ web3._extend({ call: 'debug_writeBlockProfile', params: 1 }), + new web3._extend.Method({ + name: 'mutexProfile', + call: 'debug_mutexProfile', + params: 2 + }), + new web3._extend.Method({ + name: 'setMutexProfileRate', + call: 'debug_setMutexProfileRate', + params: 1 + }), + new web3._extend.Method({ + name: 'writeMutexProfile', + call: 'debug_writeMutexProfile', + params: 1 + }), new web3._extend.Method({ name: 'writeMemProfile', call: 'debug_writeMemProfile', From ca64a122d33008c155c35a9d0e78cfbcafb1820a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Sat, 3 Mar 2018 01:52:39 +0200 Subject: [PATCH 27/33] eth/downloader: save and load trie sync progress (#16224) --- core/database_util.go | 20 ++++++++++++++++++++ eth/downloader/downloader.go | 6 +++++- eth/downloader/statesync.go | 4 ++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/core/database_util.go b/core/database_util.go index 61ab701349..8c46989854 100644 --- a/core/database_util.go +++ b/core/database_util.go @@ -47,6 +47,7 @@ var ( headHeaderKey = []byte("LastHeader") headBlockKey = []byte("LastBlock") headFastKey = []byte("LastFast") + trieSyncKey = []byte("TrieSync") // Data item prefixes (use single byte to avoid mixing data types, avoid `i`). headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header @@ -146,6 +147,16 @@ func GetHeadFastBlockHash(db DatabaseReader) common.Hash { return common.BytesToHash(data) } +// GetTrieSyncProgress retrieves the number of tries nodes fast synced to allow +// reportinc correct numbers across restarts. +func GetTrieSyncProgress(db DatabaseReader) uint64 { + data, _ := db.Get(trieSyncKey) + if len(data) == 0 { + return 0 + } + return new(big.Int).SetBytes(data).Uint64() +} + // GetHeaderRLP retrieves a block header in its raw RLP database encoding, or nil // if the header's not found. func GetHeaderRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue { @@ -374,6 +385,15 @@ func WriteHeadFastBlockHash(db ethdb.Putter, hash common.Hash) error { return nil } +// WriteTrieSyncProgress stores the fast sync trie process counter to support +// retrieving it across restarts. +func WriteTrieSyncProgress(db ethdb.Putter, count uint64) error { + if err := db.Put(trieSyncKey, new(big.Int).SetUint64(count).Bytes()); err != nil { + log.Crit("Failed to store fast sync trie progress", "err", err) + } + return nil +} + // WriteHeader serializes a block header into the database. func WriteHeader(db ethdb.Putter, header *types.Header) error { data, err := rlp.EncodeToBytes(header) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index d13247766a..70febf4cb1 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -27,6 +27,7 @@ import ( ethereum "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -221,7 +222,10 @@ func New(mode SyncMode, stateDb ethdb.Database, mux *event.TypeMux, chain BlockC quitCh: make(chan struct{}), stateCh: make(chan dataPack), stateSyncStart: make(chan *stateSync), - trackStateReq: make(chan *stateReq), + syncStatsState: stateSyncStats{ + processed: core.GetTrieSyncProgress(stateDb), + }, + trackStateReq: make(chan *stateReq), } go dl.qosTuner() go dl.stateFetcher() diff --git a/eth/downloader/statesync.go b/eth/downloader/statesync.go index 9cc65a208c..ee6c7b4914 100644 --- a/eth/downloader/statesync.go +++ b/eth/downloader/statesync.go @@ -23,6 +23,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/ethdb" @@ -466,4 +467,7 @@ func (s *stateSync) updateStats(written, duplicate, unexpected int, duration tim if written > 0 || duplicate > 0 || unexpected > 0 { log.Info("Imported new state entries", "count", written, "elapsed", common.PrettyDuration(duration), "processed", s.d.syncStatsState.processed, "pending", s.d.syncStatsState.pending, "retry", len(s.tasks), "duplicate", s.d.syncStatsState.duplicate, "unexpected", s.d.syncStatsState.unexpected) } + if written > 0 { + core.WriteTrieSyncProgress(s.d.stateDB, s.d.syncStatsState.processed) + } } From 5ad7b9123ce4688a2e3370e7487ef979e97c4373 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Sat, 3 Mar 2018 00:52:54 +0100 Subject: [PATCH 28/33] light: new CHTs (#16233) --- .github/CODEOWNERS | 2 ++ light/postprocess.go | 16 ++++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6076fe46a4..a7b6176552 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,5 +5,7 @@ accounts/usbwallet @karalabe consensus @karalabe core/ @karalabe @holiman eth/ @karalabe +les/ @zsfelfoldi +light/ @zsfelfoldi mobile/ @karalabe p2p/ @fjl @zsfelfoldi diff --git a/light/postprocess.go b/light/postprocess.go index 84149fdaae..384a635f76 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -58,18 +58,18 @@ type trustedCheckpoint struct { var ( mainnetCheckpoint = trustedCheckpoint{ name: "mainnet", - sectionIdx: 153, - sectionHead: common.HexToHash("04c2114a8cbe49ba5c37a03cc4b4b8d3adfc0bd2c78e0e726405dd84afca1d63"), - chtRoot: common.HexToHash("d7ec603e5d30b567a6e894ee7704e4603232f206d3e5a589794cec0c57bf318e"), - bloomTrieRoot: common.HexToHash("0b139b8fb692e21f663ff200da287192201c28ef5813c1ac6ba02a0a4799eef9"), + sectionIdx: 157, + sectionHead: common.HexToHash("1963c080887ca7f406c2bb114293eea83e54f783f94df24b447f7e3b6317c747"), + chtRoot: common.HexToHash("42abc436567dfb678a38fa6a9f881aa4c8a4cc8eaa2def08359292c3d0bd48ec"), + bloomTrieRoot: common.HexToHash("281c9f8fb3cb8b37ae45e9907ef8f3b19cd22c54e297c2d6c09c1db1593dce42"), } ropstenCheckpoint = trustedCheckpoint{ name: "ropsten", - sectionIdx: 79, - sectionHead: common.HexToHash("1b1ba890510e06411fdee9bb64ca7705c56a1a4ce3559ddb34b3680c526cb419"), - chtRoot: common.HexToHash("71d60207af74e5a22a3e1cfbfc89f9944f91b49aa980c86fba94d568369eaf44"), - bloomTrieRoot: common.HexToHash("70aca4b3b6d08dde8704c95cedb1420394453c1aec390947751e69ff8c436360"), + sectionIdx: 83, + sectionHead: common.HexToHash("3ca623586bc0da35f1fc8d9b6b55950f3b1f69be9c6501846a2df672adb61236"), + chtRoot: common.HexToHash("8f08ec7783969768c6ef06e5fe3398223cbf4ae2907b676da7b6fe6c7f55b059"), + bloomTrieRoot: common.HexToHash("02d86d3c6a87f8f8a92c2a59bbba2132ff6f9f61b0915a5dc28a9d8279219fd0"), } ) From fa375955ad52bc7936f33c8d8cec68fb9007baaa Mon Sep 17 00:00:00 2001 From: gluk256 Date: Sat, 3 Mar 2018 00:54:15 +0100 Subject: [PATCH 29/33] whisper/whisperv6: delete unused function (#16234) --- whisper/whisperv6/filter.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/whisper/whisperv6/filter.go b/whisper/whisperv6/filter.go index e4171f85c9..2f170ddebe 100644 --- a/whisper/whisperv6/filter.go +++ b/whisper/whisperv6/filter.go @@ -191,20 +191,6 @@ func (fs *Filters) NotifyWatchers(env *Envelope, p2pMessage bool) { } } -func (f *Filter) processEnvelope(env *Envelope) *ReceivedMessage { - if f.MatchEnvelope(env) { - msg := env.Open(f) - if msg != nil { - return msg - } - - log.Trace("processing envelope: failed to open", "hash", env.Hash().Hex()) - } else { - log.Trace("processing envelope: does not match", "hash", env.Hash().Hex()) - } - return nil -} - func (f *Filter) expectsAsymmetricEncryption() bool { return f.KeyAsym != nil } From 95cca85d6d13514284f2ced54196d2b3b0aaa3d9 Mon Sep 17 00:00:00 2001 From: Vlad Date: Sat, 3 Mar 2018 21:37:16 +0100 Subject: [PATCH 30/33] whisper: minor refactoring --- cmd/wnode/main.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/cmd/wnode/main.go b/cmd/wnode/main.go index 76590e7f5f..00a854fe87 100644 --- a/cmd/wnode/main.go +++ b/cmd/wnode/main.go @@ -439,6 +439,8 @@ func run() { } else { sendLoop() } + + close(done) } func sendLoop() { @@ -446,11 +448,9 @@ func sendLoop() { s := scanLine("") if s == quitCommand { fmt.Println("Quit command received") - close(done) - break + return } sendMsg([]byte(s)) - if *asymmetricMode { // print your own message for convenience, // because in asymmetric mode it is impossible to decrypt it @@ -466,13 +466,11 @@ func sendFilesLoop() { s := scanLine("") if s == quitCommand { fmt.Println("Quit command received") - close(done) - break + return } b, err := ioutil.ReadFile(s) if err != nil { fmt.Printf(">>> Error: %s \n", err) - continue } else { h := sendMsg(b) if (h == common.Hash{}) { @@ -491,7 +489,6 @@ func fileReaderLoop() { watcher2 := shh.GetFilter(asymFilterID) if watcher1 == nil && watcher2 == nil { fmt.Println("Error: neither symmetric nor asymmetric filter is installed") - close(done) return } @@ -499,7 +496,6 @@ func fileReaderLoop() { s := scanLine("") if s == quitCommand { fmt.Println("Quit command received") - close(done) return } raw, err := ioutil.ReadFile(s) From 0b814d32f8737b194874942f11dc3e9e7399cf7b Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 4 Mar 2018 23:24:17 +0100 Subject: [PATCH 31/33] accounts/abi: Abi binding support for nested arrays, fixes #15648, including nested array unpack fix (#15676) * accounts/abi/bind: support for multi-dim arrays Also: - reduce usage of regexes a bit. - fix minor Java syntax problems Fixes #15648 * accounts/abi/bind: Add some more documentation * accounts/abi/bind: Improve code readability * accounts/abi: bugfix for unpacking nested arrays The code previously assumed the arrays/slices were always 1 level deep. While the packing supports nested arrays (!!!). The current code for unpacking doesn't return the "consumed" length, so this fix had to work around that by calculating it (i.e. packing and getting resulting length) after the unpacking of the array element. It's far from ideal, but unpacking behaviour is fixed now. * accounts/abi: Fix unpacking of nested arrays Removed the temporary workaround of packing to calculate size, which was incorrect for slice-like types anyway. Full size of nested arrays is used now. * accounts/abi: deeply nested array unpack test Test unpacking of an array nested more than one level. * accounts/abi: Add deeply nested array pack test Same as the deep nested array unpack test, but the other way around. * accounts/abi/bind: deeply nested arrays bind test Test the usage of bindings that were generated for methods with multi-dimensional (and not just a single extra dimension, like foo[2][3]) array arguments and returns. edit: trigger rebuild, CI failed to fetch linter module. * accounts/abi/bind: improve array binding wrapArray uses a regex now, and arrayBindingJava is improved. * accounts/abi: Improve naming of element size func The full step size for unpacking an array is now retrieved with "getFullElemSize". * accounts/abi: support nested nested array args Previously, the code only considered the outer-size of the array, ignoring the size of the contents. This was fine for most types, but nested arrays are packed directly into it, and count towards the total size. This resulted in arguments following a nested array to replicate some of the binary contents of the array. The fix: for arrays, calculate their complete contents size: count the arg.Type.Elem.Size when Elem is an Array, and repeat when their child is an array too, etc. The count is the number of 32 byte elements, similar to how it previously counted, but nested. * accounts/abi: Test deep nested arr multi-arguments Arguments with a deeply nested array should not cause the next arguments to be read from the wrong position. --- accounts/abi/argument.go | 26 ++++- accounts/abi/bind/bind.go | 169 +++++++++++++++++++-------------- accounts/abi/bind/bind_test.go | 66 +++++++++++++ accounts/abi/pack_test.go | 5 + accounts/abi/unpack.go | 27 ++++-- accounts/abi/unpack_test.go | 45 +++++++++ 6 files changed, 259 insertions(+), 79 deletions(-) diff --git a/accounts/abi/argument.go b/accounts/abi/argument.go index f171f4cc63..1b480da609 100644 --- a/accounts/abi/argument.go +++ b/accounts/abi/argument.go @@ -169,6 +169,21 @@ func (arguments Arguments) unpackAtomic(v interface{}, marshalledValues []interf return set(elem, reflectValue, arguments.NonIndexed()[0]) } +// Computes the full size of an array; +// i.e. counting nested arrays, which count towards size for unpacking. +func getArraySize(arr *Type) int { + size := arr.Size + // Arrays can be nested, with each element being the same size + arr = arr.Elem + for arr.T == ArrayTy { + // Keep multiplying by elem.Size while the elem is an array. + size *= arr.Size + arr = arr.Elem + } + // Now we have the full array size, including its children. + return size +} + // UnpackValues can be used to unpack ABI-encoded hexdata according to the ABI-specification, // without supplying a struct to unpack into. Instead, this method returns a list containing the // values. An atomic argument will be a list with one element. @@ -181,9 +196,14 @@ func (arguments Arguments) UnpackValues(data []byte) ([]interface{}, error) { // If we have a static array, like [3]uint256, these are coded as // just like uint256,uint256,uint256. // This means that we need to add two 'virtual' arguments when - // we count the index from now on - - virtualArgs += arg.Type.Size - 1 + // we count the index from now on. + // + // Array values nested multiple levels deep are also encoded inline: + // [2][3]uint256: uint256,uint256,uint256,uint256,uint256,uint256 + // + // Calculate the full array size to get the correct offset for the next argument. + // Decrement it by 1, as the normal index increment is still applied. + virtualArgs += getArraySize(&arg.Type) - 1 } if err != nil { return nil, err diff --git a/accounts/abi/bind/bind.go b/accounts/abi/bind/bind.go index e31b454812..7fdd2c624f 100644 --- a/accounts/abi/bind/bind.go +++ b/accounts/abi/bind/bind.go @@ -164,118 +164,147 @@ var bindType = map[Lang]func(kind abi.Type) string{ LangJava: bindTypeJava, } +// Helper function for the binding generators. +// It reads the unmatched characters after the inner type-match, +// (since the inner type is a prefix of the total type declaration), +// looks for valid arrays (possibly a dynamic one) wrapping the inner type, +// and returns the sizes of these arrays. +// +// Returned array sizes are in the same order as solidity signatures; inner array size first. +// Array sizes may also be "", indicating a dynamic array. +func wrapArray(stringKind string, innerLen int, innerMapping string) (string, []string) { + remainder := stringKind[innerLen:] + //find all the sizes + matches := regexp.MustCompile(`\[(\d*)\]`).FindAllStringSubmatch(remainder, -1) + parts := make([]string, 0, len(matches)) + for _, match := range matches { + //get group 1 from the regex match + parts = append(parts, match[1]) + } + return innerMapping, parts +} + +// Translates the array sizes to a Go-lang declaration of a (nested) array of the inner type. +// Simply returns the inner type if arraySizes is empty. +func arrayBindingGo(inner string, arraySizes []string) string { + out := "" + //prepend all array sizes, from outer (end arraySizes) to inner (start arraySizes) + for i := len(arraySizes) - 1; i >= 0; i-- { + out += "[" + arraySizes[i] + "]" + } + out += inner + return out +} + // bindTypeGo converts a Solidity type to a Go one. Since there is no clear mapping // from all Solidity types to Go ones (e.g. uint17), those that cannot be exactly // mapped will use an upscaled type (e.g. *big.Int). func bindTypeGo(kind abi.Type) string { stringKind := kind.String() + innerLen, innerMapping := bindUnnestedTypeGo(stringKind) + return arrayBindingGo(wrapArray(stringKind, innerLen, innerMapping)) +} + +// The inner function of bindTypeGo, this finds the inner type of stringKind. +// (Or just the type itself if it is not an array or slice) +// The length of the matched part is returned, with the the translated type. +func bindUnnestedTypeGo(stringKind string) (int, string) { switch { case strings.HasPrefix(stringKind, "address"): - parts := regexp.MustCompile(`address(\[[0-9]*\])?`).FindStringSubmatch(stringKind) - if len(parts) != 2 { - return stringKind - } - return fmt.Sprintf("%scommon.Address", parts[1]) + return len("address"), "common.Address" case strings.HasPrefix(stringKind, "bytes"): - parts := regexp.MustCompile(`bytes([0-9]*)(\[[0-9]*\])?`).FindStringSubmatch(stringKind) - if len(parts) != 3 { - return stringKind - } - return fmt.Sprintf("%s[%s]byte", parts[2], parts[1]) + parts := regexp.MustCompile(`bytes([0-9]*)`).FindStringSubmatch(stringKind) + return len(parts[0]), fmt.Sprintf("[%s]byte", parts[1]) case strings.HasPrefix(stringKind, "int") || strings.HasPrefix(stringKind, "uint"): - parts := regexp.MustCompile(`(u)?int([0-9]*)(\[[0-9]*\])?`).FindStringSubmatch(stringKind) - if len(parts) != 4 { - return stringKind - } + parts := regexp.MustCompile(`(u)?int([0-9]*)`).FindStringSubmatch(stringKind) switch parts[2] { case "8", "16", "32", "64": - return fmt.Sprintf("%s%sint%s", parts[3], parts[1], parts[2]) + return len(parts[0]), fmt.Sprintf("%sint%s", parts[1], parts[2]) } - return fmt.Sprintf("%s*big.Int", parts[3]) + return len(parts[0]), "*big.Int" - case strings.HasPrefix(stringKind, "bool") || strings.HasPrefix(stringKind, "string"): - parts := regexp.MustCompile(`([a-z]+)(\[[0-9]*\])?`).FindStringSubmatch(stringKind) - if len(parts) != 3 { - return stringKind - } - return fmt.Sprintf("%s%s", parts[2], parts[1]) + case strings.HasPrefix(stringKind, "bool"): + return len("bool"), "bool" + + case strings.HasPrefix(stringKind, "string"): + return len("string"), "string" default: - return stringKind + return len(stringKind), stringKind } } +// Translates the array sizes to a Java declaration of a (nested) array of the inner type. +// Simply returns the inner type if arraySizes is empty. +func arrayBindingJava(inner string, arraySizes []string) string { + // Java array type declarations do not include the length. + return inner + strings.Repeat("[]", len(arraySizes)) +} + // bindTypeJava converts a Solidity type to a Java one. Since there is no clear mapping // from all Solidity types to Java ones (e.g. uint17), those that cannot be exactly // mapped will use an upscaled type (e.g. BigDecimal). func bindTypeJava(kind abi.Type) string { stringKind := kind.String() + innerLen, innerMapping := bindUnnestedTypeJava(stringKind) + return arrayBindingJava(wrapArray(stringKind, innerLen, innerMapping)) +} + +// The inner function of bindTypeJava, this finds the inner type of stringKind. +// (Or just the type itself if it is not an array or slice) +// The length of the matched part is returned, with the the translated type. +func bindUnnestedTypeJava(stringKind string) (int, string) { switch { case strings.HasPrefix(stringKind, "address"): parts := regexp.MustCompile(`address(\[[0-9]*\])?`).FindStringSubmatch(stringKind) if len(parts) != 2 { - return stringKind + return len(stringKind), stringKind } if parts[1] == "" { - return fmt.Sprintf("Address") + return len("address"), "Address" } - return fmt.Sprintf("Addresses") + return len(parts[0]), "Addresses" case strings.HasPrefix(stringKind, "bytes"): - parts := regexp.MustCompile(`bytes([0-9]*)(\[[0-9]*\])?`).FindStringSubmatch(stringKind) - if len(parts) != 3 { - return stringKind + parts := regexp.MustCompile(`bytes([0-9]*)`).FindStringSubmatch(stringKind) + if len(parts) != 2 { + return len(stringKind), stringKind } - if parts[2] != "" { - return "byte[][]" - } - return "byte[]" + return len(parts[0]), "byte[]" case strings.HasPrefix(stringKind, "int") || strings.HasPrefix(stringKind, "uint"): - parts := regexp.MustCompile(`(u)?int([0-9]*)(\[[0-9]*\])?`).FindStringSubmatch(stringKind) - if len(parts) != 4 { - return stringKind + //Note that uint and int (without digits) are also matched, + // these are size 256, and will translate to BigInt (the default). + parts := regexp.MustCompile(`(u)?int([0-9]*)`).FindStringSubmatch(stringKind) + if len(parts) != 3 { + return len(stringKind), stringKind } - switch parts[2] { - case "8", "16", "32", "64": - if parts[1] == "" { - if parts[3] == "" { - return fmt.Sprintf("int%s", parts[2]) - } - return fmt.Sprintf("int%s[]", parts[2]) - } + + namedSize := map[string]string{ + "8": "byte", + "16": "short", + "32": "int", + "64": "long", + }[parts[2]] + + //default to BigInt + if namedSize == "" { + namedSize = "BigInt" } - if parts[3] == "" { - return fmt.Sprintf("BigInt") - } - return fmt.Sprintf("BigInts") + return len(parts[0]), namedSize case strings.HasPrefix(stringKind, "bool"): - parts := regexp.MustCompile(`bool(\[[0-9]*\])?`).FindStringSubmatch(stringKind) - if len(parts) != 2 { - return stringKind - } - if parts[1] == "" { - return fmt.Sprintf("bool") - } - return fmt.Sprintf("bool[]") + return len("bool"), "boolean" case strings.HasPrefix(stringKind, "string"): - parts := regexp.MustCompile(`string(\[[0-9]*\])?`).FindStringSubmatch(stringKind) - if len(parts) != 2 { - return stringKind - } - if parts[1] == "" { - return fmt.Sprintf("String") - } - return fmt.Sprintf("String[]") + return len("string"), "String" default: - return stringKind + return len(stringKind), stringKind } } @@ -325,11 +354,13 @@ func namedTypeJava(javaKind string, solKind abi.Type) string { return "String" case "string[]": return "Strings" - case "bool": + case "boolean": return "Bool" - case "bool[]": + case "boolean[]": return "Bools" - case "BigInt": + case "BigInt[]": + return "BigInts" + default: parts := regexp.MustCompile(`(u)?int([0-9]*)(\[[0-9]*\])?`).FindStringSubmatch(solKind.String()) if len(parts) != 4 { return javaKind @@ -344,8 +375,6 @@ func namedTypeJava(javaKind string, solKind abi.Type) string { default: return javaKind } - default: - return javaKind } } diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index c4838e6470..26816ec20d 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -737,6 +737,72 @@ var bindTests = []struct { } `, }, + { + `DeeplyNestedArray`, + ` + contract DeeplyNestedArray { + uint64[3][4][5] public deepUint64Array; + function storeDeepUintArray(uint64[3][4][5] arr) public { + deepUint64Array = arr; + } + function retrieveDeepArray() public view returns (uint64[3][4][5]) { + return deepUint64Array; + } + } + `, + `6060604052341561000f57600080fd5b6106438061001e6000396000f300606060405260043610610057576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff168063344248551461005c5780638ed4573a1461011457806398ed1856146101ab575b600080fd5b341561006757600080fd5b610112600480806107800190600580602002604051908101604052809291906000905b828210156101055783826101800201600480602002604051908101604052809291906000905b828210156100f25783826060020160038060200260405190810160405280929190826003602002808284378201915050505050815260200190600101906100b0565b505050508152602001906001019061008a565b5050505091905050610208565b005b341561011f57600080fd5b61012761021d565b604051808260056000925b8184101561019b578284602002015160046000925b8184101561018d5782846020020151600360200280838360005b8381101561017c578082015181840152602081019050610161565b505050509050019260010192610147565b925050509260010192610132565b9250505091505060405180910390f35b34156101b657600080fd5b6101de6004808035906020019091908035906020019091908035906020019091905050610309565b604051808267ffffffffffffffff1667ffffffffffffffff16815260200191505060405180910390f35b80600090600561021992919061035f565b5050565b6102256103b0565b6000600580602002604051908101604052809291906000905b8282101561030057838260040201600480602002604051908101604052809291906000905b828210156102ed578382016003806020026040519081016040528092919082600380156102d9576020028201916000905b82829054906101000a900467ffffffffffffffff1667ffffffffffffffff16815260200190600801906020826007010492830192600103820291508084116102945790505b505050505081526020019060010190610263565b505050508152602001906001019061023e565b50505050905090565b60008360058110151561031857fe5b600402018260048110151561032957fe5b018160038110151561033757fe5b6004918282040191900660080292509250509054906101000a900467ffffffffffffffff1681565b826005600402810192821561039f579160200282015b8281111561039e5782518290600461038e9291906103df565b5091602001919060040190610375565b5b5090506103ac919061042d565b5090565b610780604051908101604052806005905b6103c9610459565b8152602001906001900390816103c15790505090565b826004810192821561041c579160200282015b8281111561041b5782518290600361040b929190610488565b50916020019190600101906103f2565b5b5090506104299190610536565b5090565b61045691905b8082111561045257600081816104499190610562565b50600401610433565b5090565b90565b610180604051908101604052806004905b6104726105a7565b81526020019060019003908161046a5790505090565b82600380016004900481019282156105255791602002820160005b838211156104ef57835183826101000a81548167ffffffffffffffff021916908367ffffffffffffffff16021790555092602001926008016020816007010492830192600103026104a3565b80156105235782816101000a81549067ffffffffffffffff02191690556008016020816007010492830192600103026104ef565b505b50905061053291906105d9565b5090565b61055f91905b8082111561055b57600081816105529190610610565b5060010161053c565b5090565b90565b50600081816105719190610610565b50600101600081816105839190610610565b50600101600081816105959190610610565b5060010160006105a59190610610565b565b6060604051908101604052806003905b600067ffffffffffffffff168152602001906001900390816105b75790505090565b61060d91905b8082111561060957600081816101000a81549067ffffffffffffffff0219169055506001016105df565b5090565b90565b50600090555600a165627a7a7230582087e5a43f6965ab6ef7a4ff056ab80ed78fd8c15cff57715a1bf34ec76a93661c0029`, + `[{"constant":false,"inputs":[{"name":"arr","type":"uint64[3][4][5]"}],"name":"storeDeepUintArray","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"retrieveDeepArray","outputs":[{"name":"","type":"uint64[3][4][5]"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"uint256"}],"name":"deepUint64Array","outputs":[{"name":"","type":"uint64"}],"payable":false,"stateMutability":"view","type":"function"}]`, + ` + // Generate a new random account and a funded simulator + key, _ := crypto.GenerateKey() + auth := bind.NewKeyedTransactor(key) + sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}) + + //deploy the test contract + _, _, testContract, err := DeployDeeplyNestedArray(auth, sim) + if err != nil { + t.Fatalf("Failed to deploy test contract: %v", err) + } + + // Finish deploy. + sim.Commit() + + //Create coordinate-filled array, for testing purposes. + testArr := [5][4][3]uint64{} + for i := 0; i < 5; i++ { + testArr[i] = [4][3]uint64{} + for j := 0; j < 4; j++ { + testArr[i][j] = [3]uint64{} + for k := 0; k < 3; k++ { + //pack the coordinates, each array value will be unique, and can be validated easily. + testArr[i][j][k] = uint64(i) << 16 | uint64(j) << 8 | uint64(k) + } + } + } + + if _, err := testContract.StoreDeepUintArray(&bind.TransactOpts{ + From: auth.From, + Signer: auth.Signer, + }, testArr); err != nil { + t.Fatalf("Failed to store nested array in test contract: %v", err) + } + + sim.Commit() + + retrievedArr, err := testContract.RetrieveDeepArray(&bind.CallOpts{ + From: auth.From, + Pending: false, + }) + if err != nil { + t.Fatalf("Failed to retrieve nested array from test contract: %v", err) + } + + //quick check to see if contents were copied + // (See accounts/abi/unpack_test.go for more extensive testing) + if retrievedArr[4][3][2] != testArr[4][3][2] { + t.Fatalf("Retrieved value does not match expected value! got: %d, expected: %d. %v", retrievedArr[4][3][2], testArr[4][3][2], err) + }`, + }, } // Tests that packages generated by the binder can be successfully compiled and diff --git a/accounts/abi/pack_test.go b/accounts/abi/pack_test.go index 14ab516ac2..58a5b7a581 100644 --- a/accounts/abi/pack_test.go +++ b/accounts/abi/pack_test.go @@ -299,6 +299,11 @@ func TestPack(t *testing.T) { [32]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, + { + "uint32[2][3][4]", + [4][3][2]uint32{{{1, 2}, {3, 4}, {5, 6}}, {{7, 8}, {9, 10}, {11, 12}}, {{13, 14}, {15, 16}, {17, 18}}, {{19, 20}, {21, 22}, {23, 24}}}, + common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000050000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000000000000000000000700000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000009000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000b000000000000000000000000000000000000000000000000000000000000000c000000000000000000000000000000000000000000000000000000000000000d000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000000f000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000110000000000000000000000000000000000000000000000000000000000000012000000000000000000000000000000000000000000000000000000000000001300000000000000000000000000000000000000000000000000000000000000140000000000000000000000000000000000000000000000000000000000000015000000000000000000000000000000000000000000000000000000000000001600000000000000000000000000000000000000000000000000000000000000170000000000000000000000000000000000000000000000000000000000000018"), + }, { "address[]", []common.Address{{1}, {2}}, diff --git a/accounts/abi/unpack.go b/accounts/abi/unpack.go index 761c80edfd..793d515adf 100644 --- a/accounts/abi/unpack.go +++ b/accounts/abi/unpack.go @@ -93,6 +93,17 @@ func readFixedBytes(t Type, word []byte) (interface{}, error) { } +func getFullElemSize(elem *Type) int { + //all other should be counted as 32 (slices have pointers to respective elements) + size := 32 + //arrays wrap it, each element being the same size + for elem.T == ArrayTy { + size *= elem.Size + elem = elem.Elem + } + return size +} + // iteratively unpack elements func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error) { if size < 0 { @@ -104,7 +115,6 @@ func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error) // this value will become our slice or our array, depending on the type var refSlice reflect.Value - slice := output[start : start+size*32] if t.T == SliceTy { // declare our slice @@ -116,15 +126,20 @@ func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error) return nil, fmt.Errorf("abi: invalid type in array/slice unpacking stage") } - for i, j := start, 0; j*32 < len(slice); i, j = i+32, j+1 { - // this corrects the arrangement so that we get all the underlying array values - if t.Elem.T == ArrayTy && j != 0 { - i = start + t.Elem.Size*32*j - } + // Arrays have packed elements, resulting in longer unpack steps. + // Slices have just 32 bytes per element (pointing to the contents). + elemSize := 32 + if t.T == ArrayTy { + elemSize = getFullElemSize(t.Elem) + } + + for i, j := start, 0; j < size; i, j = i+elemSize, j+1 { + inter, err := toGoType(i, *t.Elem, output) if err != nil { return nil, err } + // append the item to our reflect slice refSlice.Index(j).Set(reflect.ValueOf(inter)) } diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go index 742211244b..ee62567094 100644 --- a/accounts/abi/unpack_test.go +++ b/accounts/abi/unpack_test.go @@ -189,6 +189,11 @@ var unpackTests = []unpackTest{ enc: "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002", want: [2]uint32{1, 2}, }, + { + def: `[{"type": "uint32[2][3][4]"}]`, + enc: "000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000050000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000000000000000000000700000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000009000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000b000000000000000000000000000000000000000000000000000000000000000c000000000000000000000000000000000000000000000000000000000000000d000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000000f000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000110000000000000000000000000000000000000000000000000000000000000012000000000000000000000000000000000000000000000000000000000000001300000000000000000000000000000000000000000000000000000000000000140000000000000000000000000000000000000000000000000000000000000015000000000000000000000000000000000000000000000000000000000000001600000000000000000000000000000000000000000000000000000000000000170000000000000000000000000000000000000000000000000000000000000018", + want: [4][3][2]uint32{{{1, 2}, {3, 4}, {5, 6}}, {{7, 8}, {9, 10}, {11, 12}}, {{13, 14}, {15, 16}, {17, 18}}, {{19, 20}, {21, 22}, {23, 24}}}, + }, { def: `[{"type": "uint64[]"}]`, enc: "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002", @@ -435,6 +440,46 @@ func TestMultiReturnWithArray(t *testing.T) { } } +func TestMultiReturnWithDeeplyNestedArray(t *testing.T) { + // Similar to TestMultiReturnWithArray, but with a special case in mind: + // values of nested static arrays count towards the size as well, and any element following + // after such nested array argument should be read with the correct offset, + // so that it does not read content from the previous array argument. + const definition = `[{"name" : "multi", "outputs": [{"type": "uint64[3][2][4]"}, {"type": "uint64"}]}]` + abi, err := JSON(strings.NewReader(definition)) + if err != nil { + t.Fatal(err) + } + buff := new(bytes.Buffer) + // construct the test array, each 3 char element is joined with 61 '0' chars, + // to from the ((3 + 61) * 0.5) = 32 byte elements in the array. + buff.Write(common.Hex2Bytes(strings.Join([]string{ + "", //empty, to apply the 61-char separator to the first element as well. + "111", "112", "113", "121", "122", "123", + "211", "212", "213", "221", "222", "223", + "311", "312", "313", "321", "322", "323", + "411", "412", "413", "421", "422", "423", + }, "0000000000000000000000000000000000000000000000000000000000000"))) + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000009876")) + + ret1, ret1Exp := new([4][2][3]uint64), [4][2][3]uint64{ + {{0x111, 0x112, 0x113}, {0x121, 0x122, 0x123}}, + {{0x211, 0x212, 0x213}, {0x221, 0x222, 0x223}}, + {{0x311, 0x312, 0x313}, {0x321, 0x322, 0x323}}, + {{0x411, 0x412, 0x413}, {0x421, 0x422, 0x423}}, + } + ret2, ret2Exp := new(uint64), uint64(0x9876) + if err := abi.Unpack(&[]interface{}{ret1, ret2}, "multi", buff.Bytes()); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(*ret1, ret1Exp) { + t.Error("array result", *ret1, "!= Expected", ret1Exp) + } + if *ret2 != ret2Exp { + t.Error("int result", *ret2, "!= Expected", ret2Exp) + } +} + func TestUnmarshal(t *testing.T) { const definition = `[ { "name" : "int", "constant" : false, "outputs": [ { "type": "uint256" } ] }, From 61a061c9b4a5dd789b936ba3607f3617db361d1e Mon Sep 17 00:00:00 2001 From: Vlad Date: Sun, 4 Mar 2018 23:30:18 +0100 Subject: [PATCH 32/33] whisper: refactoring go-routines --- cmd/wnode/main.go | 56 +++++++++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/cmd/wnode/main.go b/cmd/wnode/main.go index 00a854fe87..84bdfa4c35 100644 --- a/cmd/wnode/main.go +++ b/cmd/wnode/main.go @@ -110,6 +110,7 @@ func main() { processArgs() initialize() run() + shutdown() } func processArgs() { @@ -209,21 +210,6 @@ func initialize() { MinimumAcceptedPOW: *argPoW, } - if *mailServerMode { - if len(msPassword) == 0 { - msPassword, err = console.Stdin.PromptPassword("Please enter the Mail Server password: ") - if err != nil { - utils.Fatalf("Failed to read Mail Server password: %s", err) - } - } - - shh = whisper.New(cfg) - shh.RegisterServer(&mailServer) - mailServer.Init(shh, *argDBPath, msPassword, *argServerPoW) - } else { - shh = whisper.New(cfg) - } - if *argPoW != whisper.DefaultMinimumPoW { err := shh.SetMinimumPoW(*argPoW) if err != nil { @@ -265,6 +251,26 @@ func initialize() { maxPeers = 800 } + _, err = crand.Read(entropy[:]) + if err != nil { + utils.Fatalf("crypto/rand failed: %s", err) + } + + if *mailServerMode { + if len(msPassword) == 0 { + msPassword, err = console.Stdin.PromptPassword("Please enter the Mail Server password: ") + if err != nil { + utils.Fatalf("Failed to read Mail Server password: %s", err) + } + } + + shh = whisper.New(cfg) + shh.RegisterServer(&mailServer) + mailServer.Init(shh, *argDBPath, msPassword, *argServerPoW) + } else { + shh = whisper.New(cfg) + } + server = &p2p.Server{ Config: p2p.Config{ PrivateKey: nodeid, @@ -278,17 +284,13 @@ func initialize() { TrustedNodes: peers, }, } - - _, err = crand.Read(entropy[:]) - if err != nil { - utils.Fatalf("crypto/rand failed: %s", err) - } } -func startServer() { +func startServer() error { err := server.Start() if err != nil { - utils.Fatalf("Failed to start Whisper peer: %s.", err) + fmt.Printf("Failed to start Whisper peer: %s.", err) + return err } fmt.Printf("my public key: %s \n", common.ToHex(crypto.FromECDSAPub(&asymKey.PublicKey))) @@ -307,6 +309,7 @@ func startServer() { if !*forwarderMode { fmt.Printf("Please type the message. To quit type: '%s'\n", quitCommand) } + return nil } func isKeyValid(k *ecdsa.PublicKey) bool { @@ -420,8 +423,10 @@ func waitForConnection(timeout bool) { } func run() { - defer mailServer.Close() - startServer() + err := startServer() + if err != nil { + return + } defer server.Stop() shh.Start(nil) defer shh.Stop() @@ -439,8 +444,11 @@ func run() { } else { sendLoop() } +} +func shutdown() { close(done) + mailServer.Close() } func sendLoop() { From d429a92f09e476c431830cef48690dc3784153c7 Mon Sep 17 00:00:00 2001 From: Kyuntae Ethan Kim Date: Mon, 5 Mar 2018 18:32:56 +0900 Subject: [PATCH 33/33] consensus/ethash: fixed typo (#16253) --- consensus/ethash/consensus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 3f860bf473..99eec82211 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -355,7 +355,7 @@ func calcDifficultyByzantium(time uint64, parent *types.Header) *big.Int { if x.Cmp(params.MinimumDifficulty) < 0 { x.Set(params.MinimumDifficulty) } - // calculate a fake block numer for the ice-age delay: + // calculate a fake block number for the ice-age delay: // https://github.com/ethereum/EIPs/pull/669 // fake_block_number = min(0, block.number - 3_000_000 fakeBlockNumber := new(big.Int)